all the project copy

api/ai_index.py

@@ -0,0 +1,414 @@
+from datetime import datetime
+from html import unescape
+from pathlib import Path
+import re
+import sys
+
+import psycopg2
+from psycopg2.extras import execute_values
+
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+CONN_DIR = ROOT_DIR / "conn"
+if str(CONN_DIR) not in sys.path:
+    sys.path.insert(0, str(CONN_DIR))
+
+import config
+import mysql5
+
+
+TAG_RE = re.compile(r"<[^>]+>")
+SPACE_RE = re.compile(r"\s+")
+CODE_RE = re.compile(r"\b[A-Z]{1,8}[-_/]?\d{2,}[A-Z0-9._/-]*\b|\b[A-Z0-9_]{3,}\.[A-Z0-9_.$-]+\b", re.IGNORECASE)
+
+
+def get_pg_connection():
+    return psycopg2.connect(**config.POSTGRES_CONFIG)
+
+
+def clean_text(value):
+    if value is None:
+        return ""
+    text = unescape(str(value))
+    text = TAG_RE.sub(" ", text)
+    text = text.replace("\x00", " ")
+    return SPACE_RE.sub(" ", text).strip()
+
+
+def short_text(value, max_len=420):
+    text = clean_text(value)
+    if len(text) <= max_len:
+        return text
+    return text[:max_len].rsplit(" ", 1)[0] + "..."
+
+
+def extract_codes(*values):
+    codes = set()
+    for value in values:
+        for match in CODE_RE.findall(clean_text(value).upper()):
+            codes.add(match.strip(".,;:()[]{}"))
+    return sorted(code for code in codes if code)
+
+
+def build_index_document(row):
+    subject = clean_text(row.get("subject"))
+    problem = clean_text(row.get("problem"))
+    solution = clean_text(row.get("solution"))
+    metadata = " ".join(
+        clean_text(row.get(name))
+        for name in ("competence", "product", "area", "subarea", "client", "gravity", "status", "program", "program_type")
+        if row.get(name) is not None
+    )
+    codes = extract_codes(subject, problem, solution, metadata)
+    search_text = " ".join(part for part in (subject, problem, solution, metadata, " ".join(codes)) if part)
+    return {
+        "ticket_id": int(row["ticket_id"]),
+        "subject_text": subject,
+        "problem_text": problem,
+        "solution_text": solution,
+        "solution_preview": short_text(solution),
+        "metadata_text": metadata,
+        "search_text": search_text,
+        "codes": codes,
+    }
+
+
+def ensure_ai_schema():
+    connection = get_pg_connection()
+    cursor = connection.cursor()
+
+    cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
+
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS ticket_ai_index (
+            ticket_id BIGINT PRIMARY KEY,
+            subject_text TEXT NOT NULL DEFAULT '',
+            problem_text TEXT NOT NULL DEFAULT '',
+            solution_text TEXT NOT NULL DEFAULT '',
+            solution_preview TEXT NOT NULL DEFAULT '',
+            metadata_text TEXT NOT NULL DEFAULT '',
+            search_text TEXT NOT NULL DEFAULT '',
+            codes TEXT[] NOT NULL DEFAULT '{}',
+            competence_id INTEGER,
+            competence TEXT,
+            product_id INTEGER,
+            product TEXT,
+            area_id INTEGER,
+            area TEXT,
+            subarea_id INTEGER,
+            subarea TEXT,
+            client_id INTEGER,
+            client TEXT,
+            gravity_id INTEGER,
+            gravity TEXT,
+            status_id INTEGER,
+            status TEXT,
+            opened_at TIMESTAMP,
+            closed_at TIMESTAMP,
+            program TEXT,
+            program_type TEXT,
+            version_id INTEGER,
+            patch_id INTEGER,
+            classification_id INTEGER,
+            os_id INTEGER,
+            search_vector TSVECTOR,
+            source_updated_at TIMESTAMP,
+            indexed_at TIMESTAMP NOT NULL DEFAULT now()
+        )
+        """
+    )
+
+    cursor.execute(
+        """
+        CREATE OR REPLACE FUNCTION ticket_ai_index_search_vector_update()
+        RETURNS trigger AS $$
+        BEGIN
+            NEW.search_vector :=
+                setweight(to_tsvector('simple', COALESCE(NEW.subject_text, '')), 'A') ||
+                setweight(to_tsvector('simple', COALESCE(NEW.problem_text, '')), 'A') ||
+                setweight(to_tsvector('simple', COALESCE(NEW.solution_text, '')), 'B') ||
+                setweight(to_tsvector('simple', COALESCE(NEW.metadata_text, '')), 'C') ||
+                setweight(to_tsvector('simple', array_to_string(COALESCE(NEW.codes, '{}'), ' ')), 'A');
+            RETURN NEW;
+        END
+        $$ LANGUAGE plpgsql
+        """
+    )
+
+    cursor.execute(
+        """
+        DROP TRIGGER IF EXISTS ticket_ai_index_search_vector_trigger ON ticket_ai_index
+        """
+    )
+    cursor.execute(
+        """
+        CREATE TRIGGER ticket_ai_index_search_vector_trigger
+        BEFORE INSERT OR UPDATE ON ticket_ai_index
+        FOR EACH ROW EXECUTE FUNCTION ticket_ai_index_search_vector_update()
+        """
+    )
+
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_index_search_vector_idx ON ticket_ai_index USING GIN (search_vector)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_index_problem_trgm_idx ON ticket_ai_index USING GIN (problem_text gin_trgm_ops)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_index_subject_trgm_idx ON ticket_ai_index USING GIN (subject_text gin_trgm_ops)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_index_codes_idx ON ticket_ai_index USING GIN (codes)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_index_product_idx ON ticket_ai_index (product_id)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_index_area_idx ON ticket_ai_index (area_id)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_index_subarea_idx ON ticket_ai_index (subarea_id)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_index_competence_idx ON ticket_ai_index (competence_id)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_index_opened_idx ON ticket_ai_index (opened_at)")
+
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS ticket_ai_query_log (
+            id BIGSERIAL PRIMARY KEY,
+            created_at TIMESTAMP NOT NULL DEFAULT now(),
+            user_id BIGINT,
+            query_ticket_id BIGINT NOT NULL,
+            suggested_ticket_id BIGINT NOT NULL,
+            rank INTEGER NOT NULL,
+            total_score DOUBLE PRECISION NOT NULL,
+            lexical_score DOUBLE PRECISION NOT NULL DEFAULT 0,
+            trigram_score DOUBLE PRECISION NOT NULL DEFAULT 0,
+            code_score DOUBLE PRECISION NOT NULL DEFAULT 0,
+            metadata_score DOUBLE PRECISION NOT NULL DEFAULT 0,
+            feedback_score DOUBLE PRECISION NOT NULL DEFAULT 0,
+            confidence TEXT NOT NULL,
+            query_text TEXT NOT NULL,
+            reasons TEXT[] NOT NULL DEFAULT '{}'
+        )
+        """
+    )
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_query_log_query_idx ON ticket_ai_query_log (query_ticket_id)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_query_log_suggestion_idx ON ticket_ai_query_log (suggested_ticket_id)")
+    cursor.execute("ALTER TABLE ticket_ai_query_log ADD COLUMN IF NOT EXISTS code_score DOUBLE PRECISION NOT NULL DEFAULT 0")
+
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS ticket_ai_search_log (
+            id BIGSERIAL PRIMARY KEY,
+            created_at TIMESTAMP NOT NULL DEFAULT now(),
+            user_id BIGINT,
+            query_kind TEXT NOT NULL DEFAULT 'ticket',
+            query_ticket_id BIGINT,
+            query_text TEXT NOT NULL DEFAULT '',
+            filters JSONB NOT NULL DEFAULT '{}'::jsonb,
+            result_count INTEGER NOT NULL DEFAULT 0,
+            elapsed_ms INTEGER NOT NULL DEFAULT 0
+        )
+        """
+    )
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_search_log_created_idx ON ticket_ai_search_log (created_at)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_search_log_result_idx ON ticket_ai_search_log (result_count)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_search_log_ticket_idx ON ticket_ai_search_log (query_ticket_id)")
+
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS ticket_ai_feedback (
+            id BIGSERIAL PRIMARY KEY,
+            created_at TIMESTAMP NOT NULL DEFAULT now(),
+            user_id BIGINT,
+            query_ticket_id BIGINT NOT NULL,
+            suggested_ticket_id BIGINT NOT NULL,
+            feedback TEXT NOT NULL CHECK (feedback IN ('useful', 'not_useful', 'favorite', 'opened')),
+            note TEXT
+        )
+        """
+    )
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_feedback_pair_idx ON ticket_ai_feedback (query_ticket_id, suggested_ticket_id)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS ticket_ai_feedback_suggestion_idx ON ticket_ai_feedback (suggested_ticket_id)")
+
+    cursor.execute(
+        """
+        CREATE MATERIALIZED VIEW IF NOT EXISTS ticket_ai_feedback_summary AS
+        SELECT
+            suggested_ticket_id AS ticket_id,
+            SUM(CASE WHEN feedback IN ('useful', 'favorite', 'opened') THEN 1 ELSE 0 END) AS positive_count,
+            SUM(CASE WHEN feedback = 'not_useful' THEN 1 ELSE 0 END) AS negative_count
+        FROM ticket_ai_feedback
+        GROUP BY suggested_ticket_id
+        """
+    )
+    cursor.execute("CREATE UNIQUE INDEX IF NOT EXISTS ticket_ai_feedback_summary_ticket_idx ON ticket_ai_feedback_summary (ticket_id)")
+
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS ticket_bot_user_settings (
+            user_id BIGINT PRIMARY KEY,
+            notifications_enabled BOOLEAN NOT NULL DEFAULT TRUE,
+            quiet_start_hour INTEGER,
+            quiet_end_hour INTEGER,
+            digest_minutes INTEGER NOT NULL DEFAULT 0,
+            high_gravity_only BOOLEAN NOT NULL DEFAULT FALSE,
+            last_notification_at TIMESTAMP,
+            updated_at TIMESTAMP NOT NULL DEFAULT now(),
+            CHECK (quiet_start_hour IS NULL OR (quiet_start_hour >= 0 AND quiet_start_hour <= 23)),
+            CHECK (quiet_end_hour IS NULL OR (quiet_end_hour >= 0 AND quiet_end_hour <= 23)),
+            CHECK (digest_minutes >= 0)
+        )
+        """
+    )
+
+    connection.commit()
+    cursor.close()
+    connection.close()
+
+
+def _row_values(row):
+    doc = build_index_document(row)
+    return (
+        doc["ticket_id"],
+        doc["subject_text"],
+        doc["problem_text"],
+        doc["solution_text"],
+        doc["solution_preview"],
+        doc["metadata_text"],
+        doc["search_text"],
+        doc["codes"],
+        row.get("competence_id"),
+        row.get("competence"),
+        row.get("product_id"),
+        row.get("product"),
+        row.get("area_id"),
+        row.get("area"),
+        row.get("subarea_id"),
+        row.get("subarea"),
+        row.get("client_id"),
+        row.get("client"),
+        row.get("gravity_id"),
+        row.get("gravity"),
+        row.get("status_id"),
+        row.get("status"),
+        row.get("opened_at"),
+        row.get("closed_at"),
+        row.get("program"),
+        row.get("program_type"),
+        row.get("version_id"),
+        row.get("patch_id"),
+        row.get("classification_id"),
+        row.get("os_id"),
+        datetime.utcnow(),
+    )
+
+
+def upsert_ticket_batch(rows):
+    if not rows:
+        return 0
+
+    values = [_row_values(row) for row in rows]
+    connection = get_pg_connection()
+    cursor = connection.cursor()
+    execute_values(
+        cursor,
+        """
+        INSERT INTO ticket_ai_index (
+            ticket_id,
+            subject_text,
+            problem_text,
+            solution_text,
+            solution_preview,
+            metadata_text,
+            search_text,
+            codes,
+            competence_id,
+            competence,
+            product_id,
+            product,
+            area_id,
+            area,
+            subarea_id,
+            subarea,
+            client_id,
+            client,
+            gravity_id,
+            gravity,
+            status_id,
+            status,
+            opened_at,
+            closed_at,
+            program,
+            program_type,
+            version_id,
+            patch_id,
+            classification_id,
+            os_id,
+            source_updated_at
+        ) VALUES %s
+        ON CONFLICT (ticket_id) DO UPDATE SET
+            subject_text = EXCLUDED.subject_text,
+            problem_text = EXCLUDED.problem_text,
+            solution_text = EXCLUDED.solution_text,
+            solution_preview = EXCLUDED.solution_preview,
+            metadata_text = EXCLUDED.metadata_text,
+            search_text = EXCLUDED.search_text,
+            codes = EXCLUDED.codes,
+            competence_id = EXCLUDED.competence_id,
+            competence = EXCLUDED.competence,
+            product_id = EXCLUDED.product_id,
+            product = EXCLUDED.product,
+            area_id = EXCLUDED.area_id,
+            area = EXCLUDED.area,
+            subarea_id = EXCLUDED.subarea_id,
+            subarea = EXCLUDED.subarea,
+            client_id = EXCLUDED.client_id,
+            client = EXCLUDED.client,
+            gravity_id = EXCLUDED.gravity_id,
+            gravity = EXCLUDED.gravity,
+            status_id = EXCLUDED.status_id,
+            status = EXCLUDED.status,
+            opened_at = EXCLUDED.opened_at,
+            closed_at = EXCLUDED.closed_at,
+            program = EXCLUDED.program,
+            program_type = EXCLUDED.program_type,
+            version_id = EXCLUDED.version_id,
+            patch_id = EXCLUDED.patch_id,
+            classification_id = EXCLUDED.classification_id,
+            os_id = EXCLUDED.os_id,
+            source_updated_at = EXCLUDED.source_updated_at,
+            indexed_at = now()
+        """,
+        values,
+        page_size=500,
+    )
+    connection.commit()
+    cursor.close()
+    connection.close()
+    return len(rows)
+
+
+def rebuild_index(batch_size=1000, max_rows=None):
+    ensure_ai_schema()
+    last_ticket = 0
+    indexed = 0
+
+    while True:
+        rows = mysql5.fetch_solved_ticket_batch(last_ticket=last_ticket, limit=batch_size)
+        if not rows:
+            break
+        upsert_ticket_batch(rows)
+        indexed += len(rows)
+        last_ticket = int(rows[-1]["ticket_id"])
+        print(f"Indexed {indexed} tickets, last ticket {last_ticket}")
+        if max_rows is not None and indexed >= max_rows:
+            break
+
+    refresh_feedback_summary()
+    return indexed
+
+
+def refresh_feedback_summary():
+    connection = get_pg_connection()
+    cursor = connection.cursor()
+    cursor.execute("REFRESH MATERIALIZED VIEW ticket_ai_feedback_summary")
+    connection.commit()
+    cursor.close()
+    connection.close()
+
+
+if __name__ == "__main__":
+    batch_size = int(sys.argv[1]) if len(sys.argv) > 1 else 1000
+    max_rows = int(sys.argv[2]) if len(sys.argv) > 2 else None
+    print(f"Starting AI index rebuild with batch_size={batch_size}, max_rows={max_rows}")
+    total = rebuild_index(batch_size=batch_size, max_rows=max_rows)
+    print(f"Finished indexing {total} solved tickets")

api/ai_retriever.py

@@ -0,0 +1,926 @@
+from datetime import datetime
+from difflib import SequenceMatcher
+from pathlib import Path
+import re
+import sys
+import time
+import unicodedata
+
+import psycopg2
+from psycopg2.extras import DictCursor, Json
+
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+CONN_DIR = ROOT_DIR / "conn"
+if str(CONN_DIR) not in sys.path:
+    sys.path.insert(0, str(CONN_DIR))
+
+import config
+import mysql5
+from ai_index import clean_text, ensure_ai_schema, extract_codes, refresh_feedback_summary
+
+
+MIN_SCORE = 0.07
+CONFIDENCE_MIN_SCORE = {
+    "bassa": MIN_SCORE,
+    "media": 0.18,
+    "alta": 0.35,
+}
+DEFAULT_LIMIT = 5
+DIVERSITY_POOL_MULTIPLIER = 8
+MAX_SOLUTION_TOKEN_SIMILARITY = 0.62
+MAX_SOLUTION_TOKEN_OVERLAP = 0.65
+MAX_SOLUTION_TEXT_SIMILARITY = 0.70
+_SCHEMA_READY = False
+TOKEN_RE = re.compile(r"\b\w{3,}\b", re.IGNORECASE)
+SIGNATURE_SPACE_RE = re.compile(r"\s+")
+
+
+def get_pg_connection():
+    return psycopg2.connect(**config.POSTGRES_CONFIG)
+
+
+def ensure_runtime_schema():
+    global _SCHEMA_READY
+    if not _SCHEMA_READY:
+        ensure_ai_schema()
+        _SCHEMA_READY = True
+
+
+def _id_value(value):
+    if value is None or value == "":
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _normalize_int_list(value):
+    if value is None:
+        return []
+    if not isinstance(value, (list, tuple, set)):
+        value = [value]
+
+    values = []
+    for item in value:
+        parsed = _id_value(item)
+        if parsed is not None and parsed not in values:
+            values.append(parsed)
+    return values
+
+
+def _query_text(ticket):
+    parts = [
+        ticket.get("subject"),
+        ticket.get("problem"),
+        ticket.get("product"),
+        ticket.get("area"),
+        ticket.get("subarea"),
+        ticket.get("competence"),
+        ticket.get("program"),
+        ticket.get("program_type"),
+    ]
+    codes = extract_codes(*parts)
+    cleaned = clean_text(" ".join(str(p) for p in parts if p))
+    return " ".join(part for part in (cleaned, " ".join(codes)) if part)
+
+
+def _confidence(score):
+    if score >= CONFIDENCE_MIN_SCORE["alta"]:
+        return "Alta"
+    if score >= CONFIDENCE_MIN_SCORE["media"]:
+        return "Media"
+    return "Bassa"
+
+
+def _min_score_from_filters(filters):
+    filters = filters or {}
+    if filters.get("min_score") is not None:
+        try:
+            return float(filters["min_score"])
+        except (TypeError, ValueError):
+            return MIN_SCORE
+
+    confidence = str(filters.get("min_confidence") or "").lower()
+    return CONFIDENCE_MIN_SCORE.get(confidence, MIN_SCORE)
+
+
+def _solution_tokens(solution):
+    text = clean_text(solution).lower()
+    return set(TOKEN_RE.findall(text))
+
+
+def _solution_signature(solution):
+    text = clean_text(solution).lower()
+    text = "".join(
+        char for char in unicodedata.normalize("NFKD", text)
+        if not unicodedata.combining(char)
+    )
+    text = re.sub(r"\b(gentile|buongiorno|ciao|salve)\b", " ", text)
+    text = re.sub(r"\b(cliente|mauro|martina|lisa|cinzia|nadia)\b", " ", text)
+    return SIGNATURE_SPACE_RE.sub(" ", text).strip()
+
+
+def _jaccard_similarity(left, right):
+    if not left or not right:
+        return 0
+    return len(left & right) / len(left | right)
+
+
+def _overlap_similarity(left, right):
+    if not left or not right:
+        return 0
+    return len(left & right) / min(len(left), len(right))
+
+
+def _matches_filter(row, filters, key, row_key):
+    values = _normalize_int_list((filters or {}).get(key))
+    return values and row.get(row_key) in values
+
+
+def _passes_strict_metadata_filters(row, filters):
+    filters = filters or {}
+    filter_columns = (
+        ("product_ids", "product_id"),
+        ("area_ids", "area_id"),
+        ("subarea_ids", "subarea_id"),
+        ("competence_ids", "competence_id"),
+        ("client_ids", "client_id"),
+    )
+    for key, column in filter_columns:
+        values = _normalize_int_list(filters.get(key))
+        if not values:
+            values = _normalize_int_list(filters.get(column))
+        if values and row.get(column) not in values:
+            return False
+
+    excluded_clients = _normalize_int_list(filters.get("exclude_client_ids"))
+    if not excluded_clients:
+        excluded_clients = _normalize_int_list(filters.get("exclude_client_id"))
+    if excluded_clients and row.get("client_id") in excluded_clients:
+        return False
+
+    return True
+
+
+def _diverse_suggestions(rows, query_ticket, limit, filters=None):
+    selected = []
+    selected_solution_tokens = []
+    selected_solution_signatures = []
+    duplicates_skipped = 0
+    min_score = _min_score_from_filters(filters)
+
+    for row in rows:
+        row_dict = dict(row)
+        if not _passes_strict_metadata_filters(row_dict, filters):
+            continue
+        if row_dict["total_score"] < min_score:
+            continue
+
+        solution_tokens = _solution_tokens(row_dict.get("solution_preview"))
+        solution_signature = _solution_signature(row_dict.get("solution_preview"))
+        is_duplicate = any(
+            _jaccard_similarity(solution_tokens, existing_tokens) >= MAX_SOLUTION_TOKEN_SIMILARITY
+            or _overlap_similarity(solution_tokens, existing_tokens) >= MAX_SOLUTION_TOKEN_OVERLAP
+            or SequenceMatcher(None, solution_signature, existing_signature).ratio() >= MAX_SOLUTION_TEXT_SIMILARITY
+            for existing_tokens, existing_signature in zip(selected_solution_tokens, selected_solution_signatures)
+        )
+        if is_duplicate:
+            duplicates_skipped += 1
+            continue
+
+        row_dict["confidence"] = _confidence(row_dict["total_score"])
+        row_dict["reasons"] = _reasons(row_dict, query_ticket, filters)
+        row_dict["duplicates_skipped_before"] = duplicates_skipped
+        selected.append(row_dict)
+        selected_solution_tokens.append(solution_tokens)
+        selected_solution_signatures.append(solution_signature)
+
+        if len(selected) >= limit:
+            break
+
+    return selected
+
+
+def _reasons(row, query_ticket=None, filters=None):
+    query_ticket = query_ticket or {}
+    filters = filters or {}
+    reasons = []
+
+    if row["lexical_score"] >= 0.08:
+        reasons.append("termini esatti")
+    if row["trigram_score"] >= 0.12:
+        reasons.append("testo simile")
+    if row["code_score"] > 0:
+        reasons.append("stesso codice")
+
+    if _matches_filter(row, filters, "product_ids", "product_id"):
+        reasons.append("filtro prodotto")
+    elif row["product_id"] is not None and row["product_id"] == _id_value(query_ticket.get("product_id")):
+        reasons.append("stesso prodotto")
+
+    if _matches_filter(row, filters, "subarea_ids", "subarea_id"):
+        reasons.append("filtro sottoarea")
+    elif row["subarea_id"] is not None and row["subarea_id"] == _id_value(query_ticket.get("subarea_id")):
+        reasons.append("stessa sottoarea")
+    elif _matches_filter(row, filters, "area_ids", "area_id"):
+        reasons.append("filtro area")
+    elif row["area_id"] is not None and row["area_id"] == _id_value(query_ticket.get("area_id")):
+        reasons.append("stessa area")
+
+    if _matches_filter(row, filters, "competence_ids", "competence_id"):
+        reasons.append("filtro competenza")
+    elif row["competence_id"] is not None and row["competence_id"] == _id_value(query_ticket.get("competence_id")):
+        reasons.append("stessa competenza")
+
+    if _matches_filter(row, filters, "client_ids", "client_id"):
+        reasons.append("stesso cliente")
+    if row["feedback_score"] > 0:
+        reasons.append("feedback positivo")
+    if filters.get("recent_months"):
+        reasons.append("periodo recente")
+    if not reasons:
+        reasons.append("somiglianza generale")
+    return reasons[:5]
+
+
+def _format_date(value):
+    if not value:
+        return "-"
+    if hasattr(value, "strftime"):
+        return value.strftime("%Y-%m-%d")
+    return str(value)
+
+
+def _format_suggestion(query_ticket_id, suggestion, rank):
+    ticket_id = suggestion["ticket_id"]
+    link = f"https://tsnew.sanmarcoweb.com/it/ticket/index/index/operation/view/id/{ticket_id}"
+    product = suggestion.get("product") or "-"
+    area = suggestion.get("area") or "-"
+    subarea = suggestion.get("subarea") or "-"
+    competence = suggestion.get("competence") or "-"
+    client = suggestion.get("client") or "-"
+    confidence = suggestion["confidence"]
+    reasons = ", ".join(suggestion["reasons"])
+    solution = suggestion.get("solution_preview") or "Soluzione non disponibile"
+    diversity_note = ""
+    if suggestion.get("duplicates_skipped_before"):
+        diversity_note = f"Alternative simili saltate: {suggestion['duplicates_skipped_before']}\n"
+
+    feedback_line = ""
+    if query_ticket_id:
+        feedback_line = f"Feedback: /utile_{query_ticket_id}_{ticket_id} /non_utile_{query_ticket_id}_{ticket_id}\n"
+
+    return (
+        f"\n\n----- {rank}. Confidenza {confidence} -----\n"
+        f"[ /Ticket_dettaglio_{ticket_id} ]\n"
+        f"{link}\n"
+        f"Cliente: {client}\n"
+        f"Prodotto: {product} | Competenza: {competence}\n"
+        f"Area: {area} | Sottoarea: {subarea}\n"
+        f"Apertura: {_format_date(suggestion.get('opened_at'))}\n"
+        f"Perche: {reasons}\n"
+        f"{diversity_note}"
+        f"Score: {suggestion['total_score']:.3f}\n"
+        f"SOLUZIONE: {solution}\n"
+        f"{feedback_line}"
+    )
+
+
+def _has_metadata_filters(filters):
+    filters = filters or {}
+    return any(
+        _normalize_int_list(filters.get(key))
+        for key in ("product_ids", "area_ids", "subarea_ids", "competence_ids", "client_ids")
+    )
+
+
+def _metadata_candidate_clause(ticket, filters=None):
+    clauses = []
+    params = {}
+    for field in ("product_id", "area_id", "subarea_id", "competence_id"):
+        value = _id_value((ticket or {}).get(field))
+        params[field] = value
+        if value is not None:
+            clauses.append(f"i.{field} = %({field})s")
+    if _has_metadata_filters(filters):
+        clauses.append("TRUE")
+    if not clauses:
+        clauses.append("FALSE")
+    return " OR ".join(clauses), params
+
+
+def _strict_filter_clause(filters):
+    filters = filters or {}
+    clauses = []
+    params = {}
+
+    filter_columns = (
+        ("product_ids", "product_id"),
+        ("area_ids", "area_id"),
+        ("subarea_ids", "subarea_id"),
+        ("competence_ids", "competence_id"),
+        ("client_ids", "client_id"),
+    )
+    for key, column in filter_columns:
+        values = _normalize_int_list(filters.get(key))
+        if not values:
+            values = _normalize_int_list(filters.get(column))
+        if values:
+            clauses.append(f"i.{column} = ANY(%({key})s::int[])")
+            params[key] = values
+
+    excluded_clients = _normalize_int_list(filters.get("exclude_client_ids"))
+    if not excluded_clients:
+        excluded_clients = _normalize_int_list(filters.get("exclude_client_id"))
+    if excluded_clients:
+        clauses.append("NOT (i.client_id = ANY(%(exclude_client_ids)s::int[]))")
+        params["exclude_client_ids"] = excluded_clients
+
+    if filters.get("require_code_match"):
+        clauses.append("i.codes && %(query_codes)s::text[]")
+
+    recent_months = _id_value(filters.get("recent_months"))
+    if recent_months is not None and recent_months > 0:
+        clauses.append("i.opened_at >= now() - (%(recent_days)s::int * interval '1 day')")
+        params["recent_days"] = recent_months * 31
+
+    return " AND ".join(clauses) if clauses else "TRUE", params
+
+
+def _safe_filters(filters):
+    safe = {}
+    for key, value in (filters or {}).items():
+        if value is None or value is False or value == []:
+            continue
+        if isinstance(value, (list, tuple, set)):
+            safe[key] = list(value)
+        else:
+            safe[key] = value
+    return safe
+
+
+def _search(query_kind, query_text, query_ticket_id=None, query_ticket=None, user_id=None, limit=DEFAULT_LIMIT, offset=0, filters=None, log_results=True):
+    ensure_runtime_schema()
+    query_text = clean_text(query_text)
+    if not query_text:
+        return []
+
+    query_ticket = query_ticket or {}
+    filters = filters or {}
+    limit = max(int(limit or DEFAULT_LIMIT), 1)
+    offset = max(int(offset or 0), 0)
+    pool_target = offset + limit
+    query_codes = extract_codes(query_text)
+    metadata_clause, metadata_params = _metadata_candidate_clause(query_ticket, filters)
+    strict_clause, strict_params = _strict_filter_clause(filters)
+
+    params = {
+        "query_ticket_id": int(query_ticket_id) if query_ticket_id is not None else None,
+        "query_text": query_text,
+        "problem_text": clean_text(query_ticket.get("problem") or query_text),
+        "subject_text": clean_text(query_ticket.get("subject") or query_text),
+        "candidate_limit": max(pool_target * 50, 250),
+        "final_limit": max(pool_target * DIVERSITY_POOL_MULTIPLIER * 2, 40),
+        "query_product_id": _id_value(query_ticket.get("product_id")),
+        "query_area_id": _id_value(query_ticket.get("area_id")),
+        "query_subarea_id": _id_value(query_ticket.get("subarea_id")),
+        "query_competence_id": _id_value(query_ticket.get("competence_id")),
+        "query_codes": query_codes,
+    }
+    params.update(metadata_params)
+    params.update(strict_params)
+
+    sql = f"""
+        WITH q AS (
+            SELECT websearch_to_tsquery('simple', %(query_text)s) AS tsq
+        ),
+        lexical_candidates AS (
+            SELECT i.ticket_id
+            FROM ticket_ai_index i, q
+            WHERE (%(query_ticket_id)s IS NULL OR i.ticket_id <> %(query_ticket_id)s)
+              AND ({strict_clause})
+              AND i.search_vector @@ q.tsq
+            ORDER BY ts_rank_cd(i.search_vector, q.tsq) DESC
+            LIMIT %(candidate_limit)s
+        ),
+        metadata_candidates AS (
+            SELECT i.ticket_id
+            FROM ticket_ai_index i
+            WHERE (%(query_ticket_id)s IS NULL OR i.ticket_id <> %(query_ticket_id)s)
+              AND ({strict_clause})
+              AND ({metadata_clause})
+            ORDER BY i.opened_at DESC NULLS LAST
+            LIMIT %(candidate_limit)s
+        ),
+        code_candidates AS (
+            SELECT i.ticket_id
+            FROM ticket_ai_index i
+            WHERE (%(query_ticket_id)s IS NULL OR i.ticket_id <> %(query_ticket_id)s)
+              AND ({strict_clause})
+              AND i.codes && %(query_codes)s::text[]
+            ORDER BY i.opened_at DESC NULLS LAST
+            LIMIT %(candidate_limit)s
+        ),
+        candidate_ids AS (
+            SELECT ticket_id FROM lexical_candidates
+            UNION
+            SELECT ticket_id FROM code_candidates
+            UNION
+            SELECT ticket_id FROM metadata_candidates
+        ),
+        scored AS (
+            SELECT
+                i.ticket_id,
+                i.subject_text,
+                i.problem_text,
+                i.solution_preview,
+                i.competence_id,
+                i.competence,
+                i.product_id,
+                i.product,
+                i.area_id,
+                i.area,
+                i.subarea_id,
+                i.subarea,
+                i.client_id,
+                i.client,
+                i.opened_at,
+                ts_rank_cd(i.search_vector, q.tsq) AS lexical_score,
+                GREATEST(
+                    similarity(i.problem_text, %(problem_text)s),
+                    similarity(i.subject_text, %(subject_text)s)
+                ) AS trigram_score,
+                CASE WHEN i.codes && %(query_codes)s::text[] THEN 0.18 ELSE 0 END AS code_score,
+                (
+                    CASE WHEN i.product_id IS NOT DISTINCT FROM %(query_product_id)s THEN 0.045 ELSE 0 END +
+                    CASE WHEN i.subarea_id IS NOT DISTINCT FROM %(query_subarea_id)s THEN 0.035 ELSE 0 END +
+                    CASE WHEN i.area_id IS NOT DISTINCT FROM %(query_area_id)s THEN 0.020 ELSE 0 END +
+                    CASE WHEN i.competence_id IS NOT DISTINCT FROM %(query_competence_id)s THEN 0.015 ELSE 0 END
+                ) AS metadata_score,
+                LEAST(
+                    GREATEST(COALESCE(fs.positive_count, 0) - COALESCE(fs.negative_count, 0), 0) * 0.015,
+                    0.12
+                ) AS feedback_score
+            FROM candidate_ids c
+            JOIN ticket_ai_index i ON i.ticket_id = c.ticket_id
+            CROSS JOIN q
+            LEFT JOIN ticket_ai_feedback_summary fs ON fs.ticket_id = i.ticket_id
+        )
+        SELECT
+            *,
+            (
+                lexical_score * 0.62 +
+                trigram_score * 0.28 +
+                code_score +
+                metadata_score +
+                feedback_score
+            ) AS total_score
+        FROM scored
+        ORDER BY total_score DESC, lexical_score DESC, opened_at DESC NULLS LAST
+        LIMIT %(final_limit)s
+    """
+
+    started_at = time.perf_counter()
+    connection = get_pg_connection()
+    cursor = connection.cursor(cursor_factory=DictCursor)
+    cursor.execute(sql, params)
+    rows = cursor.fetchall()
+    all_suggestions = _diverse_suggestions(rows, query_ticket, pool_target, filters)
+    suggestions = all_suggestions[offset:offset + limit]
+    elapsed_ms = int((time.perf_counter() - started_at) * 1000)
+
+    if log_results:
+        _log_search(cursor, user_id, query_kind, query_ticket_id, query_text, filters, len(all_suggestions), elapsed_ms)
+        if query_kind == "ticket" and query_ticket_id is not None and suggestions:
+            _log_suggestions(cursor, user_id, int(query_ticket_id), query_text, suggestions, rank_offset=offset)
+        connection.commit()
+
+    cursor.close()
+    connection.close()
+    return suggestions
+
+
+def search_similar_tickets(query_ticket_id, user_id=None, limit=DEFAULT_LIMIT, offset=0, filters=None, log_results=True):
+    query_ticket = mysql5.get_ticket_context(query_ticket_id)
+    if not query_ticket:
+        return []
+
+    query_text = _query_text(query_ticket)
+    return _search(
+        "ticket",
+        query_text,
+        query_ticket_id=int(query_ticket_id),
+        query_ticket=query_ticket,
+        user_id=user_id,
+        limit=limit,
+        offset=offset,
+        filters=filters,
+        log_results=log_results,
+    )
+
+
+def search_similar_text(query_text, user_id=None, limit=DEFAULT_LIMIT, offset=0, filters=None, log_results=True):
+    query_text = clean_text(query_text)
+    if not query_text:
+        return []
+    return _search(
+        "text",
+        query_text,
+        query_ticket_id=None,
+        query_ticket={},
+        user_id=user_id,
+        limit=limit,
+        offset=offset,
+        filters=filters,
+        log_results=log_results,
+    )
+
+
+def _log_search(cursor, user_id, query_kind, query_ticket_id, query_text, filters, result_count, elapsed_ms):
+    cursor.execute(
+        """
+        INSERT INTO ticket_ai_search_log (
+            user_id,
+            query_kind,
+            query_ticket_id,
+            query_text,
+            filters,
+            result_count,
+            elapsed_ms
+        ) VALUES (%s, %s, %s, %s, %s, %s, %s)
+        """,
+        (
+            user_id,
+            query_kind,
+            int(query_ticket_id) if query_ticket_id is not None else None,
+            query_text,
+            Json(_safe_filters(filters)),
+            int(result_count),
+            int(elapsed_ms),
+        ),
+    )
+
+
+def _log_suggestions(cursor, user_id, query_ticket_id, query_text, suggestions, rank_offset=0):
+    for rank, suggestion in enumerate(suggestions, start=1 + rank_offset):
+        cursor.execute(
+            """
+            INSERT INTO ticket_ai_query_log (
+                user_id,
+                query_ticket_id,
+                suggested_ticket_id,
+                rank,
+                total_score,
+                lexical_score,
+                trigram_score,
+                code_score,
+                metadata_score,
+                feedback_score,
+                confidence,
+                query_text,
+                reasons
+            ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+            """,
+            (
+                user_id,
+                query_ticket_id,
+                suggestion["ticket_id"],
+                rank,
+                float(suggestion["total_score"]),
+                float(suggestion["lexical_score"]),
+                float(suggestion["trigram_score"]),
+                float(suggestion["code_score"]),
+                float(suggestion["metadata_score"]),
+                float(suggestion["feedback_score"]),
+                suggestion["confidence"],
+                query_text,
+                suggestion["reasons"],
+            ),
+        )
+
+
+def record_feedback(user_id, query_ticket_id, suggested_ticket_id, feedback, note=None):
+    ensure_runtime_schema()
+    if feedback not in ("useful", "not_useful", "favorite", "opened"):
+        raise ValueError("feedback must be useful, not_useful, favorite, or opened")
+
+    connection = get_pg_connection()
+    cursor = connection.cursor()
+    cursor.execute(
+        """
+        INSERT INTO ticket_ai_feedback (
+            user_id,
+            query_ticket_id,
+            suggested_ticket_id,
+            feedback,
+            note
+        ) VALUES (%s, %s, %s, %s, %s)
+        """,
+        (user_id, int(query_ticket_id), int(suggested_ticket_id), feedback, note),
+    )
+    connection.commit()
+    cursor.close()
+    connection.close()
+    refresh_feedback_summary()
+    return True
+
+
+def record_favorite(user_id, ticket_id, query_ticket_id=None, note=None):
+    ensure_runtime_schema()
+    ticket_id = int(ticket_id)
+    query_ticket_id = int(query_ticket_id) if query_ticket_id is not None else ticket_id
+
+    connection = get_pg_connection()
+    cursor = connection.cursor()
+    cursor.execute(
+        """
+        SELECT 1
+        FROM ticket_ai_feedback
+        WHERE user_id = %s
+          AND suggested_ticket_id = %s
+          AND feedback = 'favorite'
+        LIMIT 1
+        """,
+        (user_id, ticket_id),
+    )
+    if cursor.fetchone():
+        cursor.close()
+        connection.close()
+        return False
+
+    cursor.execute(
+        """
+        INSERT INTO ticket_ai_feedback (
+            user_id,
+            query_ticket_id,
+            suggested_ticket_id,
+            feedback,
+            note
+        ) VALUES (%s, %s, %s, 'favorite', %s)
+        """,
+        (user_id, query_ticket_id, ticket_id, note),
+    )
+    connection.commit()
+    cursor.close()
+    connection.close()
+    refresh_feedback_summary()
+    return True
+
+
+def get_user_favorites(user_id, limit=20):
+    ensure_runtime_schema()
+    connection = get_pg_connection()
+    cursor = connection.cursor(cursor_factory=DictCursor)
+    cursor.execute(
+        """
+        SELECT *
+        FROM (
+            SELECT DISTINCT ON (f.suggested_ticket_id)
+                f.suggested_ticket_id AS ticket_id,
+                f.query_ticket_id,
+                f.created_at,
+                i.product,
+                i.area,
+                i.subject_text
+            FROM ticket_ai_feedback f
+            LEFT JOIN ticket_ai_index i ON i.ticket_id = f.suggested_ticket_id
+            WHERE f.user_id = %s
+              AND f.feedback = 'favorite'
+            ORDER BY f.suggested_ticket_id, f.created_at DESC
+        ) favorites
+        ORDER BY created_at DESC
+        LIMIT %s
+        """,
+        (user_id, int(limit)),
+    )
+    rows = [dict(row) for row in cursor.fetchall()]
+    cursor.close()
+    connection.close()
+    return rows
+
+
+def get_user_bot_settings(user_id):
+    ensure_runtime_schema()
+    connection = get_pg_connection()
+    cursor = connection.cursor(cursor_factory=DictCursor)
+    cursor.execute(
+        """
+        INSERT INTO ticket_bot_user_settings (user_id)
+        VALUES (%s)
+        ON CONFLICT (user_id) DO NOTHING
+        """,
+        (int(user_id),),
+    )
+    cursor.execute(
+        """
+        SELECT *
+        FROM ticket_bot_user_settings
+        WHERE user_id = %s
+        """,
+        (int(user_id),),
+    )
+    settings = dict(cursor.fetchone())
+    connection.commit()
+    cursor.close()
+    connection.close()
+    return settings
+
+
+def update_user_bot_settings(user_id, **updates):
+    allowed = {
+        "notifications_enabled",
+        "quiet_start_hour",
+        "quiet_end_hour",
+        "digest_minutes",
+        "high_gravity_only",
+        "last_notification_at",
+    }
+    clean_updates = {key: value for key, value in updates.items() if key in allowed}
+    if not clean_updates:
+        return get_user_bot_settings(user_id)
+
+    get_user_bot_settings(user_id)
+    assignments = [f"{key} = %s" for key in clean_updates]
+    values = list(clean_updates.values())
+    values.append(int(user_id))
+
+    connection = get_pg_connection()
+    cursor = connection.cursor()
+    cursor.execute(
+        f"""
+        UPDATE ticket_bot_user_settings
+        SET {", ".join(assignments)},
+            updated_at = now()
+        WHERE user_id = %s
+        """,
+        values,
+    )
+    connection.commit()
+    cursor.close()
+    connection.close()
+    return get_user_bot_settings(user_id)
+
+
+def set_notifications_enabled(user_id, enabled):
+    return update_user_bot_settings(user_id, notifications_enabled=bool(enabled))
+
+
+def set_quiet_hours(user_id, start_hour, end_hour):
+    return update_user_bot_settings(
+        user_id,
+        quiet_start_hour=max(0, min(23, int(start_hour))),
+        quiet_end_hour=max(0, min(23, int(end_hour))),
+    )
+
+
+def clear_quiet_hours(user_id):
+    return update_user_bot_settings(user_id, quiet_start_hour=None, quiet_end_hour=None)
+
+
+def set_digest_minutes(user_id, minutes):
+    return update_user_bot_settings(user_id, digest_minutes=max(0, int(minutes)))
+
+
+def set_high_gravity_only(user_id, enabled):
+    return update_user_bot_settings(user_id, high_gravity_only=bool(enabled))
+
+
+def _is_quiet_now(settings, now=None):
+    start = settings.get("quiet_start_hour")
+    end = settings.get("quiet_end_hour")
+    if start is None or end is None or start == end:
+        return False
+
+    hour = (now or datetime.now()).hour
+    if start < end:
+        return start <= hour < end
+    return hour >= start or hour < end
+
+
+def should_send_notifications(user_id, now=None):
+    settings = get_user_bot_settings(user_id)
+    now = now or datetime.now()
+    if not settings.get("notifications_enabled", True):
+        return False
+    if _is_quiet_now(settings, now=now):
+        return False
+
+    digest_minutes = int(settings.get("digest_minutes") or 0)
+    last_notification_at = settings.get("last_notification_at")
+    if digest_minutes > 0 and last_notification_at:
+        elapsed_seconds = (now - last_notification_at).total_seconds()
+        if elapsed_seconds < digest_minutes * 60:
+            return False
+    return True
+
+
+def mark_notification_sent(user_id):
+    return update_user_bot_settings(user_id, last_notification_at=datetime.now())
+
+
+def get_ai_stats():
+    ensure_runtime_schema()
+    connection = get_pg_connection()
+    cursor = connection.cursor(cursor_factory=DictCursor)
+    cursor.execute("SELECT COUNT(*) AS indexed_count, MAX(indexed_at) AS last_indexed_at FROM ticket_ai_index")
+    index_stats = dict(cursor.fetchone())
+    cursor.execute(
+        """
+        SELECT
+            COUNT(*) AS search_count,
+            COALESCE(AVG(elapsed_ms), 0)::integer AS avg_elapsed_ms,
+            COALESCE(SUM(CASE WHEN result_count = 0 THEN 1 ELSE 0 END), 0) AS no_result_count
+        FROM ticket_ai_search_log
+        """
+    )
+    search_stats = dict(cursor.fetchone())
+    cursor.execute("SELECT feedback, COUNT(*) AS count FROM ticket_ai_feedback GROUP BY feedback")
+    feedback_counts = {row["feedback"]: row["count"] for row in cursor.fetchall()}
+    cursor.close()
+    connection.close()
+    return {
+        **index_stats,
+        **search_stats,
+        "feedback_counts": feedback_counts,
+    }
+
+
+def get_no_result_searches(limit=10):
+    ensure_runtime_schema()
+    connection = get_pg_connection()
+    cursor = connection.cursor(cursor_factory=DictCursor)
+    cursor.execute(
+        """
+        SELECT created_at, user_id, query_kind, query_ticket_id, query_text, filters, elapsed_ms
+        FROM ticket_ai_search_log
+        WHERE result_count = 0
+        ORDER BY created_at DESC
+        LIMIT %s
+        """,
+        (int(limit),),
+    )
+    rows = [dict(row) for row in cursor.fetchall()]
+    cursor.close()
+    connection.close()
+    return rows
+
+
+def get_negative_feedback(limit=10):
+    ensure_runtime_schema()
+    connection = get_pg_connection()
+    cursor = connection.cursor(cursor_factory=DictCursor)
+    cursor.execute(
+        """
+        SELECT
+            f.created_at,
+            f.user_id,
+            f.query_ticket_id,
+            f.suggested_ticket_id,
+            i.product,
+            i.area,
+            i.subject_text
+        FROM ticket_ai_feedback f
+        LEFT JOIN ticket_ai_index i ON i.ticket_id = f.suggested_ticket_id
+        WHERE f.feedback = 'not_useful'
+        ORDER BY f.created_at DESC
+        LIMIT %s
+        """,
+        (int(limit),),
+    )
+    rows = [dict(row) for row in cursor.fetchall()]
+    cursor.close()
+    connection.close()
+    return rows
+
+
+def get_top_useful(limit=10):
+    ensure_runtime_schema()
+    connection = get_pg_connection()
+    cursor = connection.cursor(cursor_factory=DictCursor)
+    cursor.execute(
+        """
+        SELECT
+            fs.ticket_id,
+            fs.positive_count,
+            fs.negative_count,
+            i.product,
+            i.area,
+            i.subject_text
+        FROM ticket_ai_feedback_summary fs
+        LEFT JOIN ticket_ai_index i ON i.ticket_id = fs.ticket_id
+        ORDER BY fs.positive_count DESC, fs.negative_count ASC
+        LIMIT %s
+        """,
+        (int(limit),),
+    )
+    rows = [dict(row) for row in cursor.fetchall()]
+    cursor.close()
+    connection.close()
+    return rows
+
+
+def start_guessing_manual(ticket, user_id=None, filters=None, limit=DEFAULT_LIMIT, offset=0):
+    suggestions = search_similar_tickets(ticket, user_id=user_id, limit=limit, offset=offset, filters=filters)
+    return [_format_suggestion(int(ticket), suggestion, rank + offset) for rank, suggestion in enumerate(suggestions, start=1)]

api/bootstrap_db.py

@@ -0,0 +1,66 @@
+from pathlib import Path
+import sys
+
+import psycopg2
+
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+CONN_DIR = ROOT_DIR / "conn"
+API_DIR = ROOT_DIR / "api"
+for path in (CONN_DIR, API_DIR):
+    if str(path) not in sys.path:
+        sys.path.insert(0, str(path))
+
+import config
+from ai_index import ensure_ai_schema
+
+
+BOT_SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS users (
+    tid BIGINT PRIMARY KEY,
+    username TEXT,
+    surname TEXT,
+    status CHAR(1) NOT NULL DEFAULT 'P',
+    user_admin BOOLEAN NOT NULL DEFAULT FALSE,
+    created_at TIMESTAMP NOT NULL DEFAULT now(),
+    updated_at TIMESTAMP NOT NULL DEFAULT now(),
+    CHECK (status IN ('P', 'A'))
+);
+
+CREATE TABLE IF NOT EXISTS filters (
+    fid BIGSERIAL PRIMARY KEY,
+    tid BIGINT NOT NULL REFERENCES users(tid) ON DELETE CASCADE,
+    sort CHAR(1) NOT NULL,
+    sort_value TEXT NOT NULL,
+    created_at TIMESTAMP NOT NULL DEFAULT now(),
+    CHECK (sort IN ('A', 'S', 'C', 'P')),
+    UNIQUE (tid, sort, sort_value)
+);
+
+CREATE INDEX IF NOT EXISTS filters_tid_idx ON filters (tid);
+CREATE INDEX IF NOT EXISTS filters_sort_idx ON filters (sort);
+
+CREATE TABLE IF NOT EXISTS tickets (
+    fid BIGINT NOT NULL REFERENCES filters(fid) ON DELETE CASCADE,
+    ticket BIGINT NOT NULL,
+    created_at TIMESTAMP NOT NULL DEFAULT now(),
+    PRIMARY KEY (fid, ticket)
+);
+
+CREATE INDEX IF NOT EXISTS tickets_ticket_idx ON tickets (ticket);
+"""
+
+
+def ensure_bot_schema():
+    connection = psycopg2.connect(**config.POSTGRES_CONFIG)
+    cursor = connection.cursor()
+    cursor.execute(BOT_SCHEMA_SQL)
+    connection.commit()
+    cursor.close()
+    connection.close()
+
+
+if __name__ == "__main__":
+    ensure_bot_schema()
+    ensure_ai_schema()
+    print("PostgreSQL schema is ready")

api/cockie.py

@@ -0,0 +1,2 @@
+
+id = "r0gmffchr3kpq77ls6nish7o92"

api/connection.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+import sys
+
+import psycopg2
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+CONN_DIR = ROOT_DIR / "conn"
+if str(CONN_DIR) not in sys.path:
+    sys.path.insert(0, str(CONN_DIR))
+
+import config
+
+
+
+
+
+def conn():
+    try:
+        connection = psycopg2.connect(**config.POSTGRES_CONFIG)
+
+        return connection
+        # cursor = connection.cursor()
+
+        # postgres_insert_query = """ INSERT INTO mobile (ID, MODEL, PRICE) VALUES (%s,%s,%s)"""
+        # record_to_insert = (5, 'One Plus 6', 950)
+        # cursor.execute(postgres_insert_query, record_to_insert)
+
+        # connection.commit()
+        # count = cursor.rowcount
+        # print(count, "Record inserted successfully into mobile table")
+
+    except (Exception, psycopg2.Error) as error:
+        print("Failed to insert record into mobile table", error)
+        return False
+
+    # finally:
+    #     # closing database connection.
+    #     if connection:
+    #         connection.cursor().close()
+    #         connection.close()
+    #         print("PostgreSQL connection is closed")
+

api/detailTicket.py

@@ -0,0 +1,105 @@
+import requests
+import os
+
+
+import traceback
+import logging
+import cockie
+import time
+
+
+
+
+
+def deleteDocs(ticketId):
+    time.sleep(2)
+    if os.path.exists("json/detailTicket.html"):
+        os.remove("json/detailTicket.html")
+        # os.remove('json/' + str(ticketId)+'_details.pdf')
+
+
+def getDetail(ticketId):
+    print("arrivato con :   " + str(ticketId))
+
+    pdf_file = "json/" + str(ticketId) + "_details.pdf"
+    html_file = "json/" + str(ticketId) + "_details.html"
+    try:
+
+        cookies = {
+            'language': 'it',
+            'sid': cockie.id,
+        }
+
+        headers = {
+            'Accept': '*/*',
+            'Accept-Language': 'it-IT,it;q=0.9,la;q=0.8',
+            'Connection': 'keep-alive',
+            'Content-Type': 'application/x-www-form-urlencoded',
+            # Requests sorts cookies= alphabetically
+            # 'Cookie': 'language=it; sid=89q0fl4j3r54600e2fdbf952t1',
+            'Referer': 'https://tsnew.sanmarcoweb.com/it/ticket',
+            'Sec-Fetch-Dest': 'empty',
+            'Sec-Fetch-Mode': 'cors',
+            'Sec-Fetch-Site': 'same-origin',
+            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36',
+            'X-Requested-With': 'XMLHttpRequest',
+            'sec-ch-ua': '"Chromium";v="104", " Not A;Brand";v="99", "Google Chrome";v="104"',
+            'sec-ch-ua-mobile': '?0',
+            'sec-ch-ua-platform': '"Linux"',
+        }   
+
+        response = requests.get('https://tsnew.sanmarcoweb.com/it/ticket/details/index/id/'+ str(ticketId), cookies=cookies, headers=headers)
+
+        content = response.text
+        print(html_file)
+        with open(html_file, 'w') as f:
+            f.write(str(content.replace('rows="7"', ' rows="10" cols="100" ')).encode('latin1').decode('utf8'))  
+
+            #  json/" + str(ticketId) + "_details.pdf" 
+
+
+    # Write the messages 
+        response2 = requests.get('https://tsnew.sanmarcoweb.com/it/ticket/messages/index/id/' + str(ticketId), cookies=cookies, headers=headers)
+
+        content2 = response2.text
+        
+        with open(html_file, 'a', encoding='utf-8') as b:
+            b.write(content2.replace('rows="7"', ' rows="10" cols="100" ').encode('latin1').decode('utf8'))  
+
+    # Append notes
+        response3 = requests.get('https://tsnew.sanmarcoweb.com/it/ticket/notes/index/id/' + str(ticketId), cookies=cookies, headers=headers)
+
+        content3 = response3.text     
+
+        with open(html_file, 'a', encoding='utf-8') as c:
+            c.write(content3.replace('rows="7"', ' rows="10" cols="100" ').encode('latin1').decode('utf8'))    
+
+    # Append the attachments
+
+        response4 = requests.get('https://tsnew.sanmarcoweb.com/it/ticket/attachments/index/id/' + str(ticketId), cookies=cookies, headers=headers)
+
+        content4 = response4.text
+        with open(html_file, 'a', encoding='utf-8') as d:
+            d.write(content4.replace('rows="7"', ' rows="10" cols="100" ').encode('latin1').decode('utf8'))
+
+    # Convert into pdf
+        # path_wkhtmltopdf = '/usr/bin/wkhtmltopdf'
+        # config = pdfkit.configuration(wkhtmltopdf=path_wkhtmltopdf)
+        # pdfkit.from_file(html_file, pdf_file, configuration=config)  
+
+        return True
+    
+    except Exception as e:
+          logging.error(traceback.format_exc())
+          return False
+
+
+# getDetail(456427)
+
+# deleteDocs(456292)
+
+
+
+
+
+

api/limbo/compareText.py

@@ -0,0 +1,205 @@
+import requests
+import re
+import json
+
+import traceback
+import logging
+import cockie
+import connection
+import spacy_match
+
+import spacy
+
+nlp = spacy.load('it_core_news_lg')
+
+def getConn():
+    c = connection.conn()
+    return c
+
+def closeConnection(connection,cursor):
+    if connection:
+        cursor.close()
+        connection.close()
+
+file_path = "output.json"
+
+def extract_text_between_markers(text, start_marker, end_marker):
+    pattern = re.compile(f'{re.escape(start_marker)}(.*?){re.escape(end_marker)}', re.DOTALL)
+    match = pattern.search(text)
+    if match:
+        return match.group(1).strip()
+    return None
+
+
+
+# Returns all the filters inserted by a user
+def getObjectAndDescription(area,prodotto):
+    connection = getConn()
+    cursor = connection.cursor()
+    
+    postgreSQL_select_Query = "select descrizione AS result from ticket_big where competenze ='" + str(area) + "' AND prodotto = '" + prodotto + "' " #+ " OR prodotto ='' "
+    print(postgreSQL_select_Query)
+    cursor.execute(postgreSQL_select_Query)
+    publisher_records = cursor.fetchall()
+
+    closeConnection(connection,cursor)
+    # print(publisher_records)
+    return publisher_records
+
+
+def getDetail(ticketId):
+
+    try:
+
+        cookies = {
+            'language': 'it',
+            'sid': cockie.id,
+        }
+
+        headers = {
+            'Accept': '*/*',
+            'Accept-Language': 'it-IT,it;q=0.9,la;q=0.8',
+            'Connection': 'keep-alive',
+            'Content-Type': 'application/x-www-form-urlencoded',
+            # Requests sorts cookies= alphabetically
+            # 'Cookie': 'language=it; sid=89q0fl4j3r54600e2fdbf952t1',
+            'Referer': 'https://tsnew.sanmarcoweb.com/it/ticket',
+            'Sec-Fetch-Dest': 'empty',
+            'Sec-Fetch-Mode': 'cors',
+            'Sec-Fetch-Site': 'same-origin',
+            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36',
+            'X-Requested-With': 'XMLHttpRequest',
+            'sec-ch-ua': '"Chromium";v="104", " Not A;Brand";v="99", "Google Chrome";v="104"',
+            'sec-ch-ua-mobile': '?0',
+            'sec-ch-ua-platform': '"Linux"',
+        }   
+
+        response = requests.get('https://tsnew.sanmarcoweb.com/it/ticket/details/index/id/'+ str(ticketId), cookies=cookies, headers=headers)
+
+        oggetto_start = 'name="oggetto" value="'
+        oggetto_end = '"'
+        start_marker = 'true,required: false">'
+        end_marker = '</textarea>'
+        competenza_start = '<img class="rightIcon" width="16" height="16" src="/img/_core/ticket/competence.png" />'
+        competenza_end = '/option>'
+        competenze_final_start = '" selected="selected">'
+        competenze_final_end = '<'
+        soluzione_start = '[]);}">'
+        soluzione_end = '</textarea>'
+        prodotto_start = 'id="prodotto_'+ str(ticketId)
+        prodotto_end = '/option>'
+        prodotto_final_start = 'selected="selected">'
+        prodotto_final_end = '<'
+        
+        content = response.text
+        # print(content)
+
+        oggetto = extract_text_between_markers(content, oggetto_start, oggetto_end)
+        description = extract_text_between_markers(content, start_marker, end_marker)
+        competenze = extract_text_between_markers(content, competenza_start, competenza_end)
+        competenze = extract_text_between_markers(competenze, competenze_final_start, competenze_final_end)
+        soluzione = extract_text_between_markers(content, soluzione_start, soluzione_end)
+        prodotto = extract_text_between_markers(content, prodotto_start, prodotto_end)
+        prodotto = extract_text_between_markers(prodotto, prodotto_final_start, prodotto_final_end)
+
+        print(ticketId+ " --> Cercando --> ", description)
+        # print("Descrizione: ", description)
+        # print("Competenze: ", competenze)
+        # print("Soluzione: ", soluzione)
+        # print("Prodotto: ", prodotto)
+
+
+        # getObjectAndDescription('Assistenza Vinicoli','JExp')
+        # s_match(description)
+
+         
+        if not soluzione.replace("\n", ""):
+            return False
+        
+        connection = getConn()
+        cursor = connection.cursor()
+
+        postgres_insert_query = """ INSERT INTO ticket_big (id, oggetto,descrizione, competenze, soluzione,prodotto) VALUES (%s,%s,%s,%s,%s,%s)"""
+        record_to_insert = (str(ticketId), oggetto.replace("\n", ""),description.replace("\n", ""), competenze.replace("\n", ""),soluzione.replace("\n", ""),prodotto.replace("\n", ""))
+        cursor.execute(postgres_insert_query, record_to_insert)
+
+        connection.commit()
+        count = cursor.rowcount
+        if count <1:
+            return False
+        print(count, "Record inserted successfully into sort table")
+        closeConnection(connection,cursor)
+        return True
+    
+    except Exception as e:
+          logging.error(traceback.format_exc())
+          return False
+
+
+# for ticket in range(456424,456429):
+#         getDetail(ticket)
+
+def get_ids_from_json_file(file_path):
+    with open(file_path, 'r') as json_file:
+        data = json.load(json_file)
+
+    ids = [item['id'] for item in data['data']]
+    return ids
+
+
+
+file_path = 'data.json'
+ids_list = get_ids_from_json_file("/Users/tommal/Desktop/tickets.json")
+# print(ids_list)
+
+
+for ticket in ids_list:
+    getDetail(ticket)
+
+
+############################################################
+
+
+def s_match(w_target):
+    # w_target = "JExp non funziona per un problema delle dogane:Esiste una procedura di emergenza per inviare i DAA?"
+
+    # List of w1 to w500
+    # w_list = [
+    #     ("VIOLAZIONE REGOLA C070 non riusciamo a fare l'EAD"),
+    #     ("L'invio dell'E-ad si blocca per la presenza dell'errore violazione regola C070")
+    # ]
+
+    w_list = getObjectAndDescription('Assistenza Vinicoli','JExp')
+    # print("\n\n")
+    # print(w_list)
+    # print("\n\n")
+    w_list = [item[0] for item in w_list]
+    # w_list = ["tempo bello", "molto", "usciamo fuori"]
+
+
+    # Calculate similarity scores for each w in w_list and create a list of tuples
+    similarity_scores = [(w, nlp(w_target).similarity(nlp(w))) for w in w_list]
+
+    # Sort the list of tuples based on similarity score in descending order
+    sorted_similarity_scores = sorted(similarity_scores, key=lambda x: x[1], reverse=True)
+
+    # Get the best 5 matches from the sorted list
+    best_matches = sorted_similarity_scores[:5]
+
+    # Print the best 5 matches
+    print("\n\n")
+    print("Top 5 matches:")
+    for match, similarity_score in best_matches:
+        print("Match:", match)
+        print("Similarity score:", similarity_score)
+        print("\n")
+
+
+
+
+
+
+
+
+
+

api/limbo/diff.py

@@ -0,0 +1,27 @@
+import difflib
+import compareText
+
+def get_similarity_ratio(s1, s2):
+    matcher = difflib.SequenceMatcher(None, s1, s2)
+    return matcher.ratio()
+
+w_target = "CERTIFICATO DI FIRMA SCADUTO"
+
+w_list = compareText.getObjectAndDescription('Assistenza Vinicoli','JExp')
+w2_list = [item[0] for item in w_list]
+w_list = w2_list
+# Calculate similarity scores for each w in w_list
+similarity_scores = [(w, get_similarity_ratio(w_target, w)) for w in w_list]
+
+# Sort the list of tuples based on similarity score in descending order
+sorted_similarity_scores = sorted(similarity_scores, key=lambda x: x[1], reverse=True)
+
+# Get the best 5 matches from the sorted list
+best_matches = sorted_similarity_scores[:5]
+
+# Print the best 5 matches
+print("Top 5 matches:")
+for match, similarity_score in best_matches:
+    print("Match:", match)
+    print("Similarity score:", similarity_score)
+    print()

api/phrase_similarity.py

@@ -0,0 +1,64 @@
+from pathlib import Path
+import sys
+import traceback
+
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+CONN_DIR = ROOT_DIR / "conn"
+if str(CONN_DIR) not in sys.path:
+    sys.path.insert(0, str(CONN_DIR))
+
+import mysql5
+
+
+def _legacy_start_guessing_manual(ticket):
+    try:
+        from sklearn.feature_extraction.text import TfidfVectorizer
+        from sklearn.metrics.pairwise import cosine_similarity
+    except Exception:
+        traceback.print_exc()
+        return []
+
+    competence_origin, product_origin, problem_origin, area_origin, subarea_origin = mysql5.getCompetenceOrProdotto(ticket)
+    w_list = mysql5.getProblemaBYCompetenzaOrProgramma_manul(competence_origin, product_origin, area_origin, subarea_origin)
+
+    if len(w_list) == 0:
+        return []
+
+    phrases = [item[0] for item in w_list]
+    vectorizer = TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5), lowercase=True)
+    tfidf_matrix = vectorizer.fit_transform([problem_origin] + phrases)
+    similarity_matrix = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:])
+
+    # Keep the real top five. The old code used [-6:-1], which skipped the best result.
+    best_matches_indices = similarity_matrix.argsort()[0][-5:][::-1]
+
+    final_list = []
+    for i, index in enumerate(best_matches_indices, start=1):
+        complete_ticket = mysql5.getTicketByProblema(str(phrases[index]))
+        solution = str(complete_ticket[1])
+        if len(solution) > 400:
+            solution = solution[:400] + "..."
+
+        final_list.append(
+            f"\n\n----- {i}. Metodo legacy -----\n"
+            f"[ /Ticket_dettaglio_{complete_ticket[0]} ]\n"
+            f"https://tsnew.sanmarcoweb.com/it/ticket/index/index/operation/view/id/{complete_ticket[0]}\n"
+            f"SOLUZIONE: {solution}\n"
+        )
+    return final_list
+
+
+def start_guessing_manual(ticket, user_id=None, filters=None, limit=None, offset=0):
+    try:
+        import ai_retriever
+
+        kwargs = {"user_id": user_id, "filters": filters, "offset": offset}
+        if limit is not None:
+            kwargs["limit"] = limit
+        return ai_retriever.start_guessing_manual(ticket, **kwargs)
+    except Exception:
+        traceback.print_exc()
+        if filters:
+            return []
+        return _legacy_start_guessing_manual(ticket)