Import IODA applications + table qualys_missing_servers

- Migration deploy/migrations/2026-04-16_ioda_qualys_missing.sql:
  * ALTER applications: 13 colonnes ioda_* (libelle, code_pos, type, statut,
    perimetre, dept_domaine, resp_metier, resp_dsi, nb_components, etc.)
  * Index unique sur ioda_libelle (cle d'upsert idempotente)
  * Nouvelle table qualys_missing_servers avec server_id FK,
    reason_category enum (appliance/ot_scada/virtualisation/oubli/...),
    status (a_traiter/a_enroler/exempt/enrole/decom), priority 1-5,
    trigger updated_at

- tools/import_applications_ioda.py: lit deploy/ServeursAssoci*IODA*.xlsx
  sheet "Services Metiers", upsert sur ioda_libelle

- tools/import_qualys_missing.py: lit deploy/comparaison*.xlsx sheet
  RECAP, filtre col J (COMP1) sans 'Qualys', categorise auto via
  heuristique nom (BAC_/BEU_/... = ot_scada, esx*/vp*esx = virtu,
  presence 3 sources sans Qualys = oubli urgent), lien auto vers
  servers.id si match hostname/fqdn

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

deploy/migrations/2026-04-16_ioda_qualys_missing.sql

@@ -0,0 +1,78 @@
+-- Migration 2026-04-16 : enrichissement applications IODA + table qualys_missing_servers
+-- Idempotent (re-jouable sans casse)
+
+BEGIN;
+
+-- =========================================================================
+-- 1) Extension table applications avec champs IODA
+-- =========================================================================
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_libelle      varchar(200);
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_lib_court    varchar(50);
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_code_pos     varchar(20);
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_type         varchar(50);
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_statut       varchar(50);
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_alias        text;
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_perimetre    varchar(100);
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_dept_domaine varchar(200);
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_resp_metier  varchar(100);
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_resp_dsi     varchar(100);
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_nb_components integer;
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_commentaire  text;
+ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_imported_at  timestamptz;
+
+CREATE UNIQUE INDEX IF NOT EXISTS applications_ioda_libelle_uniq
+    ON applications (ioda_libelle) WHERE ioda_libelle IS NOT NULL;
+
+COMMENT ON COLUMN applications.ioda_libelle      IS 'Libellé service métier IODA (clé d''import)';
+COMMENT ON COLUMN applications.ioda_code_pos     IS 'Code zone POS IODA (TRA, ADV, …)';
+COMMENT ON COLUMN applications.ioda_resp_metier  IS 'Responsable Service Métier (à notifier patching)';
+COMMENT ON COLUMN applications.ioda_resp_dsi     IS 'Responsable Service DSI (à notifier patching)';
+
+-- =========================================================================
+-- 2) Table qualys_missing_servers (serveurs absents de Qualys + raison)
+-- =========================================================================
+CREATE TABLE IF NOT EXISTS qualys_missing_servers (
+    id                serial PRIMARY KEY,
+    hostname          varchar(255) NOT NULL,
+    hostname_norm     varchar(255) GENERATED ALWAYS AS (lower(hostname)) STORED,
+    environnement     varchar(50),
+    sources_present   varchar(100),         -- ex: "Cyberark+S1+ITOP"
+    in_cyberark       boolean DEFAULT false,
+    in_sentinel       boolean DEFAULT false,
+    in_itop           boolean DEFAULT false,
+    server_id         integer REFERENCES servers(id) ON DELETE SET NULL,
+    reason_category   varchar(30),          -- appliance | ot_scada | virtualisation | embedded | oubli | decom | inconnu | other
+    reason_detail     text,
+    status            varchar(20) DEFAULT 'a_traiter',  -- a_traiter | a_enroler | exempt | enrole | decom
+    priority          smallint DEFAULT 3,   -- 1 (urgent) → 5 (faible)
+    notes             text,
+    source_file       varchar(100),         -- fichier d'origine
+    last_seen_at      timestamptz DEFAULT now(),
+    created_at        timestamptz DEFAULT now(),
+    updated_at        timestamptz DEFAULT now(),
+    CONSTRAINT qms_status_check CHECK (status IN ('a_traiter','a_enroler','exempt','enrole','decom')),
+    CONSTRAINT qms_reason_check CHECK (reason_category IS NULL OR reason_category IN
+        ('appliance','ot_scada','virtualisation','embedded','oubli','decom','inconnu','other'))
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS qms_hostname_norm_uniq ON qualys_missing_servers (hostname_norm);
+CREATE INDEX IF NOT EXISTS qms_status_idx       ON qualys_missing_servers (status);
+CREATE INDEX IF NOT EXISTS qms_reason_idx       ON qualys_missing_servers (reason_category);
+CREATE INDEX IF NOT EXISTS qms_server_id_idx    ON qualys_missing_servers (server_id);
+
+COMMENT ON TABLE  qualys_missing_servers IS 'Serveurs détectés ailleurs (CA/S1/iTop) mais absents de Qualys + raison';
+COMMENT ON COLUMN qualys_missing_servers.reason_category IS 'appliance, ot_scada, virtualisation (ESXi), embedded, oubli (à enrôler), decom, inconnu, other';
+COMMENT ON COLUMN qualys_missing_servers.status          IS 'a_traiter, a_enroler, exempt (légitimement hors Qualys), enrole (fait), decom';
+COMMENT ON COLUMN qualys_missing_servers.priority        IS '1 urgent → 5 faible (auto-calc selon sources)';
+
+-- Trigger updated_at
+CREATE OR REPLACE FUNCTION qms_set_updated_at() RETURNS trigger AS $$
+BEGIN NEW.updated_at = now(); RETURN NEW; END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS qms_updated_at_trg ON qualys_missing_servers;
+CREATE TRIGGER qms_updated_at_trg
+    BEFORE UPDATE ON qualys_missing_servers
+    FOR EACH ROW EXECUTE FUNCTION qms_set_updated_at();
+
+COMMIT;

tools/import_applications_ioda.py

@@ -0,0 +1,142 @@
+"""Import applications depuis le fichier IODA Sanef.
+
+Lit `deploy/ServeursAssociesAuxServicesIODA_*.xlsx` (sheet "Services Metiers")
+et UPSERT dans la table `applications` en utilisant `ioda_libelle` comme cle.
+
+Usage:
+    python tools/import_applications_ioda.py [chemin_fichier.xlsx]
+
+Si chemin omis: cherche dans deploy/ le plus recent
+ServeursAssoci*AuxServicesIODA_*.xlsx.
+"""
+import os
+import sys
+import glob
+from pathlib import Path
+from datetime import datetime
+
+import openpyxl
+from sqlalchemy import create_engine, text
+
+ROOT = Path(__file__).resolve().parent.parent
+DATABASE_URL = (os.getenv("DATABASE_URL_DEMO")
+                or os.getenv("DATABASE_URL")
+                or "postgresql://patchcenter:PatchCenter2026!@localhost:5432/patchcenter_db")
+
+
+def find_ioda_file():
+    """Trouve le fichier IODA le plus recent dans deploy/."""
+    pattern = str(ROOT / "deploy" / "ServeursAssoci*AuxServicesIODA_*.xlsx")
+    files = sorted(glob.glob(pattern))
+    if not files:
+        # Fallback: deploy/ sans variantes accent
+        files = sorted(glob.glob(str(ROOT / "deploy" / "*IODA*.xlsx")))
+    return files[-1] if files else None
+
+
+def parse_services(xlsx_path):
+    """Renvoie liste de dicts {libelle, lib_court, code_pos, ...} depuis sheet 'Services Metiers'."""
+    wb = openpyxl.load_workbook(xlsx_path, data_only=True)
+    ws_name = next((s for s in wb.sheetnames if "Services" in s and "Metier" in s), None)
+    if not ws_name:
+        raise SystemExit(f"[ERR] Sheet 'Services Metiers' introuvable. Sheets: {wb.sheetnames}")
+    ws = wb[ws_name]
+
+    services = []
+    for i, row in enumerate(ws.iter_rows(values_only=True)):
+        # Header sur 2 lignes (i=0 mega-header, i=1 vrais en-tetes)
+        if i < 2:
+            continue
+        # Cols: 0 Actions, 1 Type, 2 Statut, 3 Code Zone POS, 4 Lib court, 5 Libelle,
+        # 6 Alias, 7 Editeur, 8 Description, 9 Commentaire, 10 Nb composants,
+        # 11 Perimetre, 12 Dept/Domaine, 13 Resp Metier, 14 Resp DSI
+        libelle = row[5]
+        if not libelle or not str(libelle).strip():
+            continue
+        services.append({
+            "libelle":      str(libelle).strip(),
+            "type":         (row[1] or "").strip() if row[1] else None,
+            "statut":       (row[2] or "").strip() if row[2] else None,
+            "code_pos":     (row[3] or "").strip() if row[3] else None,
+            "lib_court":    (row[4] or "").strip() if row[4] else None,
+            "alias":        (row[6] or "").strip() if row[6] else None,
+            "editeur":      (row[7] or "").strip() if row[7] else None,
+            "description":  (row[8] or "").strip() if row[8] else None,
+            "commentaire":  (row[9] or "").strip() if row[9] else None,
+            "nb_components": int(row[10]) if row[10] and str(row[10]).strip().isdigit() else None,
+            "perimetre":    (row[11] or "").strip() if row[11] else None,
+            "dept_domaine": (row[12] or "").strip() if row[12] else None,
+            "resp_metier":  (row[13] or "").strip() if row[13] else None,
+            "resp_dsi":     (row[14] or "").strip() if row[14] else None,
+        })
+    return services
+
+
+SQL_UPSERT = text("""
+    INSERT INTO applications (
+        nom_court, nom_complet, description, editeur, criticite,
+        ioda_libelle, ioda_lib_court, ioda_code_pos, ioda_type, ioda_statut,
+        ioda_alias, ioda_perimetre, ioda_dept_domaine, ioda_resp_metier,
+        ioda_resp_dsi, ioda_nb_components, ioda_commentaire, ioda_imported_at,
+        created_at, updated_at
+    ) VALUES (
+        :nom_court, :nom_complet, :description, :editeur, 'standard',
+        :libelle, :lib_court, :code_pos, :type, :statut,
+        :alias, :perimetre, :dept_domaine, :resp_metier,
+        :resp_dsi, :nb_components, :commentaire, now(),
+        now(), now()
+    )
+    ON CONFLICT (ioda_libelle) WHERE ioda_libelle IS NOT NULL
+    DO UPDATE SET
+        nom_complet        = EXCLUDED.nom_complet,
+        description        = COALESCE(EXCLUDED.description, applications.description),
+        editeur            = COALESCE(EXCLUDED.editeur, applications.editeur),
+        ioda_lib_court     = EXCLUDED.ioda_lib_court,
+        ioda_code_pos      = EXCLUDED.ioda_code_pos,
+        ioda_type          = EXCLUDED.ioda_type,
+        ioda_statut        = EXCLUDED.ioda_statut,
+        ioda_alias         = EXCLUDED.ioda_alias,
+        ioda_perimetre     = EXCLUDED.ioda_perimetre,
+        ioda_dept_domaine  = EXCLUDED.ioda_dept_domaine,
+        ioda_resp_metier   = EXCLUDED.ioda_resp_metier,
+        ioda_resp_dsi      = EXCLUDED.ioda_resp_dsi,
+        ioda_nb_components = EXCLUDED.ioda_nb_components,
+        ioda_commentaire   = EXCLUDED.ioda_commentaire,
+        ioda_imported_at   = now(),
+        updated_at         = now()
+""")
+
+
+def main():
+    xlsx = sys.argv[1] if len(sys.argv) > 1 else find_ioda_file()
+    if not xlsx or not os.path.exists(xlsx):
+        print(f"[ERR] Fichier IODA introuvable. Place-le dans deploy/ (ex: deploy/ServeursAssociesAuxServicesIODA_YYYYMMDD.xlsx)")
+        sys.exit(1)
+
+    print(f"[INFO] Fichier: {xlsx}")
+    services = parse_services(xlsx)
+    print(f"[INFO] Services parses: {len(services)}")
+
+    engine = create_engine(DATABASE_URL)
+    print(f"[INFO] DB: {DATABASE_URL.rsplit('@', 1)[-1]}")
+
+    inserted = updated = 0
+    with engine.begin() as conn:
+        for svc in services:
+            # nom_court max 50 chars: prefere lib_court, sinon truncate libelle
+            nom_court = (svc["lib_court"] or svc["libelle"])[:50]
+            params = {
+                **svc,
+                "nom_court": nom_court,
+                "nom_complet": svc["libelle"][:200],
+            }
+            r = conn.execute(SQL_UPSERT, params)
+            # Discriminer insert vs update via xmin trick? Plus simple: compter rowcount
+            inserted += 1  # upsert ne distingue pas, on log la totale
+
+    print(f"[OK] Upsert termine — {inserted} services traites")
+    print(f"[INFO] Verif: SELECT COUNT(*) FROM applications WHERE ioda_libelle IS NOT NULL;")
+
+
+if __name__ == "__main__":
+    main()

tools/import_qualys_missing.py

@@ -0,0 +1,204 @@
+"""Import des serveurs absents de Qualys depuis comparaisonv2.xlsx.
+
+Lit la sheet 'RECAP' (col J = COMP1) du fichier deploy/comparaisonv2.xlsx,
+filtre les lignes ou 'Qualys' n'est PAS dans la chaine, et UPSERT dans
+la table `qualys_missing_servers`.
+
+Categorisation auto via heuristique sur le nom (prefixes connus):
+ - virtualisation    : ESXi/hyperviseurs (commencent souvent par 'esx', 'vp*esx')
+ - ot_scada          : OT bord de route (BAC_, BEU_, BOA_, BOP_, BUC_, CAG_, *_SRV_*)
+ - oubli             : present partout SAUF Qualys (Cyberark + S1 + ITOP) → priorite 1
+ - inconnu           : autres cas, a investiguer
+
+Usage:
+    python tools/import_qualys_missing.py [chemin_fichier.xlsx]
+"""
+import os
+import sys
+import glob
+import re
+from pathlib import Path
+from collections import Counter
+
+import openpyxl
+from sqlalchemy import create_engine, text
+
+ROOT = Path(__file__).resolve().parent.parent
+DATABASE_URL = (os.getenv("DATABASE_URL_DEMO")
+                or os.getenv("DATABASE_URL")
+                or "postgresql://patchcenter:PatchCenter2026!@localhost:5432/patchcenter_db")
+
+
+def find_comparison_file():
+    pattern = str(ROOT / "deploy" / "comparaison*.xlsx")
+    files = sorted(glob.glob(pattern))
+    return files[-1] if files else None
+
+
+# --- Heuristiques de categorisation --------------------------------------
+RE_OT_SCADA = re.compile(r"^(BAC|BEU|BOA|BOP|BUC|CAG|HEU|FLA|FLB|FLC|FLD|FLE|FLF|FLG|FLH|GAR|MEU|PAU|REI)_L\d+_S\d+", re.I)
+RE_VIRTU    = re.compile(r"^(esx|vp.*esx|hyp|hv\d|esxi)", re.I)
+
+
+def categorize(hostname, sources_present):
+    """Retourne (reason_category, status, priority, reason_detail)."""
+    h = hostname.lower()
+
+    # 1. OT/SCADA bord de route
+    if RE_OT_SCADA.match(hostname):
+        return ("ot_scada", "exempt", 4,
+                "Equipement bord de route / OT (pas d'agent Qualys possible)")
+
+    # 2. Virtualisation
+    if RE_VIRTU.match(h):
+        return ("virtualisation", "exempt", 5,
+                "Hyperviseur ESXi (scan via Qualys connector vCenter, pas d'agent)")
+
+    # 3. Present partout SAUF Qualys → oubli pur (urgent)
+    if sources_present and all(s in sources_present for s in ("Cyberark", "S1", "ITOP")):
+        return ("oubli", "a_enroler", 1,
+                "Present sur CyberArk + Sentinel + iTop, manque Qualys (oubli enrolement)")
+
+    # 4. Sentinel + iTop (sans CA): probable pas connu compte d'admin standard
+    if sources_present and "S1" in sources_present and "ITOP" in sources_present:
+        return ("oubli", "a_enroler", 2,
+                "Sentinel + iTop OK, manque CyberArk + Qualys")
+
+    # 5. iTop seulement: serveur orphelin a investiguer
+    if sources_present == "ITOP seulement":
+        return ("inconnu", "a_traiter", 3,
+                "Reference uniquement dans iTop. Serveur eteint? Decom? A verifier")
+
+    # 6. S1 seulement: agent S1 detecte mais pas de ref iTop
+    if sources_present == "S1 seulement":
+        return ("inconnu", "a_traiter", 2,
+                "Detecte par Sentinel mais pas iTop. Shadow IT? Asset non reference")
+
+    return ("inconnu", "a_traiter", 3, f"Sources presentes: {sources_present or 'aucune'}")
+
+
+def parse_recap(xlsx_path):
+    """Renvoie liste de dicts pour les lignes hors Qualys."""
+    wb = openpyxl.load_workbook(xlsx_path, data_only=True, read_only=True)
+    if "RECAP" not in wb.sheetnames:
+        raise SystemExit(f"[ERR] Sheet 'RECAP' introuvable. Sheets: {wb.sheetnames}")
+    ws = wb["RECAP"]
+
+    missing = []
+    for i, row in enumerate(ws.iter_rows(min_row=2, values_only=True)):
+        if not row or not row[4]:  # col E = Concatenation
+            continue
+        hostname = str(row[4]).strip()
+        env      = (row[2] or "").strip() or None
+        comp     = row[9]  # col J = COMP1
+
+        if not comp or "Qualys" in comp:
+            continue  # est dans Qualys → on garde pas
+
+        missing.append({
+            "hostname":        hostname,
+            "environnement":   env,
+            "sources_present": comp,
+            "in_cyberark":     "Cyberark" in comp,
+            "in_sentinel":     "S1" in comp,
+            "in_itop":         "ITOP" in comp,
+        })
+    return missing
+
+
+SQL_UPSERT = text("""
+    INSERT INTO qualys_missing_servers (
+        hostname, environnement, sources_present,
+        in_cyberark, in_sentinel, in_itop,
+        server_id, reason_category, reason_detail, status, priority,
+        source_file, last_seen_at, created_at, updated_at
+    ) VALUES (
+        :hostname, :environnement, :sources_present,
+        :in_cyberark, :in_sentinel, :in_itop,
+        :server_id, :reason_category, :reason_detail, :status, :priority,
+        :source_file, now(), now(), now()
+    )
+    ON CONFLICT (hostname_norm) DO UPDATE SET
+        environnement   = EXCLUDED.environnement,
+        sources_present = EXCLUDED.sources_present,
+        in_cyberark     = EXCLUDED.in_cyberark,
+        in_sentinel     = EXCLUDED.in_sentinel,
+        in_itop         = EXCLUDED.in_itop,
+        server_id       = COALESCE(EXCLUDED.server_id, qualys_missing_servers.server_id),
+        -- reason/status/priority: ne pas ecraser si l'utilisateur a deja saisi
+        reason_category = CASE
+            WHEN qualys_missing_servers.reason_category IS NULL OR qualys_missing_servers.status = 'a_traiter'
+            THEN EXCLUDED.reason_category ELSE qualys_missing_servers.reason_category END,
+        reason_detail   = CASE
+            WHEN qualys_missing_servers.reason_detail IS NULL
+            THEN EXCLUDED.reason_detail ELSE qualys_missing_servers.reason_detail END,
+        status          = CASE
+            WHEN qualys_missing_servers.status = 'a_traiter'
+            THEN EXCLUDED.status ELSE qualys_missing_servers.status END,
+        priority        = CASE
+            WHEN qualys_missing_servers.status = 'a_traiter'
+            THEN EXCLUDED.priority ELSE qualys_missing_servers.priority END,
+        source_file     = EXCLUDED.source_file,
+        last_seen_at    = now(),
+        updated_at      = now()
+""")
+
+
+SQL_LINK_SERVER = text("""
+    SELECT id FROM servers
+    WHERE lower(hostname) = :h OR lower(fqdn) = :h
+    LIMIT 1
+""")
+
+
+def main():
+    xlsx = sys.argv[1] if len(sys.argv) > 1 else find_comparison_file()
+    if not xlsx or not os.path.exists(xlsx):
+        print("[ERR] Fichier comparaison introuvable. Place comparaisonv2.xlsx dans deploy/")
+        sys.exit(1)
+
+    print(f"[INFO] Fichier: {xlsx}")
+    missing = parse_recap(xlsx)
+    print(f"[INFO] Serveurs hors Qualys parses: {len(missing)}")
+
+    engine = create_engine(DATABASE_URL)
+    print(f"[INFO] DB: {DATABASE_URL.rsplit('@', 1)[-1]}")
+
+    cat_count = Counter()
+    linked = 0
+    source_file = os.path.basename(xlsx)
+
+    with engine.begin() as conn:
+        for m in missing:
+            # Lien vers servers (si existe deja)
+            link = conn.execute(SQL_LINK_SERVER, {"h": m["hostname"].lower()}).fetchone()
+            m["server_id"] = link[0] if link else None
+            if link:
+                linked += 1
+
+            cat, status, prio, detail = categorize(m["hostname"], m["sources_present"])
+            cat_count[cat] += 1
+
+            params = {
+                **m,
+                "reason_category": cat,
+                "reason_detail":   detail,
+                "status":          status,
+                "priority":        prio,
+                "source_file":     source_file,
+            }
+            conn.execute(SQL_UPSERT, params)
+
+    print(f"[OK] Upsert termine — {len(missing)} lignes")
+    print(f"[INFO] Lies a un server existant (servers.id): {linked}")
+    print(f"[INFO] Repartition categories:")
+    for cat, c in cat_count.most_common():
+        print(f"        {c:4d}  {cat}")
+    print()
+    print("[INFO] Verifs:")
+    print("    SELECT reason_category, status, COUNT(*) FROM qualys_missing_servers GROUP BY 1,2 ORDER BY 1,2;")
+    print("    SELECT hostname, sources_present, reason_category FROM qualys_missing_servers WHERE status='a_enroler' AND priority=1;")
+
+
+if __name__ == "__main__":
+    main()