From b55e8d4e2635800f3771f0240b1c37ae5b09165f Mon Sep 17 00:00:00 2001 From: Admin MPCZ Date: Thu, 16 Apr 2026 13:59:53 +0200 Subject: [PATCH] Import IODA applications + table qualys_missing_servers - Migration deploy/migrations/2026-04-16_ioda_qualys_missing.sql: * ALTER applications: 13 colonnes ioda_* (libelle, code_pos, type, statut, perimetre, dept_domaine, resp_metier, resp_dsi, nb_components, etc.) * Index unique sur ioda_libelle (cle d'upsert idempotente) * Nouvelle table qualys_missing_servers avec server_id FK, reason_category enum (appliance/ot_scada/virtualisation/oubli/...), status (a_traiter/a_enroler/exempt/enrole/decom), priority 1-5, trigger updated_at - tools/import_applications_ioda.py: lit deploy/ServeursAssoci*IODA*.xlsx sheet "Services Metiers", upsert sur ioda_libelle - tools/import_qualys_missing.py: lit deploy/comparaison*.xlsx sheet RECAP, filtre col J (COMP1) sans 'Qualys', categorise auto via heuristique nom (BAC_/BEU_/... = ot_scada, esx*/vp*esx = virtu, presence 3 sources sans Qualys = oubli urgent), lien auto vers servers.id si match hostname/fqdn Co-Authored-By: Claude Opus 4.6 (1M context) --- .../2026-04-16_ioda_qualys_missing.sql | 78 +++++++ tools/import_applications_ioda.py | 142 ++++++++++++ tools/import_qualys_missing.py | 204 ++++++++++++++++++ 3 files changed, 424 insertions(+) create mode 100644 deploy/migrations/2026-04-16_ioda_qualys_missing.sql create mode 100644 tools/import_applications_ioda.py create mode 100644 tools/import_qualys_missing.py diff --git a/deploy/migrations/2026-04-16_ioda_qualys_missing.sql b/deploy/migrations/2026-04-16_ioda_qualys_missing.sql new file mode 100644 index 0000000..1749718 --- /dev/null +++ b/deploy/migrations/2026-04-16_ioda_qualys_missing.sql @@ -0,0 +1,78 @@ +-- Migration 2026-04-16 : enrichissement applications IODA + table qualys_missing_servers +-- Idempotent (re-jouable sans casse) + +BEGIN; + +-- ========================================================================= +-- 1) Extension table applications avec champs IODA +-- ========================================================================= +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_libelle varchar(200); +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_lib_court varchar(50); +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_code_pos varchar(20); +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_type varchar(50); +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_statut varchar(50); +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_alias text; +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_perimetre varchar(100); +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_dept_domaine varchar(200); +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_resp_metier varchar(100); +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_resp_dsi varchar(100); +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_nb_components integer; +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_commentaire text; +ALTER TABLE applications ADD COLUMN IF NOT EXISTS ioda_imported_at timestamptz; + +CREATE UNIQUE INDEX IF NOT EXISTS applications_ioda_libelle_uniq + ON applications (ioda_libelle) WHERE ioda_libelle IS NOT NULL; + +COMMENT ON COLUMN applications.ioda_libelle IS 'Libellé service métier IODA (clé d''import)'; +COMMENT ON COLUMN applications.ioda_code_pos IS 'Code zone POS IODA (TRA, ADV, …)'; +COMMENT ON COLUMN applications.ioda_resp_metier IS 'Responsable Service Métier (à notifier patching)'; +COMMENT ON COLUMN applications.ioda_resp_dsi IS 'Responsable Service DSI (à notifier patching)'; + +-- ========================================================================= +-- 2) Table qualys_missing_servers (serveurs absents de Qualys + raison) +-- ========================================================================= +CREATE TABLE IF NOT EXISTS qualys_missing_servers ( + id serial PRIMARY KEY, + hostname varchar(255) NOT NULL, + hostname_norm varchar(255) GENERATED ALWAYS AS (lower(hostname)) STORED, + environnement varchar(50), + sources_present varchar(100), -- ex: "Cyberark+S1+ITOP" + in_cyberark boolean DEFAULT false, + in_sentinel boolean DEFAULT false, + in_itop boolean DEFAULT false, + server_id integer REFERENCES servers(id) ON DELETE SET NULL, + reason_category varchar(30), -- appliance | ot_scada | virtualisation | embedded | oubli | decom | inconnu | other + reason_detail text, + status varchar(20) DEFAULT 'a_traiter', -- a_traiter | a_enroler | exempt | enrole | decom + priority smallint DEFAULT 3, -- 1 (urgent) → 5 (faible) + notes text, + source_file varchar(100), -- fichier d'origine + last_seen_at timestamptz DEFAULT now(), + created_at timestamptz DEFAULT now(), + updated_at timestamptz DEFAULT now(), + CONSTRAINT qms_status_check CHECK (status IN ('a_traiter','a_enroler','exempt','enrole','decom')), + CONSTRAINT qms_reason_check CHECK (reason_category IS NULL OR reason_category IN + ('appliance','ot_scada','virtualisation','embedded','oubli','decom','inconnu','other')) +); + +CREATE UNIQUE INDEX IF NOT EXISTS qms_hostname_norm_uniq ON qualys_missing_servers (hostname_norm); +CREATE INDEX IF NOT EXISTS qms_status_idx ON qualys_missing_servers (status); +CREATE INDEX IF NOT EXISTS qms_reason_idx ON qualys_missing_servers (reason_category); +CREATE INDEX IF NOT EXISTS qms_server_id_idx ON qualys_missing_servers (server_id); + +COMMENT ON TABLE qualys_missing_servers IS 'Serveurs détectés ailleurs (CA/S1/iTop) mais absents de Qualys + raison'; +COMMENT ON COLUMN qualys_missing_servers.reason_category IS 'appliance, ot_scada, virtualisation (ESXi), embedded, oubli (à enrôler), decom, inconnu, other'; +COMMENT ON COLUMN qualys_missing_servers.status IS 'a_traiter, a_enroler, exempt (légitimement hors Qualys), enrole (fait), decom'; +COMMENT ON COLUMN qualys_missing_servers.priority IS '1 urgent → 5 faible (auto-calc selon sources)'; + +-- Trigger updated_at +CREATE OR REPLACE FUNCTION qms_set_updated_at() RETURNS trigger AS $$ +BEGIN NEW.updated_at = now(); RETURN NEW; END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS qms_updated_at_trg ON qualys_missing_servers; +CREATE TRIGGER qms_updated_at_trg + BEFORE UPDATE ON qualys_missing_servers + FOR EACH ROW EXECUTE FUNCTION qms_set_updated_at(); + +COMMIT; diff --git a/tools/import_applications_ioda.py b/tools/import_applications_ioda.py new file mode 100644 index 0000000..d4468f5 --- /dev/null +++ b/tools/import_applications_ioda.py @@ -0,0 +1,142 @@ +"""Import applications depuis le fichier IODA Sanef. + +Lit `deploy/ServeursAssociesAuxServicesIODA_*.xlsx` (sheet "Services Metiers") +et UPSERT dans la table `applications` en utilisant `ioda_libelle` comme cle. + +Usage: + python tools/import_applications_ioda.py [chemin_fichier.xlsx] + +Si chemin omis: cherche dans deploy/ le plus recent +ServeursAssoci*AuxServicesIODA_*.xlsx. +""" +import os +import sys +import glob +from pathlib import Path +from datetime import datetime + +import openpyxl +from sqlalchemy import create_engine, text + +ROOT = Path(__file__).resolve().parent.parent +DATABASE_URL = (os.getenv("DATABASE_URL_DEMO") + or os.getenv("DATABASE_URL") + or "postgresql://patchcenter:PatchCenter2026!@localhost:5432/patchcenter_db") + + +def find_ioda_file(): + """Trouve le fichier IODA le plus recent dans deploy/.""" + pattern = str(ROOT / "deploy" / "ServeursAssoci*AuxServicesIODA_*.xlsx") + files = sorted(glob.glob(pattern)) + if not files: + # Fallback: deploy/ sans variantes accent + files = sorted(glob.glob(str(ROOT / "deploy" / "*IODA*.xlsx"))) + return files[-1] if files else None + + +def parse_services(xlsx_path): + """Renvoie liste de dicts {libelle, lib_court, code_pos, ...} depuis sheet 'Services Metiers'.""" + wb = openpyxl.load_workbook(xlsx_path, data_only=True) + ws_name = next((s for s in wb.sheetnames if "Services" in s and "Metier" in s), None) + if not ws_name: + raise SystemExit(f"[ERR] Sheet 'Services Metiers' introuvable. Sheets: {wb.sheetnames}") + ws = wb[ws_name] + + services = [] + for i, row in enumerate(ws.iter_rows(values_only=True)): + # Header sur 2 lignes (i=0 mega-header, i=1 vrais en-tetes) + if i < 2: + continue + # Cols: 0 Actions, 1 Type, 2 Statut, 3 Code Zone POS, 4 Lib court, 5 Libelle, + # 6 Alias, 7 Editeur, 8 Description, 9 Commentaire, 10 Nb composants, + # 11 Perimetre, 12 Dept/Domaine, 13 Resp Metier, 14 Resp DSI + libelle = row[5] + if not libelle or not str(libelle).strip(): + continue + services.append({ + "libelle": str(libelle).strip(), + "type": (row[1] or "").strip() if row[1] else None, + "statut": (row[2] or "").strip() if row[2] else None, + "code_pos": (row[3] or "").strip() if row[3] else None, + "lib_court": (row[4] or "").strip() if row[4] else None, + "alias": (row[6] or "").strip() if row[6] else None, + "editeur": (row[7] or "").strip() if row[7] else None, + "description": (row[8] or "").strip() if row[8] else None, + "commentaire": (row[9] or "").strip() if row[9] else None, + "nb_components": int(row[10]) if row[10] and str(row[10]).strip().isdigit() else None, + "perimetre": (row[11] or "").strip() if row[11] else None, + "dept_domaine": (row[12] or "").strip() if row[12] else None, + "resp_metier": (row[13] or "").strip() if row[13] else None, + "resp_dsi": (row[14] or "").strip() if row[14] else None, + }) + return services + + +SQL_UPSERT = text(""" + INSERT INTO applications ( + nom_court, nom_complet, description, editeur, criticite, + ioda_libelle, ioda_lib_court, ioda_code_pos, ioda_type, ioda_statut, + ioda_alias, ioda_perimetre, ioda_dept_domaine, ioda_resp_metier, + ioda_resp_dsi, ioda_nb_components, ioda_commentaire, ioda_imported_at, + created_at, updated_at + ) VALUES ( + :nom_court, :nom_complet, :description, :editeur, 'standard', + :libelle, :lib_court, :code_pos, :type, :statut, + :alias, :perimetre, :dept_domaine, :resp_metier, + :resp_dsi, :nb_components, :commentaire, now(), + now(), now() + ) + ON CONFLICT (ioda_libelle) WHERE ioda_libelle IS NOT NULL + DO UPDATE SET + nom_complet = EXCLUDED.nom_complet, + description = COALESCE(EXCLUDED.description, applications.description), + editeur = COALESCE(EXCLUDED.editeur, applications.editeur), + ioda_lib_court = EXCLUDED.ioda_lib_court, + ioda_code_pos = EXCLUDED.ioda_code_pos, + ioda_type = EXCLUDED.ioda_type, + ioda_statut = EXCLUDED.ioda_statut, + ioda_alias = EXCLUDED.ioda_alias, + ioda_perimetre = EXCLUDED.ioda_perimetre, + ioda_dept_domaine = EXCLUDED.ioda_dept_domaine, + ioda_resp_metier = EXCLUDED.ioda_resp_metier, + ioda_resp_dsi = EXCLUDED.ioda_resp_dsi, + ioda_nb_components = EXCLUDED.ioda_nb_components, + ioda_commentaire = EXCLUDED.ioda_commentaire, + ioda_imported_at = now(), + updated_at = now() +""") + + +def main(): + xlsx = sys.argv[1] if len(sys.argv) > 1 else find_ioda_file() + if not xlsx or not os.path.exists(xlsx): + print(f"[ERR] Fichier IODA introuvable. Place-le dans deploy/ (ex: deploy/ServeursAssociesAuxServicesIODA_YYYYMMDD.xlsx)") + sys.exit(1) + + print(f"[INFO] Fichier: {xlsx}") + services = parse_services(xlsx) + print(f"[INFO] Services parses: {len(services)}") + + engine = create_engine(DATABASE_URL) + print(f"[INFO] DB: {DATABASE_URL.rsplit('@', 1)[-1]}") + + inserted = updated = 0 + with engine.begin() as conn: + for svc in services: + # nom_court max 50 chars: prefere lib_court, sinon truncate libelle + nom_court = (svc["lib_court"] or svc["libelle"])[:50] + params = { + **svc, + "nom_court": nom_court, + "nom_complet": svc["libelle"][:200], + } + r = conn.execute(SQL_UPSERT, params) + # Discriminer insert vs update via xmin trick? Plus simple: compter rowcount + inserted += 1 # upsert ne distingue pas, on log la totale + + print(f"[OK] Upsert termine — {inserted} services traites") + print(f"[INFO] Verif: SELECT COUNT(*) FROM applications WHERE ioda_libelle IS NOT NULL;") + + +if __name__ == "__main__": + main() diff --git a/tools/import_qualys_missing.py b/tools/import_qualys_missing.py new file mode 100644 index 0000000..25256cf --- /dev/null +++ b/tools/import_qualys_missing.py @@ -0,0 +1,204 @@ +"""Import des serveurs absents de Qualys depuis comparaisonv2.xlsx. + +Lit la sheet 'RECAP' (col J = COMP1) du fichier deploy/comparaisonv2.xlsx, +filtre les lignes ou 'Qualys' n'est PAS dans la chaine, et UPSERT dans +la table `qualys_missing_servers`. + +Categorisation auto via heuristique sur le nom (prefixes connus): + - virtualisation : ESXi/hyperviseurs (commencent souvent par 'esx', 'vp*esx') + - ot_scada : OT bord de route (BAC_, BEU_, BOA_, BOP_, BUC_, CAG_, *_SRV_*) + - oubli : present partout SAUF Qualys (Cyberark + S1 + ITOP) → priorite 1 + - inconnu : autres cas, a investiguer + +Usage: + python tools/import_qualys_missing.py [chemin_fichier.xlsx] +""" +import os +import sys +import glob +import re +from pathlib import Path +from collections import Counter + +import openpyxl +from sqlalchemy import create_engine, text + +ROOT = Path(__file__).resolve().parent.parent +DATABASE_URL = (os.getenv("DATABASE_URL_DEMO") + or os.getenv("DATABASE_URL") + or "postgresql://patchcenter:PatchCenter2026!@localhost:5432/patchcenter_db") + + +def find_comparison_file(): + pattern = str(ROOT / "deploy" / "comparaison*.xlsx") + files = sorted(glob.glob(pattern)) + return files[-1] if files else None + + +# --- Heuristiques de categorisation -------------------------------------- +RE_OT_SCADA = re.compile(r"^(BAC|BEU|BOA|BOP|BUC|CAG|HEU|FLA|FLB|FLC|FLD|FLE|FLF|FLG|FLH|GAR|MEU|PAU|REI)_L\d+_S\d+", re.I) +RE_VIRTU = re.compile(r"^(esx|vp.*esx|hyp|hv\d|esxi)", re.I) + + +def categorize(hostname, sources_present): + """Retourne (reason_category, status, priority, reason_detail).""" + h = hostname.lower() + + # 1. OT/SCADA bord de route + if RE_OT_SCADA.match(hostname): + return ("ot_scada", "exempt", 4, + "Equipement bord de route / OT (pas d'agent Qualys possible)") + + # 2. Virtualisation + if RE_VIRTU.match(h): + return ("virtualisation", "exempt", 5, + "Hyperviseur ESXi (scan via Qualys connector vCenter, pas d'agent)") + + # 3. Present partout SAUF Qualys → oubli pur (urgent) + if sources_present and all(s in sources_present for s in ("Cyberark", "S1", "ITOP")): + return ("oubli", "a_enroler", 1, + "Present sur CyberArk + Sentinel + iTop, manque Qualys (oubli enrolement)") + + # 4. Sentinel + iTop (sans CA): probable pas connu compte d'admin standard + if sources_present and "S1" in sources_present and "ITOP" in sources_present: + return ("oubli", "a_enroler", 2, + "Sentinel + iTop OK, manque CyberArk + Qualys") + + # 5. iTop seulement: serveur orphelin a investiguer + if sources_present == "ITOP seulement": + return ("inconnu", "a_traiter", 3, + "Reference uniquement dans iTop. Serveur eteint? Decom? A verifier") + + # 6. S1 seulement: agent S1 detecte mais pas de ref iTop + if sources_present == "S1 seulement": + return ("inconnu", "a_traiter", 2, + "Detecte par Sentinel mais pas iTop. Shadow IT? Asset non reference") + + return ("inconnu", "a_traiter", 3, f"Sources presentes: {sources_present or 'aucune'}") + + +def parse_recap(xlsx_path): + """Renvoie liste de dicts pour les lignes hors Qualys.""" + wb = openpyxl.load_workbook(xlsx_path, data_only=True, read_only=True) + if "RECAP" not in wb.sheetnames: + raise SystemExit(f"[ERR] Sheet 'RECAP' introuvable. Sheets: {wb.sheetnames}") + ws = wb["RECAP"] + + missing = [] + for i, row in enumerate(ws.iter_rows(min_row=2, values_only=True)): + if not row or not row[4]: # col E = Concatenation + continue + hostname = str(row[4]).strip() + env = (row[2] or "").strip() or None + comp = row[9] # col J = COMP1 + + if not comp or "Qualys" in comp: + continue # est dans Qualys → on garde pas + + missing.append({ + "hostname": hostname, + "environnement": env, + "sources_present": comp, + "in_cyberark": "Cyberark" in comp, + "in_sentinel": "S1" in comp, + "in_itop": "ITOP" in comp, + }) + return missing + + +SQL_UPSERT = text(""" + INSERT INTO qualys_missing_servers ( + hostname, environnement, sources_present, + in_cyberark, in_sentinel, in_itop, + server_id, reason_category, reason_detail, status, priority, + source_file, last_seen_at, created_at, updated_at + ) VALUES ( + :hostname, :environnement, :sources_present, + :in_cyberark, :in_sentinel, :in_itop, + :server_id, :reason_category, :reason_detail, :status, :priority, + :source_file, now(), now(), now() + ) + ON CONFLICT (hostname_norm) DO UPDATE SET + environnement = EXCLUDED.environnement, + sources_present = EXCLUDED.sources_present, + in_cyberark = EXCLUDED.in_cyberark, + in_sentinel = EXCLUDED.in_sentinel, + in_itop = EXCLUDED.in_itop, + server_id = COALESCE(EXCLUDED.server_id, qualys_missing_servers.server_id), + -- reason/status/priority: ne pas ecraser si l'utilisateur a deja saisi + reason_category = CASE + WHEN qualys_missing_servers.reason_category IS NULL OR qualys_missing_servers.status = 'a_traiter' + THEN EXCLUDED.reason_category ELSE qualys_missing_servers.reason_category END, + reason_detail = CASE + WHEN qualys_missing_servers.reason_detail IS NULL + THEN EXCLUDED.reason_detail ELSE qualys_missing_servers.reason_detail END, + status = CASE + WHEN qualys_missing_servers.status = 'a_traiter' + THEN EXCLUDED.status ELSE qualys_missing_servers.status END, + priority = CASE + WHEN qualys_missing_servers.status = 'a_traiter' + THEN EXCLUDED.priority ELSE qualys_missing_servers.priority END, + source_file = EXCLUDED.source_file, + last_seen_at = now(), + updated_at = now() +""") + + +SQL_LINK_SERVER = text(""" + SELECT id FROM servers + WHERE lower(hostname) = :h OR lower(fqdn) = :h + LIMIT 1 +""") + + +def main(): + xlsx = sys.argv[1] if len(sys.argv) > 1 else find_comparison_file() + if not xlsx or not os.path.exists(xlsx): + print("[ERR] Fichier comparaison introuvable. Place comparaisonv2.xlsx dans deploy/") + sys.exit(1) + + print(f"[INFO] Fichier: {xlsx}") + missing = parse_recap(xlsx) + print(f"[INFO] Serveurs hors Qualys parses: {len(missing)}") + + engine = create_engine(DATABASE_URL) + print(f"[INFO] DB: {DATABASE_URL.rsplit('@', 1)[-1]}") + + cat_count = Counter() + linked = 0 + source_file = os.path.basename(xlsx) + + with engine.begin() as conn: + for m in missing: + # Lien vers servers (si existe deja) + link = conn.execute(SQL_LINK_SERVER, {"h": m["hostname"].lower()}).fetchone() + m["server_id"] = link[0] if link else None + if link: + linked += 1 + + cat, status, prio, detail = categorize(m["hostname"], m["sources_present"]) + cat_count[cat] += 1 + + params = { + **m, + "reason_category": cat, + "reason_detail": detail, + "status": status, + "priority": prio, + "source_file": source_file, + } + conn.execute(SQL_UPSERT, params) + + print(f"[OK] Upsert termine — {len(missing)} lignes") + print(f"[INFO] Lies a un server existant (servers.id): {linked}") + print(f"[INFO] Repartition categories:") + for cat, c in cat_count.most_common(): + print(f" {c:4d} {cat}") + print() + print("[INFO] Verifs:") + print(" SELECT reason_category, status, COUNT(*) FROM qualys_missing_servers GROUP BY 1,2 ORDER BY 1,2;") + print(" SELECT hostname, sources_present, reason_category FROM qualys_missing_servers WHERE status='a_enroler' AND priority=1;") + + +if __name__ == "__main__": + main()