diff --git a/tools/align_from_ayoub.py b/tools/align_from_ayoub.py index 6b9224e..e825c48 100644 --- a/tools/align_from_ayoub.py +++ b/tools/align_from_ayoub.py @@ -1,12 +1,18 @@ """Alignement servers depuis fichier Excel Ayoub (Planning Patching 2026_ayoub.xlsx). -Lit le sheet 'Serveurs patchables 2026' et met a jour pour chaque hostname : - - domaine (col D : Domaine) - - environnement (col C : Environnement — override si vide) - - responsable_nom (col H : Responsable Domaine DTS) - - referent_nom (col J : Referent technique) +Lit le sheet 'Serveurs patchables 2026' et aligne via JOINTURE: + 1. Auto-cree les domains/environments absents (valeurs iTop verbatim) + 2. Auto-cree les paires domain_environments + 3. Set servers.domain_env_id + 4. Sync servers.environnement (plain-text pour filtre) + 5. Set domain_environments.responsable_nom / referent_nom -Ajoute la colonne servers.domaine si absente (varchar(100)). +Colonnes Excel lues: + - Asset Name -> servers.hostname (match) + - Domaine -> domains.name (code auto-genere) + - Environnement -> environments.name (code auto-genere) + - Responsable Domaine DTS -> domain_environments.responsable_nom + - Referent technique -> domain_environments.referent_nom Usage: python tools/align_from_ayoub.py [--sheet "Serveurs patchables 2026"] [--dry-run] @@ -14,8 +20,9 @@ Usage: Requiert openpyxl: pip install openpyxl """ import os -import argparse import re +import argparse +import unicodedata from sqlalchemy import create_engine, text try: @@ -31,13 +38,75 @@ NBSP = "\u00a0" def clean(v): - """Normalise une cellule Excel: strip + enleve nbsp + None si vide.""" if v is None: return None s = str(v).replace(NBSP, " ").strip() return s or None +def slugify(s, maxlen=10): + """Slug ASCII lowercase pour code (domain.code varchar(10)).""" + if not s: + return None + nfkd = unicodedata.normalize("NFKD", s) + ascii_str = "".join(c for c in nfkd if not unicodedata.combining(c)) + ascii_str = re.sub(r"[^a-zA-Z0-9]+", "", ascii_str).lower() + return ascii_str[:maxlen] or None + + +def get_or_create_domain(conn, name): + row = conn.execute(text("SELECT id, code FROM domains WHERE name=:n"), + {"n": name}).fetchone() + if row: + return row.id + code = slugify(name, 10) + # Eviter collision de code + suffix = 0 + base_code = code + while conn.execute(text("SELECT 1 FROM domains WHERE code=:c"), + {"c": code}).fetchone(): + suffix += 1 + code = (base_code[: 10 - len(str(suffix))] + str(suffix)) + conn.execute(text( + "INSERT INTO domains (name, code, display_order) VALUES (:n, :c, 99)" + ), {"n": name, "c": code}) + return conn.execute(text("SELECT id FROM domains WHERE name=:n"), + {"n": name}).fetchone().id + + +def get_or_create_env(conn, name): + row = conn.execute(text("SELECT id FROM environments WHERE name=:n"), + {"n": name}).fetchone() + if row: + return row.id + code = slugify(name, 10) + suffix = 0 + base_code = code + while conn.execute(text("SELECT 1 FROM environments WHERE code=:c"), + {"c": code}).fetchone(): + suffix += 1 + code = (base_code[: 10 - len(str(suffix))] + str(suffix)) + conn.execute(text( + "INSERT INTO environments (name, code, display_order) VALUES (:n, :c, 99)" + ), {"n": name, "c": code}) + return conn.execute(text("SELECT id FROM environments WHERE name=:n"), + {"n": name}).fetchone().id + + +def get_or_create_dom_env(conn, domain_id, env_id): + row = conn.execute(text( + "SELECT id FROM domain_environments WHERE domain_id=:d AND environment_id=:e" + ), {"d": domain_id, "e": env_id}).fetchone() + if row: + return row.id + conn.execute(text( + "INSERT INTO domain_environments (domain_id, environment_id) VALUES (:d, :e)" + ), {"d": domain_id, "e": env_id}) + return conn.execute(text( + "SELECT id FROM domain_environments WHERE domain_id=:d AND environment_id=:e" + ), {"d": domain_id, "e": env_id}).fetchone().id + + def main(): parser = argparse.ArgumentParser() parser.add_argument("xlsx_path") @@ -48,31 +117,24 @@ def main(): engine = create_engine(DATABASE_URL) print(f"[INFO] DB: {DATABASE_URL.split('@')[-1]}") print(f"[INFO] Fichier: {args.xlsx_path}") - print(f"[INFO] Sheet: {args.sheet}") conn = engine.connect().execution_options(isolation_level="AUTOCOMMIT") - # 1. Ajoute colonne domaine si absente (idempotent) - conn.execute(text("ALTER TABLE servers ADD COLUMN IF NOT EXISTS domaine varchar(100)")) - - # 2. Lecture Excel wb = openpyxl.load_workbook(args.xlsx_path, data_only=True) if args.sheet not in wb.sheetnames: print(f"[ERR] Sheet '{args.sheet}' introuvable. Sheets: {wb.sheetnames}") return ws = wb[args.sheet] - # Detection dynamique des colonnes via header (ligne 1) header = [clean(c.value) for c in ws[1]] - print(f"[INFO] Header: {header[:12]}...") - def col_idx(name_candidates): + def col_idx(candidates): for i, h in enumerate(header): - if h and any(cand.lower() in h.lower() for cand in name_candidates): + if h and any(c.lower() in h.lower() for c in candidates): return i return -1 - idx_host = col_idx(["Asset Name", "Hostname", "Nom"]) + idx_host = col_idx(["Asset Name", "Hostname"]) idx_env = col_idx(["Environnement"]) idx_dom = col_idx(["Domaine"]) idx_resp = col_idx(["Responsable Domaine"]) @@ -85,81 +147,118 @@ def main(): print("[ERR] Colonne Asset Name/Hostname introuvable") return - stats = {"updated": 0, "unchanged": 0, "not_found": 0, "skipped": 0} - changes_detail = [] + stats = {"updated": 0, "dom_created": 0, "env_created": 0, "de_created": 0, + "not_found": 0, "skipped": 0, "de_resp_updated": 0} + seen_dom = {} + seen_env = {} + changes = [] for row in ws.iter_rows(min_row=2, values_only=True): hostname = clean(row[idx_host]) if idx_host < len(row) else None if not hostname or not any(c.isalpha() for c in hostname): stats["skipped"] += 1 continue - - # Extract sans suffixe FQDN si present hostname = hostname.split(".")[0].lower() - fields = {} - if idx_dom >= 0 and idx_dom < len(row): - v = clean(row[idx_dom]) - if v: - fields["domaine"] = v[:100] - if idx_env >= 0 and idx_env < len(row): - v = clean(row[idx_env]) - if v: - fields["environnement"] = v[:50] - if idx_resp >= 0 and idx_resp < len(row): - v = clean(row[idx_resp]) - if v: - # Enleve les virgules multiples, normalise espaces - v = re.sub(r"\s+", " ", v) - fields["responsable_nom"] = v[:200] - if idx_ref >= 0 and idx_ref < len(row): - v = clean(row[idx_ref]) - if v: - v = re.sub(r"\s+", " ", v) - fields["referent_nom"] = v[:200] + dom_name = clean(row[idx_dom]) if idx_dom >= 0 and idx_dom < len(row) else None + env_name = clean(row[idx_env]) if idx_env >= 0 and idx_env < len(row) else None + resp = clean(row[idx_resp]) if idx_resp >= 0 and idx_resp < len(row) else None + ref = clean(row[idx_ref]) if idx_ref >= 0 and idx_ref < len(row) else None + if resp: + resp = re.sub(r"\s+", " ", resp)[:100] + if ref: + ref = re.sub(r"\s+", " ", ref)[:100] - if not fields: - continue - - srv = conn.execute(text( - "SELECT id, domaine, environnement, responsable_nom, referent_nom " - "FROM servers WHERE hostname=:h" - ), {"h": hostname}).fetchone() + srv = conn.execute(text("SELECT id, domain_env_id, environnement " + "FROM servers WHERE hostname=:h"), + {"h": hostname}).fetchone() if not srv: stats["not_found"] += 1 continue - # Calcule diff (pour ne pas ecraser si valeur identique) - diff = {} - for k, v in fields.items(): - current = getattr(srv, k, None) - if current != v: - diff[k] = (current, v) + # Auto-create domain/env/de + de_id = srv.domain_env_id + if dom_name and env_name: + if dom_name not in seen_dom: + before = conn.execute(text("SELECT COUNT(*) FROM domains")).scalar() + if not args.dry_run: + did = get_or_create_domain(conn, dom_name) + else: + row_ = conn.execute(text("SELECT id FROM domains WHERE name=:n"), + {"n": dom_name}).fetchone() + did = row_.id if row_ else -1 + after = conn.execute(text("SELECT COUNT(*) FROM domains")).scalar() + if after > before: + stats["dom_created"] += 1 + seen_dom[dom_name] = did - if not diff: - stats["unchanged"] += 1 - continue + if env_name not in seen_env: + before = conn.execute(text("SELECT COUNT(*) FROM environments")).scalar() + if not args.dry_run: + eid = get_or_create_env(conn, env_name) + else: + row_ = conn.execute(text("SELECT id FROM environments WHERE name=:n"), + {"n": env_name}).fetchone() + eid = row_.id if row_ else -1 + after = conn.execute(text("SELECT COUNT(*) FROM environments")).scalar() + if after > before: + stats["env_created"] += 1 + seen_env[env_name] = eid - if args.dry_run: - changes_detail.append((hostname, diff)) - else: - set_clauses = ", ".join(f"{k}=:{k}" for k in diff) - params = {k: v[1] for k, v in diff.items()} - params["sid"] = srv.id - conn.execute(text(f"UPDATE servers SET {set_clauses} WHERE id=:sid"), params) - stats["updated"] += 1 + did = seen_dom[dom_name] + eid = seen_env[env_name] + if did > 0 and eid > 0: + if not args.dry_run: + before = conn.execute(text("SELECT COUNT(*) FROM domain_environments")).scalar() + de_id = get_or_create_dom_env(conn, did, eid) + after = conn.execute(text("SELECT COUNT(*) FROM domain_environments")).scalar() + if after > before: + stats["de_created"] += 1 + # Sync responsable/referent sur domain_environments (max 1 valeur — on garde la derniere vue) + if resp or ref: + up = {} + if resp: + up["resp"] = resp + if ref: + up["ref"] = ref + sets = [] + if "resp" in up: + sets.append("responsable_nom=:resp") + if "ref" in up: + sets.append("referent_nom=:ref") + if sets: + up["id"] = de_id + conn.execute(text( + f"UPDATE domain_environments SET {', '.join(sets)} " + f"WHERE id=:id AND (responsable_nom IS NULL OR referent_nom IS NULL)" + ), up) + stats["de_resp_updated"] += 1 - if args.dry_run and changes_detail: - print(f"\n[DRY-RUN] Changements ({len(changes_detail)}) :") - for hostname, diff in changes_detail[:30]: - print(f" {hostname}:") - for k, (old, new) in diff.items(): - print(f" {k}: {old!r} -> {new!r}") - if len(changes_detail) > 30: - print(f" ... ({len(changes_detail)-30} autres)") + # Update serveur: domain_env_id + environnement plain-text + updates = {} + if de_id and srv.domain_env_id != de_id: + updates["domain_env_id"] = de_id + if env_name and srv.environnement != env_name: + updates["environnement"] = env_name[:50] - print(f"\n[DONE] Maj: {stats['updated']} | Inchanges: {stats['unchanged']} " - f"| Hors base: {stats['not_found']} | Skip: {stats['skipped']}") + if updates: + if args.dry_run: + changes.append((hostname, updates)) + else: + sets = ", ".join(f"{k}=:{k}" for k in updates) + params = dict(updates); params["sid"] = srv.id + conn.execute(text(f"UPDATE servers SET {sets} WHERE id=:sid"), params) + stats["updated"] += 1 + + if args.dry_run and changes: + print(f"\n[DRY-RUN] {len(changes)} serveurs a mettre a jour (premiers 20):") + for h, u in changes[:20]: + print(f" {h}: {u}") + + print(f"\n[DONE] servers maj: {stats['updated']} | domains crees: {stats['dom_created']} " + f"| envs crees: {stats['env_created']} | (dom,env) crees: {stats['de_created']} " + f"| resp/ref syncs: {stats['de_resp_updated']} | hors base: {stats['not_found']} " + f"| skip: {stats['skipped']}") conn.close()