align_from_ayoub: switch vers jointure (auto-cree domains/envs/de)
Auto-cree les domains / environments / domain_environments absents en utilisant les valeurs iTop verbatim (domains.name='BI', etc.), avec codes slug auto-generes (evite collision). Pour chaque serveur: - set servers.domain_env_id vers la paire (domaine, env) - sync servers.environnement plain-text (pour filtre/affichage existant) - populate domain_environments.responsable_nom / referent_nom si NULL Preserve les valeurs existantes non-NULL dans domain_environments.
This commit is contained in:
parent
c16a360cdd
commit
b6bb7e2edd
@ -1,12 +1,18 @@
|
||||
"""Alignement servers depuis fichier Excel Ayoub (Planning Patching 2026_ayoub.xlsx).
|
||||
|
||||
Lit le sheet 'Serveurs patchables 2026' et met a jour pour chaque hostname :
|
||||
- domaine (col D : Domaine)
|
||||
- environnement (col C : Environnement — override si vide)
|
||||
- responsable_nom (col H : Responsable Domaine DTS)
|
||||
- referent_nom (col J : Referent technique)
|
||||
Lit le sheet 'Serveurs patchables 2026' et aligne via JOINTURE:
|
||||
1. Auto-cree les domains/environments absents (valeurs iTop verbatim)
|
||||
2. Auto-cree les paires domain_environments
|
||||
3. Set servers.domain_env_id
|
||||
4. Sync servers.environnement (plain-text pour filtre)
|
||||
5. Set domain_environments.responsable_nom / referent_nom
|
||||
|
||||
Ajoute la colonne servers.domaine si absente (varchar(100)).
|
||||
Colonnes Excel lues:
|
||||
- Asset Name -> servers.hostname (match)
|
||||
- Domaine -> domains.name (code auto-genere)
|
||||
- Environnement -> environments.name (code auto-genere)
|
||||
- Responsable Domaine DTS -> domain_environments.responsable_nom
|
||||
- Referent technique -> domain_environments.referent_nom
|
||||
|
||||
Usage:
|
||||
python tools/align_from_ayoub.py <fichier.xlsx> [--sheet "Serveurs patchables 2026"] [--dry-run]
|
||||
@ -14,8 +20,9 @@ Usage:
|
||||
Requiert openpyxl: pip install openpyxl
|
||||
"""
|
||||
import os
|
||||
import argparse
|
||||
import re
|
||||
import argparse
|
||||
import unicodedata
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
try:
|
||||
@ -31,13 +38,75 @@ NBSP = "\u00a0"
|
||||
|
||||
|
||||
def clean(v):
|
||||
"""Normalise une cellule Excel: strip + enleve nbsp + None si vide."""
|
||||
if v is None:
|
||||
return None
|
||||
s = str(v).replace(NBSP, " ").strip()
|
||||
return s or None
|
||||
|
||||
|
||||
def slugify(s, maxlen=10):
|
||||
"""Slug ASCII lowercase pour code (domain.code varchar(10))."""
|
||||
if not s:
|
||||
return None
|
||||
nfkd = unicodedata.normalize("NFKD", s)
|
||||
ascii_str = "".join(c for c in nfkd if not unicodedata.combining(c))
|
||||
ascii_str = re.sub(r"[^a-zA-Z0-9]+", "", ascii_str).lower()
|
||||
return ascii_str[:maxlen] or None
|
||||
|
||||
|
||||
def get_or_create_domain(conn, name):
|
||||
row = conn.execute(text("SELECT id, code FROM domains WHERE name=:n"),
|
||||
{"n": name}).fetchone()
|
||||
if row:
|
||||
return row.id
|
||||
code = slugify(name, 10)
|
||||
# Eviter collision de code
|
||||
suffix = 0
|
||||
base_code = code
|
||||
while conn.execute(text("SELECT 1 FROM domains WHERE code=:c"),
|
||||
{"c": code}).fetchone():
|
||||
suffix += 1
|
||||
code = (base_code[: 10 - len(str(suffix))] + str(suffix))
|
||||
conn.execute(text(
|
||||
"INSERT INTO domains (name, code, display_order) VALUES (:n, :c, 99)"
|
||||
), {"n": name, "c": code})
|
||||
return conn.execute(text("SELECT id FROM domains WHERE name=:n"),
|
||||
{"n": name}).fetchone().id
|
||||
|
||||
|
||||
def get_or_create_env(conn, name):
|
||||
row = conn.execute(text("SELECT id FROM environments WHERE name=:n"),
|
||||
{"n": name}).fetchone()
|
||||
if row:
|
||||
return row.id
|
||||
code = slugify(name, 10)
|
||||
suffix = 0
|
||||
base_code = code
|
||||
while conn.execute(text("SELECT 1 FROM environments WHERE code=:c"),
|
||||
{"c": code}).fetchone():
|
||||
suffix += 1
|
||||
code = (base_code[: 10 - len(str(suffix))] + str(suffix))
|
||||
conn.execute(text(
|
||||
"INSERT INTO environments (name, code, display_order) VALUES (:n, :c, 99)"
|
||||
), {"n": name, "c": code})
|
||||
return conn.execute(text("SELECT id FROM environments WHERE name=:n"),
|
||||
{"n": name}).fetchone().id
|
||||
|
||||
|
||||
def get_or_create_dom_env(conn, domain_id, env_id):
|
||||
row = conn.execute(text(
|
||||
"SELECT id FROM domain_environments WHERE domain_id=:d AND environment_id=:e"
|
||||
), {"d": domain_id, "e": env_id}).fetchone()
|
||||
if row:
|
||||
return row.id
|
||||
conn.execute(text(
|
||||
"INSERT INTO domain_environments (domain_id, environment_id) VALUES (:d, :e)"
|
||||
), {"d": domain_id, "e": env_id})
|
||||
return conn.execute(text(
|
||||
"SELECT id FROM domain_environments WHERE domain_id=:d AND environment_id=:e"
|
||||
), {"d": domain_id, "e": env_id}).fetchone().id
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("xlsx_path")
|
||||
@ -48,31 +117,24 @@ def main():
|
||||
engine = create_engine(DATABASE_URL)
|
||||
print(f"[INFO] DB: {DATABASE_URL.split('@')[-1]}")
|
||||
print(f"[INFO] Fichier: {args.xlsx_path}")
|
||||
print(f"[INFO] Sheet: {args.sheet}")
|
||||
|
||||
conn = engine.connect().execution_options(isolation_level="AUTOCOMMIT")
|
||||
|
||||
# 1. Ajoute colonne domaine si absente (idempotent)
|
||||
conn.execute(text("ALTER TABLE servers ADD COLUMN IF NOT EXISTS domaine varchar(100)"))
|
||||
|
||||
# 2. Lecture Excel
|
||||
wb = openpyxl.load_workbook(args.xlsx_path, data_only=True)
|
||||
if args.sheet not in wb.sheetnames:
|
||||
print(f"[ERR] Sheet '{args.sheet}' introuvable. Sheets: {wb.sheetnames}")
|
||||
return
|
||||
ws = wb[args.sheet]
|
||||
|
||||
# Detection dynamique des colonnes via header (ligne 1)
|
||||
header = [clean(c.value) for c in ws[1]]
|
||||
print(f"[INFO] Header: {header[:12]}...")
|
||||
|
||||
def col_idx(name_candidates):
|
||||
def col_idx(candidates):
|
||||
for i, h in enumerate(header):
|
||||
if h and any(cand.lower() in h.lower() for cand in name_candidates):
|
||||
if h and any(c.lower() in h.lower() for c in candidates):
|
||||
return i
|
||||
return -1
|
||||
|
||||
idx_host = col_idx(["Asset Name", "Hostname", "Nom"])
|
||||
idx_host = col_idx(["Asset Name", "Hostname"])
|
||||
idx_env = col_idx(["Environnement"])
|
||||
idx_dom = col_idx(["Domaine"])
|
||||
idx_resp = col_idx(["Responsable Domaine"])
|
||||
@ -85,81 +147,118 @@ def main():
|
||||
print("[ERR] Colonne Asset Name/Hostname introuvable")
|
||||
return
|
||||
|
||||
stats = {"updated": 0, "unchanged": 0, "not_found": 0, "skipped": 0}
|
||||
changes_detail = []
|
||||
stats = {"updated": 0, "dom_created": 0, "env_created": 0, "de_created": 0,
|
||||
"not_found": 0, "skipped": 0, "de_resp_updated": 0}
|
||||
seen_dom = {}
|
||||
seen_env = {}
|
||||
changes = []
|
||||
|
||||
for row in ws.iter_rows(min_row=2, values_only=True):
|
||||
hostname = clean(row[idx_host]) if idx_host < len(row) else None
|
||||
if not hostname or not any(c.isalpha() for c in hostname):
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
# Extract sans suffixe FQDN si present
|
||||
hostname = hostname.split(".")[0].lower()
|
||||
|
||||
fields = {}
|
||||
if idx_dom >= 0 and idx_dom < len(row):
|
||||
v = clean(row[idx_dom])
|
||||
if v:
|
||||
fields["domaine"] = v[:100]
|
||||
if idx_env >= 0 and idx_env < len(row):
|
||||
v = clean(row[idx_env])
|
||||
if v:
|
||||
fields["environnement"] = v[:50]
|
||||
if idx_resp >= 0 and idx_resp < len(row):
|
||||
v = clean(row[idx_resp])
|
||||
if v:
|
||||
# Enleve les virgules multiples, normalise espaces
|
||||
v = re.sub(r"\s+", " ", v)
|
||||
fields["responsable_nom"] = v[:200]
|
||||
if idx_ref >= 0 and idx_ref < len(row):
|
||||
v = clean(row[idx_ref])
|
||||
if v:
|
||||
v = re.sub(r"\s+", " ", v)
|
||||
fields["referent_nom"] = v[:200]
|
||||
dom_name = clean(row[idx_dom]) if idx_dom >= 0 and idx_dom < len(row) else None
|
||||
env_name = clean(row[idx_env]) if idx_env >= 0 and idx_env < len(row) else None
|
||||
resp = clean(row[idx_resp]) if idx_resp >= 0 and idx_resp < len(row) else None
|
||||
ref = clean(row[idx_ref]) if idx_ref >= 0 and idx_ref < len(row) else None
|
||||
if resp:
|
||||
resp = re.sub(r"\s+", " ", resp)[:100]
|
||||
if ref:
|
||||
ref = re.sub(r"\s+", " ", ref)[:100]
|
||||
|
||||
if not fields:
|
||||
continue
|
||||
|
||||
srv = conn.execute(text(
|
||||
"SELECT id, domaine, environnement, responsable_nom, referent_nom "
|
||||
"FROM servers WHERE hostname=:h"
|
||||
), {"h": hostname}).fetchone()
|
||||
srv = conn.execute(text("SELECT id, domain_env_id, environnement "
|
||||
"FROM servers WHERE hostname=:h"),
|
||||
{"h": hostname}).fetchone()
|
||||
if not srv:
|
||||
stats["not_found"] += 1
|
||||
continue
|
||||
|
||||
# Calcule diff (pour ne pas ecraser si valeur identique)
|
||||
diff = {}
|
||||
for k, v in fields.items():
|
||||
current = getattr(srv, k, None)
|
||||
if current != v:
|
||||
diff[k] = (current, v)
|
||||
# Auto-create domain/env/de
|
||||
de_id = srv.domain_env_id
|
||||
if dom_name and env_name:
|
||||
if dom_name not in seen_dom:
|
||||
before = conn.execute(text("SELECT COUNT(*) FROM domains")).scalar()
|
||||
if not args.dry_run:
|
||||
did = get_or_create_domain(conn, dom_name)
|
||||
else:
|
||||
row_ = conn.execute(text("SELECT id FROM domains WHERE name=:n"),
|
||||
{"n": dom_name}).fetchone()
|
||||
did = row_.id if row_ else -1
|
||||
after = conn.execute(text("SELECT COUNT(*) FROM domains")).scalar()
|
||||
if after > before:
|
||||
stats["dom_created"] += 1
|
||||
seen_dom[dom_name] = did
|
||||
|
||||
if not diff:
|
||||
stats["unchanged"] += 1
|
||||
continue
|
||||
if env_name not in seen_env:
|
||||
before = conn.execute(text("SELECT COUNT(*) FROM environments")).scalar()
|
||||
if not args.dry_run:
|
||||
eid = get_or_create_env(conn, env_name)
|
||||
else:
|
||||
row_ = conn.execute(text("SELECT id FROM environments WHERE name=:n"),
|
||||
{"n": env_name}).fetchone()
|
||||
eid = row_.id if row_ else -1
|
||||
after = conn.execute(text("SELECT COUNT(*) FROM environments")).scalar()
|
||||
if after > before:
|
||||
stats["env_created"] += 1
|
||||
seen_env[env_name] = eid
|
||||
|
||||
if args.dry_run:
|
||||
changes_detail.append((hostname, diff))
|
||||
else:
|
||||
set_clauses = ", ".join(f"{k}=:{k}" for k in diff)
|
||||
params = {k: v[1] for k, v in diff.items()}
|
||||
params["sid"] = srv.id
|
||||
conn.execute(text(f"UPDATE servers SET {set_clauses} WHERE id=:sid"), params)
|
||||
stats["updated"] += 1
|
||||
did = seen_dom[dom_name]
|
||||
eid = seen_env[env_name]
|
||||
if did > 0 and eid > 0:
|
||||
if not args.dry_run:
|
||||
before = conn.execute(text("SELECT COUNT(*) FROM domain_environments")).scalar()
|
||||
de_id = get_or_create_dom_env(conn, did, eid)
|
||||
after = conn.execute(text("SELECT COUNT(*) FROM domain_environments")).scalar()
|
||||
if after > before:
|
||||
stats["de_created"] += 1
|
||||
# Sync responsable/referent sur domain_environments (max 1 valeur — on garde la derniere vue)
|
||||
if resp or ref:
|
||||
up = {}
|
||||
if resp:
|
||||
up["resp"] = resp
|
||||
if ref:
|
||||
up["ref"] = ref
|
||||
sets = []
|
||||
if "resp" in up:
|
||||
sets.append("responsable_nom=:resp")
|
||||
if "ref" in up:
|
||||
sets.append("referent_nom=:ref")
|
||||
if sets:
|
||||
up["id"] = de_id
|
||||
conn.execute(text(
|
||||
f"UPDATE domain_environments SET {', '.join(sets)} "
|
||||
f"WHERE id=:id AND (responsable_nom IS NULL OR referent_nom IS NULL)"
|
||||
), up)
|
||||
stats["de_resp_updated"] += 1
|
||||
|
||||
if args.dry_run and changes_detail:
|
||||
print(f"\n[DRY-RUN] Changements ({len(changes_detail)}) :")
|
||||
for hostname, diff in changes_detail[:30]:
|
||||
print(f" {hostname}:")
|
||||
for k, (old, new) in diff.items():
|
||||
print(f" {k}: {old!r} -> {new!r}")
|
||||
if len(changes_detail) > 30:
|
||||
print(f" ... ({len(changes_detail)-30} autres)")
|
||||
# Update serveur: domain_env_id + environnement plain-text
|
||||
updates = {}
|
||||
if de_id and srv.domain_env_id != de_id:
|
||||
updates["domain_env_id"] = de_id
|
||||
if env_name and srv.environnement != env_name:
|
||||
updates["environnement"] = env_name[:50]
|
||||
|
||||
print(f"\n[DONE] Maj: {stats['updated']} | Inchanges: {stats['unchanged']} "
|
||||
f"| Hors base: {stats['not_found']} | Skip: {stats['skipped']}")
|
||||
if updates:
|
||||
if args.dry_run:
|
||||
changes.append((hostname, updates))
|
||||
else:
|
||||
sets = ", ".join(f"{k}=:{k}" for k in updates)
|
||||
params = dict(updates); params["sid"] = srv.id
|
||||
conn.execute(text(f"UPDATE servers SET {sets} WHERE id=:sid"), params)
|
||||
stats["updated"] += 1
|
||||
|
||||
if args.dry_run and changes:
|
||||
print(f"\n[DRY-RUN] {len(changes)} serveurs a mettre a jour (premiers 20):")
|
||||
for h, u in changes[:20]:
|
||||
print(f" {h}: {u}")
|
||||
|
||||
print(f"\n[DONE] servers maj: {stats['updated']} | domains crees: {stats['dom_created']} "
|
||||
f"| envs crees: {stats['env_created']} | (dom,env) crees: {stats['de_created']} "
|
||||
f"| resp/ref syncs: {stats['de_resp_updated']} | hors base: {stats['not_found']} "
|
||||
f"| skip: {stats['skipped']}")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user