From 55cd35eaf15011aeb1060a7b8edd6cce72d3dcde Mon Sep 17 00:00:00 2001 From: Admin MPCZ Date: Thu, 16 Apr 2026 14:21:37 +0200 Subject: [PATCH] import_applications_ioda: gestion conflits nom_court (UNIQUE existant) Strategy SELECT puis INSERT/UPDATE plutot que ON CONFLICT: - Cherche par ioda_libelle d'abord, sinon par nom_court (apps non-IODA) - Si UPDATE existant -> enrichit avec champs IODA - Si INSERT et nom_court deja pris -> suffixe avec -IODA- Co-Authored-By: Claude Opus 4.6 (1M context) --- tools/import_applications_ioda.py | 83 +++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 27 deletions(-) diff --git a/tools/import_applications_ioda.py b/tools/import_applications_ioda.py index be3957c..5b01603 100644 --- a/tools/import_applications_ioda.py +++ b/tools/import_applications_ioda.py @@ -77,7 +77,36 @@ def parse_services(xlsx_path): return services -SQL_UPSERT = text(""" +SQL_FIND = text(""" + SELECT id FROM applications + WHERE ioda_libelle = :libelle + OR (ioda_libelle IS NULL AND nom_court = :nom_court) + LIMIT 1 +""") + +SQL_UPDATE = text(""" + UPDATE applications SET + nom_complet = :nom_complet, + description = COALESCE(:description, description), + editeur = COALESCE(:editeur, editeur), + ioda_libelle = :libelle, + ioda_lib_court = :lib_court, + ioda_code_pos = :code_pos, + ioda_type = :type, + ioda_statut = :statut, + ioda_alias = :alias, + ioda_perimetre = :perimetre, + ioda_dept_domaine = :dept_domaine, + ioda_resp_metier = :resp_metier, + ioda_resp_dsi = :resp_dsi, + ioda_nb_components = :nb_components, + ioda_commentaire = :commentaire, + ioda_imported_at = now(), + updated_at = now() + WHERE id = :id +""") + +SQL_INSERT = text(""" INSERT INTO applications ( nom_court, nom_complet, description, editeur, criticite, ioda_libelle, ioda_lib_court, ioda_code_pos, ioda_type, ioda_statut, @@ -91,24 +120,6 @@ SQL_UPSERT = text(""" :resp_dsi, :nb_components, :commentaire, now(), now(), now() ) - ON CONFLICT (ioda_libelle) WHERE ioda_libelle IS NOT NULL - DO UPDATE SET - nom_complet = EXCLUDED.nom_complet, - description = COALESCE(EXCLUDED.description, applications.description), - editeur = COALESCE(EXCLUDED.editeur, applications.editeur), - ioda_lib_court = EXCLUDED.ioda_lib_court, - ioda_code_pos = EXCLUDED.ioda_code_pos, - ioda_type = EXCLUDED.ioda_type, - ioda_statut = EXCLUDED.ioda_statut, - ioda_alias = EXCLUDED.ioda_alias, - ioda_perimetre = EXCLUDED.ioda_perimetre, - ioda_dept_domaine = EXCLUDED.ioda_dept_domaine, - ioda_resp_metier = EXCLUDED.ioda_resp_metier, - ioda_resp_dsi = EXCLUDED.ioda_resp_dsi, - ioda_nb_components = EXCLUDED.ioda_nb_components, - ioda_commentaire = EXCLUDED.ioda_commentaire, - ioda_imported_at = now(), - updated_at = now() """) @@ -125,21 +136,39 @@ def main(): engine = create_engine(DATABASE_URL) print(f"[INFO] DB: {DATABASE_URL.rsplit('@', 1)[-1]}") - inserted = updated = 0 + inserted = updated = skipped = 0 with engine.begin() as conn: + used_nom_court = set() for svc in services: - # nom_court max 50 chars: prefere lib_court, sinon truncate libelle - nom_court = (svc["lib_court"] or svc["libelle"])[:50] + base_nc = (svc["lib_court"] or svc["libelle"])[:50] params = { **svc, - "nom_court": nom_court, + "nom_court": base_nc, "nom_complet": svc["libelle"][:200], } - r = conn.execute(SQL_UPSERT, params) - # Discriminer insert vs update via xmin trick? Plus simple: compter rowcount - inserted += 1 # upsert ne distingue pas, on log la totale + existing = conn.execute(SQL_FIND, params).fetchone() + if existing: + conn.execute(SQL_UPDATE, {**params, "id": existing[0]}) + updated += 1 + else: + # Eviter collision nom_court avec autre app non-IODA en BDD + # ou avec un autre service IODA deja traite cette session + nc = base_nc + if nc in used_nom_court: + # ajout suffixe POS pour unicite + suf = svc.get("code_pos") or "X" + nc = (base_nc[:46] + "-" + suf)[:50] + used_nom_court.add(nc) + # Verif collision DB + row = conn.execute(text("SELECT 1 FROM applications WHERE nom_court=:nc"), + {"nc": nc}).fetchone() + if row: + nc = (base_nc[:42] + "-IODA-" + (svc.get("code_pos") or "X"))[:50] + params["nom_court"] = nc + conn.execute(SQL_INSERT, params) + inserted += 1 - print(f"[OK] Upsert termine — {inserted} services traites") + print(f"[OK] Termine — INSERT: {inserted} | UPDATE: {updated} | SKIP: {skipped}") print(f"[INFO] Verif: SELECT COUNT(*) FROM applications WHERE ioda_libelle IS NOT NULL;")