import_applications_ioda: gestion conflits nom_court (UNIQUE existant)

Strategy SELECT puis INSERT/UPDATE plutot que ON CONFLICT:
- Cherche par ioda_libelle d'abord, sinon par nom_court (apps non-IODA)
- Si UPDATE existant -> enrichit avec champs IODA
- Si INSERT et nom_court deja pris -> suffixe avec -IODA-<POS>

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Pierre & Lumière 2026-04-16 14:21:37 +02:00
parent 34c8025b56
commit 55cd35eaf1

View File

@ -77,7 +77,36 @@ def parse_services(xlsx_path):
return services return services
SQL_UPSERT = text(""" SQL_FIND = text("""
SELECT id FROM applications
WHERE ioda_libelle = :libelle
OR (ioda_libelle IS NULL AND nom_court = :nom_court)
LIMIT 1
""")
SQL_UPDATE = text("""
UPDATE applications SET
nom_complet = :nom_complet,
description = COALESCE(:description, description),
editeur = COALESCE(:editeur, editeur),
ioda_libelle = :libelle,
ioda_lib_court = :lib_court,
ioda_code_pos = :code_pos,
ioda_type = :type,
ioda_statut = :statut,
ioda_alias = :alias,
ioda_perimetre = :perimetre,
ioda_dept_domaine = :dept_domaine,
ioda_resp_metier = :resp_metier,
ioda_resp_dsi = :resp_dsi,
ioda_nb_components = :nb_components,
ioda_commentaire = :commentaire,
ioda_imported_at = now(),
updated_at = now()
WHERE id = :id
""")
SQL_INSERT = text("""
INSERT INTO applications ( INSERT INTO applications (
nom_court, nom_complet, description, editeur, criticite, nom_court, nom_complet, description, editeur, criticite,
ioda_libelle, ioda_lib_court, ioda_code_pos, ioda_type, ioda_statut, ioda_libelle, ioda_lib_court, ioda_code_pos, ioda_type, ioda_statut,
@ -91,24 +120,6 @@ SQL_UPSERT = text("""
:resp_dsi, :nb_components, :commentaire, now(), :resp_dsi, :nb_components, :commentaire, now(),
now(), now() now(), now()
) )
ON CONFLICT (ioda_libelle) WHERE ioda_libelle IS NOT NULL
DO UPDATE SET
nom_complet = EXCLUDED.nom_complet,
description = COALESCE(EXCLUDED.description, applications.description),
editeur = COALESCE(EXCLUDED.editeur, applications.editeur),
ioda_lib_court = EXCLUDED.ioda_lib_court,
ioda_code_pos = EXCLUDED.ioda_code_pos,
ioda_type = EXCLUDED.ioda_type,
ioda_statut = EXCLUDED.ioda_statut,
ioda_alias = EXCLUDED.ioda_alias,
ioda_perimetre = EXCLUDED.ioda_perimetre,
ioda_dept_domaine = EXCLUDED.ioda_dept_domaine,
ioda_resp_metier = EXCLUDED.ioda_resp_metier,
ioda_resp_dsi = EXCLUDED.ioda_resp_dsi,
ioda_nb_components = EXCLUDED.ioda_nb_components,
ioda_commentaire = EXCLUDED.ioda_commentaire,
ioda_imported_at = now(),
updated_at = now()
""") """)
@ -125,21 +136,39 @@ def main():
engine = create_engine(DATABASE_URL) engine = create_engine(DATABASE_URL)
print(f"[INFO] DB: {DATABASE_URL.rsplit('@', 1)[-1]}") print(f"[INFO] DB: {DATABASE_URL.rsplit('@', 1)[-1]}")
inserted = updated = 0 inserted = updated = skipped = 0
with engine.begin() as conn: with engine.begin() as conn:
used_nom_court = set()
for svc in services: for svc in services:
# nom_court max 50 chars: prefere lib_court, sinon truncate libelle base_nc = (svc["lib_court"] or svc["libelle"])[:50]
nom_court = (svc["lib_court"] or svc["libelle"])[:50]
params = { params = {
**svc, **svc,
"nom_court": nom_court, "nom_court": base_nc,
"nom_complet": svc["libelle"][:200], "nom_complet": svc["libelle"][:200],
} }
r = conn.execute(SQL_UPSERT, params) existing = conn.execute(SQL_FIND, params).fetchone()
# Discriminer insert vs update via xmin trick? Plus simple: compter rowcount if existing:
inserted += 1 # upsert ne distingue pas, on log la totale conn.execute(SQL_UPDATE, {**params, "id": existing[0]})
updated += 1
else:
# Eviter collision nom_court avec autre app non-IODA en BDD
# ou avec un autre service IODA deja traite cette session
nc = base_nc
if nc in used_nom_court:
# ajout suffixe POS pour unicite
suf = svc.get("code_pos") or "X"
nc = (base_nc[:46] + "-" + suf)[:50]
used_nom_court.add(nc)
# Verif collision DB
row = conn.execute(text("SELECT 1 FROM applications WHERE nom_court=:nc"),
{"nc": nc}).fetchone()
if row:
nc = (base_nc[:42] + "-IODA-" + (svc.get("code_pos") or "X"))[:50]
params["nom_court"] = nc
conn.execute(SQL_INSERT, params)
inserted += 1
print(f"[OK] Upsert termine — {inserted} services traites") print(f"[OK] Termine — INSERT: {inserted} | UPDATE: {updated} | SKIP: {skipped}")
print(f"[INFO] Verif: SELECT COUNT(*) FROM applications WHERE ioda_libelle IS NOT NULL;") print(f"[INFO] Verif: SELECT COUNT(*) FROM applications WHERE ioda_libelle IS NOT NULL;")