import_applications_ioda: gestion conflits nom_court (UNIQUE existant)
Strategy SELECT puis INSERT/UPDATE plutot que ON CONFLICT: - Cherche par ioda_libelle d'abord, sinon par nom_court (apps non-IODA) - Si UPDATE existant -> enrichit avec champs IODA - Si INSERT et nom_court deja pris -> suffixe avec -IODA-<POS> Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
34c8025b56
commit
55cd35eaf1
@ -77,7 +77,36 @@ def parse_services(xlsx_path):
|
||||
return services
|
||||
|
||||
|
||||
SQL_UPSERT = text("""
|
||||
SQL_FIND = text("""
|
||||
SELECT id FROM applications
|
||||
WHERE ioda_libelle = :libelle
|
||||
OR (ioda_libelle IS NULL AND nom_court = :nom_court)
|
||||
LIMIT 1
|
||||
""")
|
||||
|
||||
SQL_UPDATE = text("""
|
||||
UPDATE applications SET
|
||||
nom_complet = :nom_complet,
|
||||
description = COALESCE(:description, description),
|
||||
editeur = COALESCE(:editeur, editeur),
|
||||
ioda_libelle = :libelle,
|
||||
ioda_lib_court = :lib_court,
|
||||
ioda_code_pos = :code_pos,
|
||||
ioda_type = :type,
|
||||
ioda_statut = :statut,
|
||||
ioda_alias = :alias,
|
||||
ioda_perimetre = :perimetre,
|
||||
ioda_dept_domaine = :dept_domaine,
|
||||
ioda_resp_metier = :resp_metier,
|
||||
ioda_resp_dsi = :resp_dsi,
|
||||
ioda_nb_components = :nb_components,
|
||||
ioda_commentaire = :commentaire,
|
||||
ioda_imported_at = now(),
|
||||
updated_at = now()
|
||||
WHERE id = :id
|
||||
""")
|
||||
|
||||
SQL_INSERT = text("""
|
||||
INSERT INTO applications (
|
||||
nom_court, nom_complet, description, editeur, criticite,
|
||||
ioda_libelle, ioda_lib_court, ioda_code_pos, ioda_type, ioda_statut,
|
||||
@ -91,24 +120,6 @@ SQL_UPSERT = text("""
|
||||
:resp_dsi, :nb_components, :commentaire, now(),
|
||||
now(), now()
|
||||
)
|
||||
ON CONFLICT (ioda_libelle) WHERE ioda_libelle IS NOT NULL
|
||||
DO UPDATE SET
|
||||
nom_complet = EXCLUDED.nom_complet,
|
||||
description = COALESCE(EXCLUDED.description, applications.description),
|
||||
editeur = COALESCE(EXCLUDED.editeur, applications.editeur),
|
||||
ioda_lib_court = EXCLUDED.ioda_lib_court,
|
||||
ioda_code_pos = EXCLUDED.ioda_code_pos,
|
||||
ioda_type = EXCLUDED.ioda_type,
|
||||
ioda_statut = EXCLUDED.ioda_statut,
|
||||
ioda_alias = EXCLUDED.ioda_alias,
|
||||
ioda_perimetre = EXCLUDED.ioda_perimetre,
|
||||
ioda_dept_domaine = EXCLUDED.ioda_dept_domaine,
|
||||
ioda_resp_metier = EXCLUDED.ioda_resp_metier,
|
||||
ioda_resp_dsi = EXCLUDED.ioda_resp_dsi,
|
||||
ioda_nb_components = EXCLUDED.ioda_nb_components,
|
||||
ioda_commentaire = EXCLUDED.ioda_commentaire,
|
||||
ioda_imported_at = now(),
|
||||
updated_at = now()
|
||||
""")
|
||||
|
||||
|
||||
@ -125,21 +136,39 @@ def main():
|
||||
engine = create_engine(DATABASE_URL)
|
||||
print(f"[INFO] DB: {DATABASE_URL.rsplit('@', 1)[-1]}")
|
||||
|
||||
inserted = updated = 0
|
||||
inserted = updated = skipped = 0
|
||||
with engine.begin() as conn:
|
||||
used_nom_court = set()
|
||||
for svc in services:
|
||||
# nom_court max 50 chars: prefere lib_court, sinon truncate libelle
|
||||
nom_court = (svc["lib_court"] or svc["libelle"])[:50]
|
||||
base_nc = (svc["lib_court"] or svc["libelle"])[:50]
|
||||
params = {
|
||||
**svc,
|
||||
"nom_court": nom_court,
|
||||
"nom_court": base_nc,
|
||||
"nom_complet": svc["libelle"][:200],
|
||||
}
|
||||
r = conn.execute(SQL_UPSERT, params)
|
||||
# Discriminer insert vs update via xmin trick? Plus simple: compter rowcount
|
||||
inserted += 1 # upsert ne distingue pas, on log la totale
|
||||
existing = conn.execute(SQL_FIND, params).fetchone()
|
||||
if existing:
|
||||
conn.execute(SQL_UPDATE, {**params, "id": existing[0]})
|
||||
updated += 1
|
||||
else:
|
||||
# Eviter collision nom_court avec autre app non-IODA en BDD
|
||||
# ou avec un autre service IODA deja traite cette session
|
||||
nc = base_nc
|
||||
if nc in used_nom_court:
|
||||
# ajout suffixe POS pour unicite
|
||||
suf = svc.get("code_pos") or "X"
|
||||
nc = (base_nc[:46] + "-" + suf)[:50]
|
||||
used_nom_court.add(nc)
|
||||
# Verif collision DB
|
||||
row = conn.execute(text("SELECT 1 FROM applications WHERE nom_court=:nc"),
|
||||
{"nc": nc}).fetchone()
|
||||
if row:
|
||||
nc = (base_nc[:42] + "-IODA-" + (svc.get("code_pos") or "X"))[:50]
|
||||
params["nom_court"] = nc
|
||||
conn.execute(SQL_INSERT, params)
|
||||
inserted += 1
|
||||
|
||||
print(f"[OK] Upsert termine — {inserted} services traites")
|
||||
print(f"[OK] Termine — INSERT: {inserted} | UPDATE: {updated} | SKIP: {skipped}")
|
||||
print(f"[INFO] Verif: SELECT COUNT(*) FROM applications WHERE ioda_libelle IS NOT NULL;")
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user