import_applications_ioda: gestion conflits nom_court (UNIQUE existant)
Strategy SELECT puis INSERT/UPDATE plutot que ON CONFLICT: - Cherche par ioda_libelle d'abord, sinon par nom_court (apps non-IODA) - Si UPDATE existant -> enrichit avec champs IODA - Si INSERT et nom_court deja pris -> suffixe avec -IODA-<POS> Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
34c8025b56
commit
55cd35eaf1
@ -77,7 +77,36 @@ def parse_services(xlsx_path):
|
|||||||
return services
|
return services
|
||||||
|
|
||||||
|
|
||||||
SQL_UPSERT = text("""
|
SQL_FIND = text("""
|
||||||
|
SELECT id FROM applications
|
||||||
|
WHERE ioda_libelle = :libelle
|
||||||
|
OR (ioda_libelle IS NULL AND nom_court = :nom_court)
|
||||||
|
LIMIT 1
|
||||||
|
""")
|
||||||
|
|
||||||
|
SQL_UPDATE = text("""
|
||||||
|
UPDATE applications SET
|
||||||
|
nom_complet = :nom_complet,
|
||||||
|
description = COALESCE(:description, description),
|
||||||
|
editeur = COALESCE(:editeur, editeur),
|
||||||
|
ioda_libelle = :libelle,
|
||||||
|
ioda_lib_court = :lib_court,
|
||||||
|
ioda_code_pos = :code_pos,
|
||||||
|
ioda_type = :type,
|
||||||
|
ioda_statut = :statut,
|
||||||
|
ioda_alias = :alias,
|
||||||
|
ioda_perimetre = :perimetre,
|
||||||
|
ioda_dept_domaine = :dept_domaine,
|
||||||
|
ioda_resp_metier = :resp_metier,
|
||||||
|
ioda_resp_dsi = :resp_dsi,
|
||||||
|
ioda_nb_components = :nb_components,
|
||||||
|
ioda_commentaire = :commentaire,
|
||||||
|
ioda_imported_at = now(),
|
||||||
|
updated_at = now()
|
||||||
|
WHERE id = :id
|
||||||
|
""")
|
||||||
|
|
||||||
|
SQL_INSERT = text("""
|
||||||
INSERT INTO applications (
|
INSERT INTO applications (
|
||||||
nom_court, nom_complet, description, editeur, criticite,
|
nom_court, nom_complet, description, editeur, criticite,
|
||||||
ioda_libelle, ioda_lib_court, ioda_code_pos, ioda_type, ioda_statut,
|
ioda_libelle, ioda_lib_court, ioda_code_pos, ioda_type, ioda_statut,
|
||||||
@ -91,24 +120,6 @@ SQL_UPSERT = text("""
|
|||||||
:resp_dsi, :nb_components, :commentaire, now(),
|
:resp_dsi, :nb_components, :commentaire, now(),
|
||||||
now(), now()
|
now(), now()
|
||||||
)
|
)
|
||||||
ON CONFLICT (ioda_libelle) WHERE ioda_libelle IS NOT NULL
|
|
||||||
DO UPDATE SET
|
|
||||||
nom_complet = EXCLUDED.nom_complet,
|
|
||||||
description = COALESCE(EXCLUDED.description, applications.description),
|
|
||||||
editeur = COALESCE(EXCLUDED.editeur, applications.editeur),
|
|
||||||
ioda_lib_court = EXCLUDED.ioda_lib_court,
|
|
||||||
ioda_code_pos = EXCLUDED.ioda_code_pos,
|
|
||||||
ioda_type = EXCLUDED.ioda_type,
|
|
||||||
ioda_statut = EXCLUDED.ioda_statut,
|
|
||||||
ioda_alias = EXCLUDED.ioda_alias,
|
|
||||||
ioda_perimetre = EXCLUDED.ioda_perimetre,
|
|
||||||
ioda_dept_domaine = EXCLUDED.ioda_dept_domaine,
|
|
||||||
ioda_resp_metier = EXCLUDED.ioda_resp_metier,
|
|
||||||
ioda_resp_dsi = EXCLUDED.ioda_resp_dsi,
|
|
||||||
ioda_nb_components = EXCLUDED.ioda_nb_components,
|
|
||||||
ioda_commentaire = EXCLUDED.ioda_commentaire,
|
|
||||||
ioda_imported_at = now(),
|
|
||||||
updated_at = now()
|
|
||||||
""")
|
""")
|
||||||
|
|
||||||
|
|
||||||
@ -125,21 +136,39 @@ def main():
|
|||||||
engine = create_engine(DATABASE_URL)
|
engine = create_engine(DATABASE_URL)
|
||||||
print(f"[INFO] DB: {DATABASE_URL.rsplit('@', 1)[-1]}")
|
print(f"[INFO] DB: {DATABASE_URL.rsplit('@', 1)[-1]}")
|
||||||
|
|
||||||
inserted = updated = 0
|
inserted = updated = skipped = 0
|
||||||
with engine.begin() as conn:
|
with engine.begin() as conn:
|
||||||
|
used_nom_court = set()
|
||||||
for svc in services:
|
for svc in services:
|
||||||
# nom_court max 50 chars: prefere lib_court, sinon truncate libelle
|
base_nc = (svc["lib_court"] or svc["libelle"])[:50]
|
||||||
nom_court = (svc["lib_court"] or svc["libelle"])[:50]
|
|
||||||
params = {
|
params = {
|
||||||
**svc,
|
**svc,
|
||||||
"nom_court": nom_court,
|
"nom_court": base_nc,
|
||||||
"nom_complet": svc["libelle"][:200],
|
"nom_complet": svc["libelle"][:200],
|
||||||
}
|
}
|
||||||
r = conn.execute(SQL_UPSERT, params)
|
existing = conn.execute(SQL_FIND, params).fetchone()
|
||||||
# Discriminer insert vs update via xmin trick? Plus simple: compter rowcount
|
if existing:
|
||||||
inserted += 1 # upsert ne distingue pas, on log la totale
|
conn.execute(SQL_UPDATE, {**params, "id": existing[0]})
|
||||||
|
updated += 1
|
||||||
|
else:
|
||||||
|
# Eviter collision nom_court avec autre app non-IODA en BDD
|
||||||
|
# ou avec un autre service IODA deja traite cette session
|
||||||
|
nc = base_nc
|
||||||
|
if nc in used_nom_court:
|
||||||
|
# ajout suffixe POS pour unicite
|
||||||
|
suf = svc.get("code_pos") or "X"
|
||||||
|
nc = (base_nc[:46] + "-" + suf)[:50]
|
||||||
|
used_nom_court.add(nc)
|
||||||
|
# Verif collision DB
|
||||||
|
row = conn.execute(text("SELECT 1 FROM applications WHERE nom_court=:nc"),
|
||||||
|
{"nc": nc}).fetchone()
|
||||||
|
if row:
|
||||||
|
nc = (base_nc[:42] + "-IODA-" + (svc.get("code_pos") or "X"))[:50]
|
||||||
|
params["nom_court"] = nc
|
||||||
|
conn.execute(SQL_INSERT, params)
|
||||||
|
inserted += 1
|
||||||
|
|
||||||
print(f"[OK] Upsert termine — {inserted} services traites")
|
print(f"[OK] Termine — INSERT: {inserted} | UPDATE: {updated} | SKIP: {skipped}")
|
||||||
print(f"[INFO] Verif: SELECT COUNT(*) FROM applications WHERE ioda_libelle IS NOT NULL;")
|
print(f"[INFO] Verif: SELECT COUNT(*) FROM applications WHERE ioda_libelle IS NOT NULL;")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user