Qualys refresh: pagination + per-row savepoint to isolate errors

app/services/qualys_service.py

@@ -535,31 +535,56 @@ def get_cache_stats():
 
 
 def refresh_all_agents(db):
-    """Rafraichit tous les agents depuis l'API Qualys QPS (bulk)"""
+    """Rafraichit tous les agents depuis l'API Qualys QPS (bulk, paginé)"""
     qualys_url, qualys_user, qualys_pass, qualys_proxy = _get_qualys_creds(db)
     if not qualys_user:
         return {"ok": False, "msg": "Credentials Qualys non configurés"}
     proxies = {"https": qualys_proxy, "http": qualys_proxy} if qualys_proxy else None
 
-    stats = {"created": 0, "updated": 0, "errors": 0}
+    stats = {"created": 0, "updated": 0, "errors": 0, "pages": 0}
+    last_id = None
+    max_pages = 20  # garde-fou
 
+    # Autocommit sur la session courante
+    try:
+        db.commit()  # flush any pending
+    except Exception:
+        pass
+
+    while stats["pages"] < max_pages:
+        stats["pages"] += 1
+        criteria = [{"field": "tagName", "operator": "CONTAINS", "value": "server"}]
+        if last_id:
+            criteria.append({"field": "id", "operator": "GREATER", "value": str(last_id)})
+        payload = {"ServiceRequest": {
+            "preferences": {"limitResults": 100},
+            "filters": {"Criteria": criteria}
+        }}
         try:
             r = requests.post(
                 f"{qualys_url}/qps/rest/2.0/search/am/hostasset",
-            json={"ServiceRequest": {"preferences": {"limitResults": 1000}, "filters": {"Criteria": [{"field": "tagName", "operator": "CONTAINS", "value": "server"}]}}},
+                json=payload, auth=(qualys_user, qualys_pass),
-            auth=(qualys_user, qualys_pass),
                 verify=False, timeout=120, proxies=proxies,
                 headers={"X-Requested-With": "PatchCenter", "Content-Type": "application/json"})
         except Exception as e:
-        return {"ok": False, "msg": str(e)}
+            return {"ok": False, "msg": f"page {stats['pages']}: {e}", **stats}
 
         if r.status_code != 200 or "SUCCESS" not in r.text:
-        return {"ok": False, "msg": f"API HTTP {r.status_code}"}
+            return {"ok": False, "msg": f"HTTP {r.status_code} page {stats['pages']}", **stats}
 
-    for block in r.text.split("<HostAsset>")[1:]:
+        blocks = r.text.split("<HostAsset>")[1:]
+        if not blocks:
+            break
+
+        new_last_id = last_id
+        for block in blocks:
             block = block.split("</HostAsset>")[0]
             try:
                 asset_id = (parse_xml(block, "id") or [""])[0]
+                if asset_id and asset_id.isdigit():
+                    aid_int = int(asset_id)
+                    if new_last_id is None or aid_int > new_last_id:
+                        new_last_id = aid_int
                 name = (parse_xml(block, "name") or [""])[0]
                 hostname = name.split(".")[0].lower() if name else ""
                 address = (parse_xml(block, "address") or [""])[0]
@@ -582,11 +607,12 @@ def refresh_all_agents(db):
                 elif "windows" in os_val.lower():
                     os_family = "windows"
 
-            # Match server
+                # Savepoint pour isoler les erreurs
+                try:
+                    sp = db.begin_nested()
                     srv = db.execute(text("SELECT id FROM servers WHERE LOWER(hostname)=LOWER(:h)"),
                                      {"h": hostname}).fetchone()
                     server_id = srv.id if srv else None
-            # Sync du FQDN Qualys vers servers.fqdn si présent
                     if server_id and fqdn:
                         db.execute(text("UPDATE servers SET fqdn=:fqdn WHERE id=:sid AND (fqdn IS NULL OR fqdn='')"),
                                    {"fqdn": fqdn, "sid": server_id})
@@ -612,11 +638,22 @@ def refresh_all_agents(db):
                              "ip": address or None, "os": os_val, "osf": os_family,
                              "ast": agent_status, "av": agent_version, "lc": last_checkin, "sid": server_id})
                         stats["created"] += 1
-
+                    sp.commit()
+                except Exception:
+                    try: sp.rollback()
+                    except Exception: pass
+                    stats["errors"] += 1
             except Exception:
                 stats["errors"] += 1
 
         db.commit()
+
+        if "<hasMoreRecords>true</hasMoreRecords>" not in r.text:
+            break
+        if new_last_id == last_id:
+            break
+        last_id = new_last_id
+
     stats["ok"] = True
-    stats["msg"] = f"{stats['created']} créés, {stats['updated']} mis à jour"
+    stats["msg"] = f"{stats['created']} créés, {stats['updated']} mis à jour ({stats['pages']} pages, {stats['errors']} erreurs)"
     return stats