Audit: jobs background paralleles + progression live

- Audit global/realtime: threads paralleles, job_id retourne immediat
- /audit/realtime/progress/{job_id}: KPIs + barre progression + tableau live
- Polling AJAX toutes les 2s, etapes animees (DNS/SSH/Audit/OK)
- PRETTY_NAME correction: extraction via grep -E 'PRETTY_NAME' + cut
- OS version: normalisation lors de save_audit_to_db (Debian GNU/Linux -> Debian X (Bookworm))
- Mise a jour base: itop sync bidirectionnel avec push OS version

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

app/routers/audit.py

@@ -5,7 +5,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse, StreamingResponse
 from fastapi.templating import Jinja2Templates
 from sqlalchemy import text
 from ..dependencies import get_db, get_current_user, get_user_perms, can_view, can_edit, can_admin, base_context
-from ..services.realtime_audit_service import audit_servers_list, save_audit_to_db
+from ..services.realtime_audit_service import audit_servers_list, save_audit_to_db, start_audit_job, get_audit_job
 from ..config import APP_NAME
 
 router = APIRouter()
@@ -134,7 +134,7 @@ async def audit_global(request: Request, db=Depends(get_db)):
     parallel = int(form.get("parallel", "5"))
 
     # Construire la requete
-    where = ["s.os_family = 'linux'", "s.etat = 'en_production'"]
+    where = ["s.os_family = 'linux'", "s.etat = 'production'"]
     params = {}
     if exclude_domains:
         where.append("d.code NOT IN :ed")
@@ -173,18 +173,9 @@ async def audit_global(request: Request, db=Depends(get_db)):
     if not hostnames:
         return RedirectResponse(url="/audit?msg=no_hosts", status_code=303)
 
-    # Lancer l'audit
+    # Lancer en arrière-plan
-    results = audit_servers_list(hostnames)
+    job_id = start_audit_job(hostnames)
-    request.app.state.last_audit_results = results
+    return RedirectResponse(url=f"/audit/realtime/progress/{job_id}", status_code=303)
-
-    ctx = base_context(request, db, user)
-    ctx.update({
-        "app_name": APP_NAME, "results": results,
-        "total": len(results),
-        "ok": sum(1 for r in results if r.get("status") == "OK"),
-        "failed": sum(1 for r in results if r.get("status") != "OK"),
-    })
-    return templates.TemplateResponse("audit_realtime_results.html", ctx)
 
 
 @router.post("/audit/realtime", response_class=HTMLResponse)
@@ -209,23 +200,57 @@ async def audit_realtime(request: Request, db=Depends(get_db),
     if not hostnames:
         return RedirectResponse(url="/audit?msg=no_hosts", status_code=303)
 
-    # Lancer l'audit
+    # Lancer en arrière-plan
-    results = audit_servers_list(hostnames)
+    job_id = start_audit_job(hostnames)
 
-    # Stocker en session (request.state) pour export/save
+    return RedirectResponse(url=f"/audit/realtime/progress/{job_id}", status_code=303)
-    request.app.state.last_audit_results = results
+
+
+@router.get("/audit/realtime/progress/{job_id}", response_class=HTMLResponse)
+async def audit_realtime_progress(request: Request, job_id: str, db=Depends(get_db)):
+    user = get_current_user(request)
+    if not user:
+        return RedirectResponse(url="/login")
+    perms = get_user_perms(db, user)
+    if not can_view(perms, "audit"):
+        return RedirectResponse(url="/audit")
+
+    job = get_audit_job(job_id)
+    if not job:
+        return RedirectResponse(url="/audit?msg=job_not_found", status_code=303)
 
     ctx = base_context(request, db, user)
-    ctx.update({
+    ctx.update({"app_name": APP_NAME, "job_id": job_id, "total": job["total"]})
-        "app_name": APP_NAME, "results": results,
+    return templates.TemplateResponse("audit_realtime_progress.html", ctx)
-        "total": len(results),
+
-        "ok": sum(1 for r in results if r.get("status") == "OK"),
+
-        "failed": sum(1 for r in results if r.get("status") != "OK"),
+@router.get("/audit/realtime/status/{job_id}")
+async def audit_realtime_status(request: Request, job_id: str, db=Depends(get_db)):
+    from fastapi.responses import JSONResponse
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"ok": False}, status_code=401)
+
+    job = get_audit_job(job_id)
+    if not job:
+        return JSONResponse({"ok": False, "msg": "Job introuvable"}, status_code=404)
+
+    import time
+    elapsed = int(time.time() - job["started_at"])
+
+    # When finished, store results in app state for save/export
+    if job["finished"]:
+        request.app.state.last_audit_results = job["results"]
+
+    return JSONResponse({
+        "ok": True,
+        "job_id": job_id,
+        "total": job["total"],
+        "done": job["done"],
+        "finished": job["finished"],
+        "elapsed": elapsed,
+        "servers": job["servers"],
     })
-    response = templates.TemplateResponse("audit_realtime_results.html", ctx)
-    response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0"
-    response.headers["Pragma"] = "no-cache"
-    return response
 
 
 @router.post("/audit/realtime/save")

app/services/realtime_audit_service.py

@@ -32,7 +32,7 @@ def _get_ssh_settings():
 
 # Commandes d'audit (simplifiees pour le temps reel)
 AUDIT_CMDS = {
-    "os_release": "cat /etc/redhat-release 2>/dev/null || head -1 /etc/os-release 2>/dev/null",
+    "os_release": "cat /etc/redhat-release 2>/dev/null || grep '^PRETTY_NAME=' /etc/os-release 2>/dev/null | cut -d'\"' -f2",
     "kernel": "uname -r",
     "uptime": "uptime -p 2>/dev/null || uptime",
     "selinux": "getenforce 2>/dev/null || echo N/A",
@@ -193,6 +193,124 @@ def audit_servers_list(hostnames):
     return results
 
 
+# ═══════════════════════════════════════════════
+# Background audit job manager
+# ═══════════════════════════════════════════════
+import threading
+import uuid
+import time as _time
+
+_audit_jobs = {}
+
+
+def start_audit_job(hostnames):
+    """Lance un audit en arriere-plan. Retourne le job_id."""
+    job_id = str(uuid.uuid4())[:8]
+    job = {
+        "id": job_id,
+        "started_at": _time.time(),
+        "total": len(hostnames),
+        "done": 0,
+        "servers": {},
+        "results": [],
+        "finished": False,
+    }
+    for hn in hostnames:
+        job["servers"][hn] = {"hostname": hn, "stage": "pending", "detail": "En attente", "status": None}
+    _audit_jobs[job_id] = job
+
+    def _run():
+        threads = []
+        for hn in hostnames:
+            t = threading.Thread(target=_audit_one, args=(job, hn.strip()), daemon=True)
+            threads.append(t)
+            t.start()
+        for t in threads:
+            t.join()
+        job["finished"] = True
+        job["finished_at"] = _time.time()
+
+    threading.Thread(target=_run, daemon=True).start()
+    return job_id
+
+
+def _audit_one(job, hostname):
+    job["servers"][hostname]["stage"] = "resolving"
+    job["servers"][hostname]["detail"] = "Résolution DNS"
+
+    target = _resolve(hostname)
+    if not target:
+        job["servers"][hostname]["stage"] = "failed"
+        job["servers"][hostname]["detail"] = "DNS: aucun suffixe résolu"
+        job["servers"][hostname]["status"] = "CONNECTION_FAILED"
+        result = {"hostname": hostname, "status": "CONNECTION_FAILED",
+                  "connection_method": f"DNS: aucun suffixe résolu ({hostname})", "resolved_fqdn": None}
+        job["results"].append(result)
+        job["done"] += 1
+        return
+
+    job["servers"][hostname]["stage"] = "connecting"
+    job["servers"][hostname]["detail"] = f"Connexion SSH → {target}"
+
+    client = _connect(target)
+    if not client:
+        job["servers"][hostname]["stage"] = "failed"
+        job["servers"][hostname]["detail"] = f"SSH refusé ({target})"
+        job["servers"][hostname]["status"] = "CONNECTION_FAILED"
+        result = {"hostname": hostname, "status": "CONNECTION_FAILED",
+                  "connection_method": f"SSH: connexion refusée ({target})", "resolved_fqdn": target}
+        job["results"].append(result)
+        job["done"] += 1
+        return
+
+    job["servers"][hostname]["stage"] = "auditing"
+    job["servers"][hostname]["detail"] = "Collecte des données"
+
+    result = {"hostname": hostname, "status": "OK", "resolved_fqdn": target,
+              "audit_date": datetime.now().strftime("%Y-%m-%d %H:%M")}
+    ssh_key, ssh_user = _get_ssh_settings()
+    result["connection_method"] = f"ssh_key ({ssh_user}@{target})"
+
+    for key, cmd in AUDIT_CMDS.items():
+        result[key] = _run(client, cmd)
+
+    try:
+        client.close()
+    except Exception:
+        pass
+
+    # Post-traitement
+    agents = result.get("agents", "")
+    result["qualys_active"] = "qualys" in agents and "active" in agents
+    result["sentinelone_active"] = "sentinelone" in agents and "active" in agents
+    result["disk_alert"] = False
+    for line in (result.get("disk_space") or "").split("\n"):
+        parts = line.split()
+        pcts = [p for p in parts if "%" in p]
+        if pcts:
+            try:
+                pct = int(pcts[0].replace("%", ""))
+                if pct >= 90:
+                    result["disk_alert"] = True
+            except ValueError:
+                pass
+
+    job["servers"][hostname]["stage"] = "success"
+    job["servers"][hostname]["detail"] = result.get("os_release", "OK")
+    job["servers"][hostname]["status"] = "OK"
+    job["results"].append(result)
+    job["done"] += 1
+
+
+def get_audit_job(job_id):
+    return _audit_jobs.get(job_id)
+
+
+def list_audit_jobs():
+    now = _time.time()
+    return {jid: j for jid, j in _audit_jobs.items() if now - j["started_at"] < 3600}
+
+
 def save_audit_to_db(db, results):
     """Sauvegarde/met a jour les resultats d'audit en base"""
     updated = 0
@@ -274,9 +392,10 @@ def save_audit_to_db(db, results):
                 except Exception:
                     pass
 
+            from .itop_service import _normalize_os_for_itop
             updates = {}
             if r.get("os_release"):
-                updates["os_version"] = r["os_release"].strip()
+                updates["os_version"] = _normalize_os_for_itop(r["os_release"].strip())
             if ip_addr:
                 updates["fqdn"] = resolved
 

app/templates/audit_realtime_progress.html

@@ -0,0 +1,126 @@
+{% extends 'base.html' %}
+{% block title %}Audit en cours{% endblock %}
+{% block content %}
+<div class="flex justify-between items-center mb-4">
+    <div>
+        <a href="/audit" class="text-xs text-gray-500 hover:text-gray-300">&larr; Retour audit</a>
+        <h2 class="text-xl font-bold text-cyber-accent" id="page-title">Audit en cours...</h2>
+    </div>
+    <div class="flex gap-2" id="actions-zone" style="display:none">
+        <form method="POST" action="/audit/realtime/save">
+            <button class="btn-primary px-4 py-2 text-sm" onclick="return confirm('Mettre à jour la base avec ces résultats ?')">Mettre à jour la base</button>
+        </form>
+    </div>
+</div>
+
+<!-- KPIs -->
+<div style="display:flex;gap:8px;margin-bottom:16px">
+    <div class="card p-3 text-center" style="flex:1">
+        <div class="text-2xl font-bold text-cyber-accent" id="kpi-total">{{ total }}</div>
+        <div class="text-xs text-gray-500">Total</div>
+    </div>
+    <div class="card p-3 text-center" style="flex:1">
+        <div class="text-2xl font-bold text-cyber-green" id="kpi-ok">0</div>
+        <div class="text-xs text-gray-500">Connectés</div>
+    </div>
+    <div class="card p-3 text-center" style="flex:1">
+        <div class="text-2xl font-bold text-cyber-red" id="kpi-fail">0</div>
+        <div class="text-xs text-gray-500">Échoués</div>
+    </div>
+    <div class="card p-3 text-center" style="flex:1">
+        <div class="text-2xl font-bold text-gray-400 font-mono" id="kpi-timer">0s</div>
+        <div class="text-xs text-gray-500">Temps</div>
+    </div>
+</div>
+
+<!-- Barre de progression -->
+<div class="card p-4 mb-4">
+    <div class="flex justify-between items-center mb-2">
+        <span class="text-xs text-gray-400" id="progress-text">0 / {{ total }}</span>
+        <span class="text-xs text-gray-500" id="progress-pct">0%</span>
+    </div>
+    <div style="background:#1e293b;border-radius:4px;height:10px;overflow:hidden">
+        <div id="progress-bar" style="height:100%;background:#00ffc8;width:0%;transition:width 0.5s ease"></div>
+    </div>
+</div>
+
+<!-- Tableau progression par serveur -->
+<div class="card overflow-hidden">
+    <table class="w-full table-cyber text-xs">
+        <thead><tr>
+            <th class="p-2 text-left">Hostname</th>
+            <th class="p-2">Étape</th>
+            <th class="p-2 text-left">Détail</th>
+        </tr></thead>
+        <tbody id="progress-body"></tbody>
+    </table>
+</div>
+
+<script>
+var JOB_ID = '{{ job_id }}';
+var TOTAL = {{ total }};
+var _pollTimer = null;
+
+var STAGE_MAP = {
+    'pending':    {label: 'En attente',     cls: 'badge-gray',   icon: '&#9679;'},
+    'resolving':  {label: 'Résolution DNS', cls: 'badge-yellow', icon: '&#8635;'},
+    'connecting': {label: 'Connexion SSH',  cls: 'badge-yellow', icon: '&#8635;'},
+    'auditing':   {label: 'Audit',          cls: 'badge-yellow', icon: '&#8635;'},
+    'success':    {label: 'OK',             cls: 'badge-green',  icon: '&#10003;'},
+    'failed':     {label: 'Échec',          cls: 'badge-red',    icon: '&#10007;'},
+};
+
+function stageBadge(stage) {
+    var s = STAGE_MAP[stage] || {label: stage, cls: 'badge-gray', icon: '?'};
+    var anim = (stage !== 'pending' && stage !== 'success' && stage !== 'failed')
+        ? ' style="animation:pulse 1.5s ease-in-out infinite"' : '';
+    return '<span class="badge ' + s.cls + '"' + anim + '>' + s.icon + ' ' + s.label + '</span>';
+}
+
+function poll() {
+    fetch('/audit/realtime/status/' + JOB_ID, {credentials: 'same-origin'})
+    .then(function(r){ return r.json(); })
+    .then(function(data){
+        if (!data.ok) return;
+
+        var pct = TOTAL > 0 ? Math.round((data.done / TOTAL) * 100) : 0;
+        document.getElementById('progress-bar').style.width = pct + '%';
+        document.getElementById('progress-text').textContent = data.done + ' / ' + TOTAL;
+        document.getElementById('progress-pct').textContent = pct + '%';
+        document.getElementById('kpi-timer').textContent = data.elapsed + 's';
+
+        var ok = 0, fail = 0;
+        var hostnames = Object.keys(data.servers).sort();
+        var rows = '';
+        hostnames.forEach(function(hn){
+            var s = data.servers[hn];
+            if (s.stage === 'success') ok++;
+            else if (s.stage === 'failed') fail++;
+            rows += '<tr class="border-t border-cyber-border/30">';
+            rows += '<td class="p-2 font-mono">' + hn + '</td>';
+            rows += '<td class="p-2 text-center">' + stageBadge(s.stage) + '</td>';
+            rows += '<td class="p-2 text-gray-400 text-xs">' + (s.detail || '') + '</td>';
+            rows += '</tr>';
+        });
+        document.getElementById('progress-body').innerHTML = rows;
+        document.getElementById('kpi-ok').textContent = ok;
+        document.getElementById('kpi-fail').textContent = fail;
+
+        if (data.finished) {
+            if (_pollTimer) { clearInterval(_pollTimer); _pollTimer = null; }
+            document.getElementById('page-title').innerHTML =
+                'Audit terminé — <span class="text-cyber-green">' + ok + ' OK</span> / <span class="text-cyber-red">' + fail + ' échec(s)</span>';
+            document.getElementById('progress-bar').style.background = fail > 0 ? '#ff3366' : '#00ffc8';
+            document.getElementById('actions-zone').style.display = 'flex';
+        }
+    })
+    .catch(function(){});
+}
+
+poll();
+_pollTimer = setInterval(poll, 2000);
+</script>
+<style>
+@keyframes pulse { 0%,100%{opacity:1} 50%{opacity:0.5} }
+</style>
+{% endblock %}