Audit: jobs background paralleles + progression live
- Audit global/realtime: threads paralleles, job_id retourne immediat
- /audit/realtime/progress/{job_id}: KPIs + barre progression + tableau live
- Polling AJAX toutes les 2s, etapes animees (DNS/SSH/Audit/OK)
- PRETTY_NAME correction: extraction via grep -E 'PRETTY_NAME' + cut
- OS version: normalisation lors de save_audit_to_db (Debian GNU/Linux -> Debian X (Bookworm))
- Mise a jour base: itop sync bidirectionnel avec push OS version
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
5ea4100f4c
commit
3f47fea8e6
@ -5,7 +5,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse, StreamingResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from sqlalchemy import text
|
||||
from ..dependencies import get_db, get_current_user, get_user_perms, can_view, can_edit, can_admin, base_context
|
||||
from ..services.realtime_audit_service import audit_servers_list, save_audit_to_db
|
||||
from ..services.realtime_audit_service import audit_servers_list, save_audit_to_db, start_audit_job, get_audit_job
|
||||
from ..config import APP_NAME
|
||||
|
||||
router = APIRouter()
|
||||
@ -134,7 +134,7 @@ async def audit_global(request: Request, db=Depends(get_db)):
|
||||
parallel = int(form.get("parallel", "5"))
|
||||
|
||||
# Construire la requete
|
||||
where = ["s.os_family = 'linux'", "s.etat = 'en_production'"]
|
||||
where = ["s.os_family = 'linux'", "s.etat = 'production'"]
|
||||
params = {}
|
||||
if exclude_domains:
|
||||
where.append("d.code NOT IN :ed")
|
||||
@ -173,18 +173,9 @@ async def audit_global(request: Request, db=Depends(get_db)):
|
||||
if not hostnames:
|
||||
return RedirectResponse(url="/audit?msg=no_hosts", status_code=303)
|
||||
|
||||
# Lancer l'audit
|
||||
results = audit_servers_list(hostnames)
|
||||
request.app.state.last_audit_results = results
|
||||
|
||||
ctx = base_context(request, db, user)
|
||||
ctx.update({
|
||||
"app_name": APP_NAME, "results": results,
|
||||
"total": len(results),
|
||||
"ok": sum(1 for r in results if r.get("status") == "OK"),
|
||||
"failed": sum(1 for r in results if r.get("status") != "OK"),
|
||||
})
|
||||
return templates.TemplateResponse("audit_realtime_results.html", ctx)
|
||||
# Lancer en arrière-plan
|
||||
job_id = start_audit_job(hostnames)
|
||||
return RedirectResponse(url=f"/audit/realtime/progress/{job_id}", status_code=303)
|
||||
|
||||
|
||||
@router.post("/audit/realtime", response_class=HTMLResponse)
|
||||
@ -209,23 +200,57 @@ async def audit_realtime(request: Request, db=Depends(get_db),
|
||||
if not hostnames:
|
||||
return RedirectResponse(url="/audit?msg=no_hosts", status_code=303)
|
||||
|
||||
# Lancer l'audit
|
||||
results = audit_servers_list(hostnames)
|
||||
# Lancer en arrière-plan
|
||||
job_id = start_audit_job(hostnames)
|
||||
|
||||
# Stocker en session (request.state) pour export/save
|
||||
request.app.state.last_audit_results = results
|
||||
return RedirectResponse(url=f"/audit/realtime/progress/{job_id}", status_code=303)
|
||||
|
||||
|
||||
@router.get("/audit/realtime/progress/{job_id}", response_class=HTMLResponse)
|
||||
async def audit_realtime_progress(request: Request, job_id: str, db=Depends(get_db)):
|
||||
user = get_current_user(request)
|
||||
if not user:
|
||||
return RedirectResponse(url="/login")
|
||||
perms = get_user_perms(db, user)
|
||||
if not can_view(perms, "audit"):
|
||||
return RedirectResponse(url="/audit")
|
||||
|
||||
job = get_audit_job(job_id)
|
||||
if not job:
|
||||
return RedirectResponse(url="/audit?msg=job_not_found", status_code=303)
|
||||
|
||||
ctx = base_context(request, db, user)
|
||||
ctx.update({
|
||||
"app_name": APP_NAME, "results": results,
|
||||
"total": len(results),
|
||||
"ok": sum(1 for r in results if r.get("status") == "OK"),
|
||||
"failed": sum(1 for r in results if r.get("status") != "OK"),
|
||||
ctx.update({"app_name": APP_NAME, "job_id": job_id, "total": job["total"]})
|
||||
return templates.TemplateResponse("audit_realtime_progress.html", ctx)
|
||||
|
||||
|
||||
@router.get("/audit/realtime/status/{job_id}")
|
||||
async def audit_realtime_status(request: Request, job_id: str, db=Depends(get_db)):
|
||||
from fastapi.responses import JSONResponse
|
||||
user = get_current_user(request)
|
||||
if not user:
|
||||
return JSONResponse({"ok": False}, status_code=401)
|
||||
|
||||
job = get_audit_job(job_id)
|
||||
if not job:
|
||||
return JSONResponse({"ok": False, "msg": "Job introuvable"}, status_code=404)
|
||||
|
||||
import time
|
||||
elapsed = int(time.time() - job["started_at"])
|
||||
|
||||
# When finished, store results in app state for save/export
|
||||
if job["finished"]:
|
||||
request.app.state.last_audit_results = job["results"]
|
||||
|
||||
return JSONResponse({
|
||||
"ok": True,
|
||||
"job_id": job_id,
|
||||
"total": job["total"],
|
||||
"done": job["done"],
|
||||
"finished": job["finished"],
|
||||
"elapsed": elapsed,
|
||||
"servers": job["servers"],
|
||||
})
|
||||
response = templates.TemplateResponse("audit_realtime_results.html", ctx)
|
||||
response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0"
|
||||
response.headers["Pragma"] = "no-cache"
|
||||
return response
|
||||
|
||||
|
||||
@router.post("/audit/realtime/save")
|
||||
|
||||
@ -32,7 +32,7 @@ def _get_ssh_settings():
|
||||
|
||||
# Commandes d'audit (simplifiees pour le temps reel)
|
||||
AUDIT_CMDS = {
|
||||
"os_release": "cat /etc/redhat-release 2>/dev/null || head -1 /etc/os-release 2>/dev/null",
|
||||
"os_release": "cat /etc/redhat-release 2>/dev/null || grep '^PRETTY_NAME=' /etc/os-release 2>/dev/null | cut -d'\"' -f2",
|
||||
"kernel": "uname -r",
|
||||
"uptime": "uptime -p 2>/dev/null || uptime",
|
||||
"selinux": "getenforce 2>/dev/null || echo N/A",
|
||||
@ -193,6 +193,124 @@ def audit_servers_list(hostnames):
|
||||
return results
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# Background audit job manager
|
||||
# ═══════════════════════════════════════════════
|
||||
import threading
|
||||
import uuid
|
||||
import time as _time
|
||||
|
||||
_audit_jobs = {}
|
||||
|
||||
|
||||
def start_audit_job(hostnames):
|
||||
"""Lance un audit en arriere-plan. Retourne le job_id."""
|
||||
job_id = str(uuid.uuid4())[:8]
|
||||
job = {
|
||||
"id": job_id,
|
||||
"started_at": _time.time(),
|
||||
"total": len(hostnames),
|
||||
"done": 0,
|
||||
"servers": {},
|
||||
"results": [],
|
||||
"finished": False,
|
||||
}
|
||||
for hn in hostnames:
|
||||
job["servers"][hn] = {"hostname": hn, "stage": "pending", "detail": "En attente", "status": None}
|
||||
_audit_jobs[job_id] = job
|
||||
|
||||
def _run():
|
||||
threads = []
|
||||
for hn in hostnames:
|
||||
t = threading.Thread(target=_audit_one, args=(job, hn.strip()), daemon=True)
|
||||
threads.append(t)
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
job["finished"] = True
|
||||
job["finished_at"] = _time.time()
|
||||
|
||||
threading.Thread(target=_run, daemon=True).start()
|
||||
return job_id
|
||||
|
||||
|
||||
def _audit_one(job, hostname):
|
||||
job["servers"][hostname]["stage"] = "resolving"
|
||||
job["servers"][hostname]["detail"] = "Résolution DNS"
|
||||
|
||||
target = _resolve(hostname)
|
||||
if not target:
|
||||
job["servers"][hostname]["stage"] = "failed"
|
||||
job["servers"][hostname]["detail"] = "DNS: aucun suffixe résolu"
|
||||
job["servers"][hostname]["status"] = "CONNECTION_FAILED"
|
||||
result = {"hostname": hostname, "status": "CONNECTION_FAILED",
|
||||
"connection_method": f"DNS: aucun suffixe résolu ({hostname})", "resolved_fqdn": None}
|
||||
job["results"].append(result)
|
||||
job["done"] += 1
|
||||
return
|
||||
|
||||
job["servers"][hostname]["stage"] = "connecting"
|
||||
job["servers"][hostname]["detail"] = f"Connexion SSH → {target}"
|
||||
|
||||
client = _connect(target)
|
||||
if not client:
|
||||
job["servers"][hostname]["stage"] = "failed"
|
||||
job["servers"][hostname]["detail"] = f"SSH refusé ({target})"
|
||||
job["servers"][hostname]["status"] = "CONNECTION_FAILED"
|
||||
result = {"hostname": hostname, "status": "CONNECTION_FAILED",
|
||||
"connection_method": f"SSH: connexion refusée ({target})", "resolved_fqdn": target}
|
||||
job["results"].append(result)
|
||||
job["done"] += 1
|
||||
return
|
||||
|
||||
job["servers"][hostname]["stage"] = "auditing"
|
||||
job["servers"][hostname]["detail"] = "Collecte des données"
|
||||
|
||||
result = {"hostname": hostname, "status": "OK", "resolved_fqdn": target,
|
||||
"audit_date": datetime.now().strftime("%Y-%m-%d %H:%M")}
|
||||
ssh_key, ssh_user = _get_ssh_settings()
|
||||
result["connection_method"] = f"ssh_key ({ssh_user}@{target})"
|
||||
|
||||
for key, cmd in AUDIT_CMDS.items():
|
||||
result[key] = _run(client, cmd)
|
||||
|
||||
try:
|
||||
client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Post-traitement
|
||||
agents = result.get("agents", "")
|
||||
result["qualys_active"] = "qualys" in agents and "active" in agents
|
||||
result["sentinelone_active"] = "sentinelone" in agents and "active" in agents
|
||||
result["disk_alert"] = False
|
||||
for line in (result.get("disk_space") or "").split("\n"):
|
||||
parts = line.split()
|
||||
pcts = [p for p in parts if "%" in p]
|
||||
if pcts:
|
||||
try:
|
||||
pct = int(pcts[0].replace("%", ""))
|
||||
if pct >= 90:
|
||||
result["disk_alert"] = True
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
job["servers"][hostname]["stage"] = "success"
|
||||
job["servers"][hostname]["detail"] = result.get("os_release", "OK")
|
||||
job["servers"][hostname]["status"] = "OK"
|
||||
job["results"].append(result)
|
||||
job["done"] += 1
|
||||
|
||||
|
||||
def get_audit_job(job_id):
|
||||
return _audit_jobs.get(job_id)
|
||||
|
||||
|
||||
def list_audit_jobs():
|
||||
now = _time.time()
|
||||
return {jid: j for jid, j in _audit_jobs.items() if now - j["started_at"] < 3600}
|
||||
|
||||
|
||||
def save_audit_to_db(db, results):
|
||||
"""Sauvegarde/met a jour les resultats d'audit en base"""
|
||||
updated = 0
|
||||
@ -274,9 +392,10 @@ def save_audit_to_db(db, results):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from .itop_service import _normalize_os_for_itop
|
||||
updates = {}
|
||||
if r.get("os_release"):
|
||||
updates["os_version"] = r["os_release"].strip()
|
||||
updates["os_version"] = _normalize_os_for_itop(r["os_release"].strip())
|
||||
if ip_addr:
|
||||
updates["fqdn"] = resolved
|
||||
|
||||
|
||||
126
app/templates/audit_realtime_progress.html
Normal file
126
app/templates/audit_realtime_progress.html
Normal file
@ -0,0 +1,126 @@
|
||||
{% extends 'base.html' %}
|
||||
{% block title %}Audit en cours{% endblock %}
|
||||
{% block content %}
|
||||
<div class="flex justify-between items-center mb-4">
|
||||
<div>
|
||||
<a href="/audit" class="text-xs text-gray-500 hover:text-gray-300">← Retour audit</a>
|
||||
<h2 class="text-xl font-bold text-cyber-accent" id="page-title">Audit en cours...</h2>
|
||||
</div>
|
||||
<div class="flex gap-2" id="actions-zone" style="display:none">
|
||||
<form method="POST" action="/audit/realtime/save">
|
||||
<button class="btn-primary px-4 py-2 text-sm" onclick="return confirm('Mettre à jour la base avec ces résultats ?')">Mettre à jour la base</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- KPIs -->
|
||||
<div style="display:flex;gap:8px;margin-bottom:16px">
|
||||
<div class="card p-3 text-center" style="flex:1">
|
||||
<div class="text-2xl font-bold text-cyber-accent" id="kpi-total">{{ total }}</div>
|
||||
<div class="text-xs text-gray-500">Total</div>
|
||||
</div>
|
||||
<div class="card p-3 text-center" style="flex:1">
|
||||
<div class="text-2xl font-bold text-cyber-green" id="kpi-ok">0</div>
|
||||
<div class="text-xs text-gray-500">Connectés</div>
|
||||
</div>
|
||||
<div class="card p-3 text-center" style="flex:1">
|
||||
<div class="text-2xl font-bold text-cyber-red" id="kpi-fail">0</div>
|
||||
<div class="text-xs text-gray-500">Échoués</div>
|
||||
</div>
|
||||
<div class="card p-3 text-center" style="flex:1">
|
||||
<div class="text-2xl font-bold text-gray-400 font-mono" id="kpi-timer">0s</div>
|
||||
<div class="text-xs text-gray-500">Temps</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Barre de progression -->
|
||||
<div class="card p-4 mb-4">
|
||||
<div class="flex justify-between items-center mb-2">
|
||||
<span class="text-xs text-gray-400" id="progress-text">0 / {{ total }}</span>
|
||||
<span class="text-xs text-gray-500" id="progress-pct">0%</span>
|
||||
</div>
|
||||
<div style="background:#1e293b;border-radius:4px;height:10px;overflow:hidden">
|
||||
<div id="progress-bar" style="height:100%;background:#00ffc8;width:0%;transition:width 0.5s ease"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Tableau progression par serveur -->
|
||||
<div class="card overflow-hidden">
|
||||
<table class="w-full table-cyber text-xs">
|
||||
<thead><tr>
|
||||
<th class="p-2 text-left">Hostname</th>
|
||||
<th class="p-2">Étape</th>
|
||||
<th class="p-2 text-left">Détail</th>
|
||||
</tr></thead>
|
||||
<tbody id="progress-body"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var JOB_ID = '{{ job_id }}';
|
||||
var TOTAL = {{ total }};
|
||||
var _pollTimer = null;
|
||||
|
||||
var STAGE_MAP = {
|
||||
'pending': {label: 'En attente', cls: 'badge-gray', icon: '●'},
|
||||
'resolving': {label: 'Résolution DNS', cls: 'badge-yellow', icon: '↻'},
|
||||
'connecting': {label: 'Connexion SSH', cls: 'badge-yellow', icon: '↻'},
|
||||
'auditing': {label: 'Audit', cls: 'badge-yellow', icon: '↻'},
|
||||
'success': {label: 'OK', cls: 'badge-green', icon: '✓'},
|
||||
'failed': {label: 'Échec', cls: 'badge-red', icon: '✗'},
|
||||
};
|
||||
|
||||
function stageBadge(stage) {
|
||||
var s = STAGE_MAP[stage] || {label: stage, cls: 'badge-gray', icon: '?'};
|
||||
var anim = (stage !== 'pending' && stage !== 'success' && stage !== 'failed')
|
||||
? ' style="animation:pulse 1.5s ease-in-out infinite"' : '';
|
||||
return '<span class="badge ' + s.cls + '"' + anim + '>' + s.icon + ' ' + s.label + '</span>';
|
||||
}
|
||||
|
||||
function poll() {
|
||||
fetch('/audit/realtime/status/' + JOB_ID, {credentials: 'same-origin'})
|
||||
.then(function(r){ return r.json(); })
|
||||
.then(function(data){
|
||||
if (!data.ok) return;
|
||||
|
||||
var pct = TOTAL > 0 ? Math.round((data.done / TOTAL) * 100) : 0;
|
||||
document.getElementById('progress-bar').style.width = pct + '%';
|
||||
document.getElementById('progress-text').textContent = data.done + ' / ' + TOTAL;
|
||||
document.getElementById('progress-pct').textContent = pct + '%';
|
||||
document.getElementById('kpi-timer').textContent = data.elapsed + 's';
|
||||
|
||||
var ok = 0, fail = 0;
|
||||
var hostnames = Object.keys(data.servers).sort();
|
||||
var rows = '';
|
||||
hostnames.forEach(function(hn){
|
||||
var s = data.servers[hn];
|
||||
if (s.stage === 'success') ok++;
|
||||
else if (s.stage === 'failed') fail++;
|
||||
rows += '<tr class="border-t border-cyber-border/30">';
|
||||
rows += '<td class="p-2 font-mono">' + hn + '</td>';
|
||||
rows += '<td class="p-2 text-center">' + stageBadge(s.stage) + '</td>';
|
||||
rows += '<td class="p-2 text-gray-400 text-xs">' + (s.detail || '') + '</td>';
|
||||
rows += '</tr>';
|
||||
});
|
||||
document.getElementById('progress-body').innerHTML = rows;
|
||||
document.getElementById('kpi-ok').textContent = ok;
|
||||
document.getElementById('kpi-fail').textContent = fail;
|
||||
|
||||
if (data.finished) {
|
||||
if (_pollTimer) { clearInterval(_pollTimer); _pollTimer = null; }
|
||||
document.getElementById('page-title').innerHTML =
|
||||
'Audit terminé — <span class="text-cyber-green">' + ok + ' OK</span> / <span class="text-cyber-red">' + fail + ' échec(s)</span>';
|
||||
document.getElementById('progress-bar').style.background = fail > 0 ? '#ff3366' : '#00ffc8';
|
||||
document.getElementById('actions-zone').style.display = 'flex';
|
||||
}
|
||||
})
|
||||
.catch(function(){});
|
||||
}
|
||||
|
||||
poll();
|
||||
_pollTimer = setInterval(poll, 2000);
|
||||
</script>
|
||||
<style>
|
||||
@keyframes pulse { 0%,100%{opacity:1} 50%{opacity:0.5} }
|
||||
</style>
|
||||
{% endblock %}
|
||||
Loading…
Reference in New Issue
Block a user