Cache mémoire 10min pour Qualys API, bouton Resync temps réel, page Agents (activation keys + versions)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Khalid MOUTAOUAKIL
parent
0dc0fc7643
commit
c139dfbaa2
@ -9,6 +9,7 @@ from ..services.qualys_service import (
|
||||
sync_server_qualys, search_assets_api, get_all_tags_api,
|
||||
create_tag_api, delete_tag_api, add_tag_to_asset_api,
|
||||
remove_tag_from_asset_api, resync_all_tags, get_vuln_counts,
|
||||
get_activation_keys, get_agents_summary, invalidate_search_cache, get_cache_stats,
|
||||
)
|
||||
from ..config import APP_NAME
|
||||
|
||||
@ -328,12 +329,16 @@ async def qualys_search(request: Request, db=Depends(get_db),
|
||||
assets = []
|
||||
api_msg = None
|
||||
source = None
|
||||
force = request.query_params.get("force", "") == "1"
|
||||
cache_info = get_cache_stats()
|
||||
|
||||
if search:
|
||||
# Vérifier si on a des données récentes (< 24h) en base
|
||||
from datetime import datetime, timedelta
|
||||
cutoff = datetime.now() - timedelta(hours=24)
|
||||
|
||||
if field == "hostname":
|
||||
if force:
|
||||
fresh = 0 # Forcer le resync API
|
||||
elif field == "hostname":
|
||||
fresh = db.execute(text("""
|
||||
SELECT COUNT(*) FROM qualys_assets
|
||||
WHERE (hostname ILIKE :q OR name ILIKE :q OR fqdn ILIKE :q)
|
||||
@ -360,13 +365,13 @@ async def qualys_search(request: Request, db=Depends(get_db),
|
||||
# Données fraiches en base — pas d'appel API
|
||||
source = "base (< 24h)"
|
||||
else:
|
||||
# Appel API + stockage en base
|
||||
# Appel API (cache 10min) + stockage en base
|
||||
if field == "hostname":
|
||||
result = search_assets_api(db, search, field="name", operator="CONTAINS")
|
||||
result = search_assets_api(db, search, field="name", operator="CONTAINS", force_refresh=force)
|
||||
elif field == "ip":
|
||||
result = search_assets_api(db, search, field="address", operator="CONTAINS")
|
||||
result = search_assets_api(db, search, field="address", operator="CONTAINS", force_refresh=force)
|
||||
elif field == "tag":
|
||||
result = search_assets_api(db, search, field="tagName", operator="EQUALS")
|
||||
result = search_assets_api(db, search, field="tagName", operator="EQUALS", force_refresh=force)
|
||||
else:
|
||||
result = {"ok": False, "msg": "Champ inconnu", "assets": []}
|
||||
|
||||
@ -415,7 +420,7 @@ async def qualys_search(request: Request, db=Depends(get_db),
|
||||
ips = [ip for ip in ips if ip and ip != "None"]
|
||||
if ips:
|
||||
try:
|
||||
vuln_map = get_vuln_counts(db, ",".join(ips[:50]))
|
||||
vuln_map = get_vuln_counts(db, ",".join(ips[:50]), force_refresh=force)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@ -426,12 +431,32 @@ async def qualys_search(request: Request, db=Depends(get_db),
|
||||
"app_name": APP_NAME, "assets": assets, "search": search,
|
||||
"field": field, "api_msg": api_msg,
|
||||
"all_tags": all_tags, "vuln_map": vuln_map,
|
||||
"cache_info": cache_info,
|
||||
"can_edit_qualys": can_edit(perms, "qualys"),
|
||||
"msg": request.query_params.get("msg"),
|
||||
})
|
||||
return templates.TemplateResponse("qualys_search.html", ctx)
|
||||
|
||||
|
||||
@router.get("/qualys/agents", response_class=HTMLResponse)
|
||||
async def qualys_agents_page(request: Request, db=Depends(get_db)):
|
||||
user = get_current_user(request)
|
||||
if not user:
|
||||
return RedirectResponse(url="/login")
|
||||
perms = get_user_perms(db, user)
|
||||
if not can_view(perms, "qualys"):
|
||||
return RedirectResponse(url="/dashboard")
|
||||
|
||||
keys = get_activation_keys(db)
|
||||
summary = get_agents_summary(db)
|
||||
|
||||
ctx = base_context(request, db, user)
|
||||
ctx.update({
|
||||
"app_name": APP_NAME, "keys": keys, "summary": summary,
|
||||
})
|
||||
return templates.TemplateResponse("qualys_agents.html", ctx)
|
||||
|
||||
|
||||
@router.get("/qualys/vulns/{ip}", response_class=HTMLResponse)
|
||||
async def qualys_vulns_detail(request: Request, ip: str, db=Depends(get_db)):
|
||||
"""Retourne le detail des vulns severity 3,4,5 pour une IP (fragment HTMX)"""
|
||||
|
||||
48
app/services/cache.py
Normal file
48
app/services/cache.py
Normal file
@ -0,0 +1,48 @@
|
||||
"""Cache memoire simple avec TTL — pas de dependance externe"""
|
||||
import time
|
||||
import threading
|
||||
|
||||
_store = {}
|
||||
_lock = threading.Lock()
|
||||
DEFAULT_TTL = 600 # 10 minutes
|
||||
|
||||
|
||||
def get(key):
|
||||
with _lock:
|
||||
entry = _store.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
if time.time() > entry["expires"]:
|
||||
del _store[key]
|
||||
return None
|
||||
return entry["value"]
|
||||
|
||||
|
||||
def set(key, value, ttl=DEFAULT_TTL):
|
||||
with _lock:
|
||||
_store[key] = {"value": value, "expires": time.time() + ttl}
|
||||
|
||||
|
||||
def delete(key):
|
||||
with _lock:
|
||||
_store.pop(key, None)
|
||||
|
||||
|
||||
def clear_prefix(prefix):
|
||||
with _lock:
|
||||
keys = [k for k in _store if k.startswith(prefix)]
|
||||
for k in keys:
|
||||
del _store[k]
|
||||
|
||||
|
||||
def clear_all():
|
||||
with _lock:
|
||||
_store.clear()
|
||||
|
||||
|
||||
def stats():
|
||||
with _lock:
|
||||
now = time.time()
|
||||
total = len(_store)
|
||||
active = sum(1 for v in _store.values() if now <= v["expires"])
|
||||
return {"total": total, "active": active}
|
||||
@ -1,12 +1,15 @@
|
||||
"""Service Qualys — sync tags pour un serveur via API"""
|
||||
"""Service Qualys — sync tags pour un serveur via API + cache memoire"""
|
||||
import re
|
||||
import requests
|
||||
import urllib3
|
||||
from sqlalchemy import text
|
||||
from .secrets_service import get_secret
|
||||
from . import cache as _cache
|
||||
|
||||
urllib3.disable_warnings()
|
||||
|
||||
CACHE_TTL = 600 # 10 minutes
|
||||
|
||||
|
||||
def _get_qualys_creds(db):
|
||||
"""Recupere les credentials Qualys depuis les secrets chiffres"""
|
||||
@ -24,8 +27,17 @@ def parse_xml(txt, tag):
|
||||
return re.findall(f"<{tag}>([^<]*)</{tag}>", txt)
|
||||
|
||||
|
||||
def search_assets_api(db, query, field="name", operator="CONTAINS"):
|
||||
"""Recherche des assets via l'API Qualys en temps réel"""
|
||||
def search_assets_api(db, query, field="name", operator="CONTAINS", force_refresh=False):
|
||||
"""Recherche des assets via l'API Qualys — cache 10 min"""
|
||||
cache_key = f"qualys:search:{field}:{query}"
|
||||
|
||||
if not force_refresh:
|
||||
cached = _cache.get(cache_key)
|
||||
if cached is not None:
|
||||
cached["msg"] = cached.get("msg", "") + " (cache)"
|
||||
cached["from_cache"] = True
|
||||
return cached
|
||||
|
||||
qualys_url, qualys_user, qualys_pass, qualys_proxy = _get_qualys_creds(db)
|
||||
if not qualys_user:
|
||||
return {"ok": False, "msg": "Credentials Qualys non configurés", "assets": []}
|
||||
@ -51,7 +63,9 @@ def search_assets_api(db, query, field="name", operator="CONTAINS"):
|
||||
return {"ok": False, "msg": f"API HTTP {r.status_code}", "assets": []}
|
||||
|
||||
assets = _parse_assets_full(r.text)
|
||||
return {"ok": True, "msg": f"{len(assets)} résultat(s)", "assets": assets}
|
||||
result = {"ok": True, "msg": f"{len(assets)} résultat(s)", "assets": assets, "from_cache": False}
|
||||
_cache.set(cache_key, result, CACHE_TTL)
|
||||
return result
|
||||
|
||||
|
||||
def get_all_tags_api(db):
|
||||
@ -371,11 +385,17 @@ def _find_asset_by_hostname(qualys_url, qualys_user, qualys_pass, hostname, prox
|
||||
return None
|
||||
|
||||
|
||||
def get_vuln_counts(db, ip_list):
|
||||
def get_vuln_counts(db, ip_list, force_refresh=False):
|
||||
"""Recupere le nombre de vulnerabilites actives severity 3,4,5 pour une liste d'IPs.
|
||||
ip_list: str (IPs separees par virgules)
|
||||
Retourne dict {ip: {severity3, severity4, severity5, total, confirmed, potential}}
|
||||
"""
|
||||
cache_key = f"qualys:vulns:{ip_list}"
|
||||
if not force_refresh:
|
||||
cached = _cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
qualys_url, qualys_user, qualys_pass, qualys_proxy = _get_qualys_creds(db)
|
||||
if not qualys_user or not ip_list:
|
||||
return {}
|
||||
@ -434,4 +454,77 @@ def get_vuln_counts(db, ip_list):
|
||||
|
||||
results[host_ip] = counts
|
||||
|
||||
_cache.set(cache_key, results, CACHE_TTL)
|
||||
return results
|
||||
|
||||
|
||||
def get_activation_keys(db):
|
||||
"""Recupere les activation keys Qualys"""
|
||||
cache_key = "qualys:actkeys"
|
||||
cached = _cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
qualys_url, qualys_user, qualys_pass, qualys_proxy = _get_qualys_creds(db)
|
||||
if not qualys_user:
|
||||
return []
|
||||
proxies = {"https": qualys_proxy, "http": qualys_proxy} if qualys_proxy else None
|
||||
|
||||
try:
|
||||
r = requests.post(
|
||||
f"{qualys_url}/qps/rest/1.0/search/ca/agentactkey",
|
||||
json={"ServiceRequest": {"preferences": {"limitResults": 20}}},
|
||||
auth=(qualys_user, qualys_pass),
|
||||
verify=False, timeout=30, proxies=proxies,
|
||||
headers={"X-Requested-With": "Python", "Content-Type": "application/json"})
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
if r.status_code != 200:
|
||||
return []
|
||||
|
||||
keys = []
|
||||
for block in r.text.split("<AgentActKey>")[1:]:
|
||||
block = block.split("</AgentActKey>")[0]
|
||||
keys.append({
|
||||
"id": (parse_xml(block, "id") or [""])[0],
|
||||
"key": (parse_xml(block, "activationKey") or [""])[0],
|
||||
"status": (parse_xml(block, "status") or [""])[0],
|
||||
"title": (parse_xml(block, "title") or [""])[0],
|
||||
"used": (parse_xml(block, "countUsed") or ["0"])[0],
|
||||
"type": (parse_xml(block, "type") or [""])[0],
|
||||
})
|
||||
|
||||
_cache.set(cache_key, keys, 3600) # 1h pour les keys
|
||||
return keys
|
||||
|
||||
|
||||
def get_agents_summary(db):
|
||||
"""Resume des agents installes depuis les assets en base"""
|
||||
rows = db.execute(text("""
|
||||
SELECT agent_status, COUNT(*) as cnt,
|
||||
COUNT(*) FILTER (WHERE agent_version IS NOT NULL AND agent_version != '') as with_version
|
||||
FROM qualys_assets
|
||||
WHERE agent_status IS NOT NULL AND agent_status != ''
|
||||
GROUP BY agent_status ORDER BY cnt DESC
|
||||
""")).fetchall()
|
||||
|
||||
versions = db.execute(text("""
|
||||
SELECT agent_version, COUNT(*) as cnt
|
||||
FROM qualys_assets
|
||||
WHERE agent_version IS NOT NULL AND agent_version != ''
|
||||
GROUP BY agent_version ORDER BY cnt DESC LIMIT 20
|
||||
""")).fetchall()
|
||||
|
||||
return {"statuses": rows, "versions": versions}
|
||||
|
||||
|
||||
def invalidate_search_cache():
|
||||
"""Invalide tout le cache de recherche Qualys"""
|
||||
_cache.clear_prefix("qualys:search:")
|
||||
_cache.clear_prefix("qualys:vulns:")
|
||||
|
||||
|
||||
def get_cache_stats():
|
||||
"""Stats du cache"""
|
||||
return _cache.stats()
|
||||
|
||||
@ -59,6 +59,7 @@
|
||||
{% if p.qualys %}<a href="/qualys/search" class="block px-3 py-2 rounded-md text-sm hover:bg-cyber-border/30 {% if '/qualys/' in request.url.path and 'tags' not in request.url.path and 'decoder' not in request.url.path %}bg-cyber-border/30 text-cyber-accent{% else %}text-gray-400{% endif %}">Qualys</a>{% endif %}
|
||||
{% if p.qualys %}<a href="/qualys/tags" class="block px-3 py-2 rounded-md text-sm hover:bg-cyber-border/30 {% if '/qualys/tags' in request.url.path %}bg-cyber-border/30 text-cyber-accent{% else %}text-gray-400{% endif %} pl-6 text-xs">Tags</a>{% endif %}
|
||||
{% if p.qualys %}<a href="/qualys/decoder" class="block px-3 py-2 rounded-md text-sm hover:bg-cyber-border/30 {% if 'decoder' in request.url.path %}bg-cyber-border/30 text-cyber-accent{% else %}text-gray-400{% endif %} pl-6 text-xs">Décodeur</a>{% endif %}
|
||||
{% if p.qualys %}<a href="/qualys/agents" class="block px-3 py-2 rounded-md text-sm hover:bg-cyber-border/30 {% if 'agents' in request.url.path %}bg-cyber-border/30 text-cyber-accent{% else %}text-gray-400{% endif %} pl-6 text-xs">Agents</a>{% endif %}
|
||||
{% if p.campaigns %}<a href="/safe-patching" class="block px-3 py-2 rounded-md text-sm hover:bg-cyber-border/30 {% if 'safe-patching' in request.url.path %}bg-cyber-border/30 text-cyber-accent{% else %}text-gray-400{% endif %}">Safe Patching</a>{% endif %}
|
||||
{% if p.planning %}<a href="/planning" class="block px-3 py-2 rounded-md text-sm hover:bg-cyber-border/30 {% if 'planning' in request.url.path %}bg-cyber-border/30 text-cyber-accent{% else %}text-gray-400{% endif %}">Planning</a>{% endif %}
|
||||
{% if p.audit %}<a href="/audit" class="block px-3 py-2 rounded-md text-sm hover:bg-cyber-border/30 {% if request.url.path == '/audit' %}bg-cyber-border/30 text-cyber-accent{% else %}text-gray-400{% endif %}">Audit</a>{% endif %}
|
||||
|
||||
77
app/templates/qualys_agents.html
Normal file
77
app/templates/qualys_agents.html
Normal file
@ -0,0 +1,77 @@
|
||||
{% extends 'base.html' %}
|
||||
{% block title %}Agents Qualys{% endblock %}
|
||||
{% block content %}
|
||||
<div class="flex justify-between items-center mb-4">
|
||||
<div>
|
||||
<h2 class="text-xl font-bold text-cyber-accent">Agents Qualys</h2>
|
||||
<p class="text-xs text-gray-500 mt-1">Activation keys et versions des agents déployés</p>
|
||||
</div>
|
||||
<a href="/qualys/search" class="btn-sm bg-cyber-border text-cyber-accent px-4 py-2">Recherche</a>
|
||||
</div>
|
||||
|
||||
<!-- Activation Keys -->
|
||||
<div class="card p-4 mb-4">
|
||||
<h3 class="text-sm font-bold text-cyber-accent mb-3">Activation Keys</h3>
|
||||
<table class="w-full table-cyber text-xs">
|
||||
<thead><tr>
|
||||
<th class="text-left p-2">Titre</th>
|
||||
<th class="p-2">Statut</th>
|
||||
<th class="p-2">Type</th>
|
||||
<th class="p-2">Utilisés</th>
|
||||
<th class="text-left p-2">Clé</th>
|
||||
</tr></thead>
|
||||
<tbody>
|
||||
{% for k in keys %}
|
||||
<tr>
|
||||
<td class="p-2 font-bold text-cyber-accent">{{ k.title }}</td>
|
||||
<td class="p-2 text-center"><span class="badge {% if k.status == 'ACTIVE' %}badge-green{% else %}badge-red{% endif %}">{{ k.status }}</span></td>
|
||||
<td class="p-2 text-center text-gray-400">{{ k.type }}</td>
|
||||
<td class="p-2 text-center font-bold">{{ k.used }}</td>
|
||||
<td class="p-2 font-mono text-gray-500" style="font-size:10px;">{{ k.key }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Statut agents -->
|
||||
<div class="grid grid-cols-2 gap-4 mb-4">
|
||||
<div class="card p-4">
|
||||
<h3 class="text-sm font-bold text-cyber-accent mb-3">Statut des agents</h3>
|
||||
{% if summary.statuses %}
|
||||
<table class="w-full table-cyber text-xs">
|
||||
<thead><tr><th class="text-left p-2">Statut</th><th class="p-2">Nombre</th></tr></thead>
|
||||
<tbody>
|
||||
{% for s in summary.statuses %}
|
||||
<tr>
|
||||
<td class="p-2"><span class="badge {% if 'ACTIVE' in (s.agent_status or '').upper() or 'STATUS_ACTIVE' in (s.agent_status or '').upper() %}badge-green{% elif 'INACTIVE' in (s.agent_status or '').upper() %}badge-red{% else %}badge-gray{% endif %}">{{ s.agent_status }}</span></td>
|
||||
<td class="p-2 text-center font-bold">{{ s.cnt }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
<p class="text-gray-500 text-xs">Aucune donnée</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="card p-4">
|
||||
<h3 class="text-sm font-bold text-cyber-accent mb-3">Versions déployées</h3>
|
||||
{% if summary.versions %}
|
||||
<table class="w-full table-cyber text-xs">
|
||||
<thead><tr><th class="text-left p-2">Version</th><th class="p-2">Nombre</th></tr></thead>
|
||||
<tbody>
|
||||
{% for v in summary.versions %}
|
||||
<tr>
|
||||
<td class="p-2 font-mono">{{ v.agent_version }}</td>
|
||||
<td class="p-2 text-center font-bold">{{ v.cnt }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
<p class="text-gray-500 text-xs">Aucune donnée</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@ -37,9 +37,13 @@
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if search %}
|
||||
<p class="text-xs text-gray-500 mb-2">
|
||||
<div class="flex justify-between items-center mb-2">
|
||||
<p class="text-xs text-gray-500">
|
||||
{% if api_msg %}{{ api_msg }}{% else %}{{ assets|length }} résultat(s){% endif %}
|
||||
</p>
|
||||
{% if cache_info %}<span class="text-gray-600 ml-2">(cache: {{ cache_info.active }} entrées)</span>{% endif %}
|
||||
</p>
|
||||
<a href="/qualys/search?field={{ field }}&search={{ search }}&force=1" class="btn-sm bg-cyber-border text-cyber-accent px-3 py-1 text-xs" data-loading="Resync API...">Resync temps réel</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Panel détail -->
|
||||
|
||||
Loading…
Reference in New Issue
Block a user