Merge pull request #600 from CyberMind-FR/feature/599-threat-analyst-ingest-real-crowdsec-aler

feat(threat-analyst): ingest real CrowdSec alerts (read-only sudo) + auto-collect + dedup

packages/secubox-threat-analyst/api/main.py

@@ -173,12 +173,18 @@ class ThreatAnalyzer:
             f.write(json.dumps(alert.model_dump()) + "\n")
 
     def get_recent_alerts(self, hours: int = 24, source: Optional[str] = None) -> List[ThreatAlert]:
-        """Get recent alerts."""
+        """Get recent alerts, deduplicated by id (last occurrence wins).
+
+        The collector appends on every poll, so the same CrowdSec alert id can
+        recur many times — without dedup the headline counts and Top-N
+        leaderboards are massively inflated.
+        """
         cutoff = datetime.utcnow() - timedelta(hours=hours)
-        alerts = []
+        by_id: Dict[str, ThreatAlert] = {}
+        anon = 0
 
         if not self.alerts_file.exists():
-            return alerts
+            return []
 
         with open(self.alerts_file) as f:
             for line in f:
@@ -189,35 +195,82 @@ class ThreatAnalyzer:
                         continue
                     if source and data.get("source") != source:
                         continue
-                    alerts.append(ThreatAlert(**data))
+                    aid = data.get("id")
+                    if not aid:
+                        aid = f"_anon-{anon}"; anon += 1
+                    by_id[aid] = ThreatAlert(**data)
                 except Exception:
                     continue
 
-        return alerts
+        return list(by_id.values())
+
+    def compact_alerts(self, hours: int = 48):
+        """Rewrite alerts.jsonl keeping only the last `hours`, deduped by id —
+        keeps the append-only log from growing unbounded."""
+        if not self.alerts_file.exists():
+            return
+        cutoff = datetime.utcnow() - timedelta(hours=hours)
+        recent: Dict[str, Dict[str, Any]] = {}
+        anon = 0
+        try:
+            with open(self.alerts_file) as f:
+                for line in f:
+                    try:
+                        data = json.loads(line)
+                        ts = datetime.fromisoformat(data["timestamp"].rstrip("Z"))
+                        if ts < cutoff:
+                            continue
+                        aid = data.get("id") or f"_anon-{anon}"
+                        if not data.get("id"):
+                            anon += 1
+                        recent[aid] = data
+                    except Exception:
+                        continue
+            tmp = self.alerts_file.with_suffix(".jsonl.tmp")
+            with open(tmp, "w") as f:
+                for data in recent.values():
+                    f.write(json.dumps(data) + "\n")
+            tmp.replace(self.alerts_file)
+        except Exception as e:
+            logger.warning("compact_alerts failed: %s", e)
 
     async def collect_crowdsec_alerts(self) -> List[ThreatAlert]:
         """Collect alerts from CrowdSec."""
         alerts = []
         try:
+            # The daemon runs as the unprivileged `secubox` user; `cscli` needs
+            # root (reads /etc/crowdsec/local_api_credentials.yaml). We go through
+            # the read-only sudo grant shipped in /etc/sudoers.d/secubox-threat-
+            # analyst (sudo lives here on the BACKEND only — the frontend just
+            # consumes the resulting values).
             result = subprocess.run(
-                ["cscli", "alerts", "list", "-o", "json"],
+                ["sudo", "-n", "/usr/bin/cscli",
+                 "alerts", "list", "-o", "json", "-l", "200"],
                 capture_output=True,
                 text=True,
-                timeout=10
+                timeout=15
             )
-            if result.returncode == 0:
-                data = json.loads(result.stdout)
-                for item in data[:50]:  # Limit to 50
+            if result.returncode == 0 and result.stdout.strip():
+                data = json.loads(result.stdout) or []
+                for item in data[:200]:
+                    src = item.get("source") or {}
+                    # `remediation` is a bool, not a severity — map it.
+                    severity = "high" if item.get("remediation") else "medium"
                     alert = ThreatAlert(
                         id=f"cs-{item.get('id', '')}",
                         source="crowdsec",
-                        severity=item.get("remediation", "medium"),
+                        severity=severity,
                         type=item.get("scenario", "unknown"),
-                        ip=item.get("source", {}).get("ip"),
+                        ip=src.get("ip") or src.get("value"),
                         details=item,
                         timestamp=item.get("created_at", datetime.utcnow().isoformat() + "Z")
                     )
                     alerts.append(alert)
+            else:
+                logger.warning(
+                    "cscli alerts list failed (rc=%s): %s",
+                    result.returncode, (result.stderr or "").strip()[:200]
+                )
         except Exception as e:
             logger.warning(f"CrowdSec collection failed: {e}")
 
@@ -844,9 +897,31 @@ async def get_overview():
     return await _build_overview()
 
 
+_COLLECT_TTL = 300  # 5 min
+
+
+async def _collect_refresh_loop():
+    """Backend auto-collect: keep the alerts DB fed from CrowdSec + WAF even
+    when no operator has the page open. Compacts the log after each run so it
+    stays bounded (and deduped). subprocess work is brief and best-effort."""
+    while True:
+        try:
+            cs = await analyzer.collect_crowdsec_alerts()
+            waf = await analyzer.collect_waf_alerts()
+            for a in cs + waf:
+                analyzer.record_alert(a)
+            analyzer.compact_alerts()
+            if cs or waf:
+                logger.info("auto-collect: %d crowdsec + %d waf alerts", len(cs), len(waf))
+        except Exception as e:
+            logger.warning("auto-collect failed: %s", e)
+        await asyncio.sleep(_COLLECT_TTL)
+
+
 @app.on_event("startup")
 async def startup():
     """Initialize on startup."""
     DATA_DIR.mkdir(parents=True, exist_ok=True)
     asyncio.create_task(_overview_refresh_loop())
+    asyncio.create_task(_collect_refresh_loop())
     logger.info("Threat Analyst started")

packages/secubox-threat-analyst/debian/changelog

@@ -1,3 +1,21 @@
+secubox-threat-analyst (1.4.4-1~bookworm1) bookworm; urgency=medium
+
+  * feat(ingest): populate the headline stats + Top-N leaderboards with real
+    CrowdSec data (#599). The collector shelled out to bare `cscli`, which
+    fails for the unprivileged `secubox` user the daemon runs as → the alerts
+    DB stayed empty (Unique attackers / Active threats / Countries / Top-N all
+    0). Now goes through a read-only sudo grant (`/etc/sudoers.d/
+    secubox-threat-analyst`: `cscli alerts list *` + `decisions list *`,
+    visudo-validated in postinst) and fetches `-l 200`. sudo is BACKEND-only;
+    the frontend stays value-only.
+  * feat(collect): backend auto-collect loop (~5 min) so the DB fills without
+    the page open; severity mapped correctly (`remediation` is a bool).
+  * fix(dedup): `get_recent_alerts` now dedups by alert id (last wins) and a
+    bounded 48 h compaction rewrites `alerts.jsonl` — the append-on-every-poll
+    log was massively inflating counts and leaderboards.
+
+ -- Gerald KERMA <devel@cybermind.fr>  Mon, 15 Jun 2026 12:00:00 +0200
+
 secubox-threat-analyst (1.4.3-1~bookworm1) bookworm; urgency=medium
 
   * feat(overview): global security overview — all metrics now dynamic,

packages/secubox-threat-analyst/debian/postinst

@@ -1,6 +1,19 @@
 #!/bin/sh
 set -e
 if [ "$1" = "configure" ]; then
+    # #599 — validate the read-only cscli sudo grant; drop it if malformed so a
+    # bad drop-in can never break sudo for the whole system.
+    SUDOERS=/etc/sudoers.d/secubox-threat-analyst
+    if [ -f "$SUDOERS" ]; then
+        chmod 0440 "$SUDOERS" || true
+        if command -v visudo >/dev/null 2>&1; then
+            if ! visudo -cf "$SUDOERS" >/dev/null 2>&1; then
+                echo "secubox-threat-analyst: invalid sudoers drop-in, removing" >&2
+                rm -f "$SUDOERS"
+            fi
+        fi
+    fi
+
     # #595 — build-safe enable: a plain `systemctl enable` no-ops during an
     # offline / image-build (chroot) install, leaving the unit DISABLED on
     # the live board (caught on gk2: /threat-analyst/ backend down). Use

packages/secubox-threat-analyst/debian/rules

@@ -9,3 +9,7 @@ override_dh_auto_install:
 	cp -r www/threat-analyst/. debian/secubox-threat-analyst/usr/share/secubox/www/threat-analyst/
 	install -d debian/secubox-threat-analyst/usr/share/secubox/menu.d
 	[ -d menu.d ] && cp -r menu.d/. debian/secubox-threat-analyst/usr/share/secubox/menu.d/ || true
+	# Read-only cscli sudo grant (#599) — backend ingestion of CrowdSec alerts.
+	install -d debian/secubox-threat-analyst/etc/sudoers.d
+	install -m 0440 debian/sudoers.d/secubox-threat-analyst \
+		debian/secubox-threat-analyst/etc/sudoers.d/secubox-threat-analyst

packages/secubox-threat-analyst/debian/secubox-threat-analyst.service

@@ -13,7 +13,10 @@ Restart=on-failure
 RestartSec=5
 UMask=0000
 
-NoNewPrivileges=true
+# #599 — allow the read-only `sudo cscli` ingestion grant (sibling pattern:
+# secubox-crowdsec / secubox-waf also set this). The sudo scope is tightly
+# limited to read-only `cscli alerts/decisions list` in /etc/sudoers.d.
+NoNewPrivileges=no
 RuntimeDirectory=secubox
 RuntimeDirectoryPreserve=yes
 RuntimeDirectoryMode=0775

packages/secubox-threat-analyst/debian/sudoers.d/secubox-threat-analyst

@@ -0,0 +1,7 @@
+# SecuBox threat-analyst — read-only CrowdSec ingestion.
+# The FastAPI daemon runs as the unprivileged `secubox` user and needs to read
+# CrowdSec alerts/decisions to populate the analyst dashboard. `cscli` needs
+# root (it reads /etc/crowdsec/local_api_credentials.yaml). Only READ-ONLY
+# subcommands are granted here — no add/delete/restart. CSPN least-privilege.
+secubox ALL=(root) NOPASSWD: /usr/bin/cscli alerts list *
+secubox ALL=(root) NOPASSWD: /usr/bin/cscli decisions list *