Merge pull request #706 from CyberMind-FR/feature/705-dpi-report-shows-zeros-exfil-is-a-60s-sn

dpi: cumulative 7d per-device rollup — report no longer shows zeros (closes #705)

packages/secubox-dpi/collector/main.go

@@ -45,6 +45,14 @@ var (
 	wgPeersPath = env("SECUBOX_WG_PEERS", "/var/lib/secubox/toolbox/wg-peers.json")
 	statePath   = env("SECUBOX_DPI_STATE", "/var/lib/secubox/dpi/state.json")
 	seenPath    = env("SECUBOX_DPI_SEEN", "/var/lib/secubox/dpi/seen.json")
+	// #705 — cumulative per-device rollup so the kbin report shows 7d history,
+	// not just the last 60s window (state.json). Same consumer schema as state.json.
+	cumulPath = env("SECUBOX_DPI_CUMUL", "/var/lib/secubox/dpi/cumulative.json")
+)
+
+const (
+	cumulRetention   = 7 * 24 * 3600 // drop devices/alerts not seen in 7d
+	cumulMaxServices = 80            // cap services per device to bound file size
 )
 
 // svc = (category, human service name) for a known SNI suffix. Categories are
@@ -328,9 +336,187 @@ func main() {
 	}
 	saveSeen(newseen)
 	writeState(aggs, alerts, now)
+	updateCumulative(aggs, alerts, now)
 	fmt.Printf("collector: %d flows-agg, %d alerts @ %d\n", len(aggs), len(alerts), now)
 }
 
+// cumDev is the persisted per-device cumulative rollup. Superset of the state
+// device schema (adds last_seen/first_seen) so the report's _dpi_stats reads it
+// unchanged.
+type cumDev struct {
+	Device    string         `json:"device"`
+	Flows     int            `json:"flows"`
+	UpBytes   int64          `json:"up_bytes"`
+	DownBytes int64          `json:"down_bytes"`
+	ByCat     map[string]int `json:"by_category"`
+	Services  []*agg         `json:"services"`
+	Clouds    []*agg         `json:"clouds"`
+	Alerts    []alert        `json:"alerts"`
+	FirstSeen int64          `json:"first_seen"`
+	LastSeen  int64          `json:"last_seen"`
+}
+
+// updateCumulative merges this window's aggs+alerts into the persistent 7d
+// cumulative store and rewrites it in state.json's consumer schema (devices[] +
+// top_apps/top_protocols/alerts). Idempotent-ish: best-effort, never fatal.
+func updateCumulative(aggs map[string]*agg, alerts []alert, now int64) {
+	devs := map[string]*cumDev{}
+	if b, err := os.ReadFile(cumulPath); err == nil {
+		var doc struct {
+			Devices []*cumDev `json:"devices"`
+		}
+		if json.Unmarshal(b, &doc) == nil {
+			for _, d := range doc.Devices {
+				if d.ByCat == nil {
+					d.ByCat = map[string]int{}
+				}
+				devs[d.Device] = d
+			}
+		}
+	}
+	// per-device service index (key = dst) for O(1) merge
+	svcIdx := map[string]map[string]*agg{}
+	for dev, d := range devs {
+		m := map[string]*agg{}
+		for _, s := range d.Services {
+			m[s.Dst] = s
+		}
+		svcIdx[dev] = m
+	}
+	for _, a := range aggs {
+		d := devs[a.Device]
+		if d == nil {
+			d = &cumDev{Device: a.Device, ByCat: map[string]int{}, FirstSeen: now}
+			devs[a.Device] = d
+			svcIdx[a.Device] = map[string]*agg{}
+		}
+		d.Flows += a.Flows
+		d.UpBytes += a.Up
+		d.DownBytes += a.Down
+		d.LastSeen = now
+		if a.Category != "" {
+			d.ByCat[a.Category] += a.Flows
+		}
+		si := svcIdx[a.Device]
+		if s := si[a.Dst]; s != nil {
+			s.Up += a.Up
+			s.Down += a.Down
+			s.Flows += a.Flows
+			if s.Category == "" {
+				s.Category = a.Category
+			}
+			if s.Service == "" {
+				s.Service = a.Service
+			}
+			if s.Cloud == "" {
+				s.Cloud = a.Cloud
+			}
+			if s.Proto == "" {
+				s.Proto = a.Proto
+			}
+		} else {
+			cp := *a
+			cp.iatAvg, cp.iatStd = 0, 0
+			si[a.Dst] = &cp
+		}
+	}
+	for _, al := range alerts {
+		if d := devs[al.Device]; d != nil {
+			d.Alerts = append(d.Alerts, al)
+		}
+	}
+
+	// finalise: prune stale, cap services, rebuild slices + global rollups
+	type rollup struct {
+		Name  string `json:"name"`
+		Bytes int64  `json:"bytes"`
+		Flows int    `json:"flows"`
+	}
+	apps := map[string]*rollup{}
+	protos := map[string]*rollup{}
+	var allAlerts []alert
+	list := make([]*cumDev, 0, len(devs))
+	for dev, d := range devs {
+		if now-d.LastSeen > cumulRetention {
+			continue // device idle >7d → drop
+		}
+		// rebuild services from the merged index, sort + cap
+		svcs := make([]*agg, 0, len(svcIdx[dev]))
+		for _, s := range svcIdx[dev] {
+			svcs = append(svcs, s)
+		}
+		sort.Slice(svcs, func(i, j int) bool { return svcs[i].Up > svcs[j].Up })
+		if len(svcs) > cumulMaxServices {
+			svcs = svcs[:cumulMaxServices]
+		}
+		d.Services = svcs
+		d.Clouds = d.Clouds[:0]
+		for _, s := range svcs {
+			if s.Cloud != "" {
+				d.Clouds = append(d.Clouds, s)
+			}
+			an := s.Service
+			if an == "" {
+				an = s.Dst
+			}
+			if an != "" {
+				if apps[an] == nil {
+					apps[an] = &rollup{Name: an}
+				}
+				apps[an].Bytes += s.Up + s.Down
+				apps[an].Flows += s.Flows
+			}
+			pn := s.Proto
+			if pn == "" {
+				pn = "unknown"
+			}
+			if protos[pn] == nil {
+				protos[pn] = &rollup{Name: pn}
+			}
+			protos[pn].Bytes += s.Up + s.Down
+			protos[pn].Flows += s.Flows
+		}
+		// prune old alerts, cap
+		fresh := d.Alerts[:0]
+		for _, al := range d.Alerts {
+			if now-al.TS <= cumulRetention {
+				fresh = append(fresh, al)
+			}
+		}
+		if len(fresh) > 50 {
+			fresh = fresh[len(fresh)-50:]
+		}
+		d.Alerts = fresh
+		allAlerts = append(allAlerts, fresh...)
+		if len(d.Clouds) > topN {
+			d.Clouds = d.Clouds[:topN]
+		}
+		list = append(list, d)
+	}
+	sort.Slice(list, func(i, j int) bool { return list[i].UpBytes > list[j].UpBytes })
+	rank := func(m map[string]*rollup) []*rollup {
+		out := make([]*rollup, 0, len(m))
+		for _, r := range m {
+			out = append(out, r)
+		}
+		sort.Slice(out, func(i, j int) bool { return out[i].Bytes > out[j].Bytes })
+		if len(out) > topN {
+			out = out[:topN]
+		}
+		return out
+	}
+	out := map[string]any{
+		"generated_at":  now,
+		"window_days":   cumulRetention / 86400,
+		"devices":       list,
+		"alerts":        allAlerts,
+		"alert_count":   len(allAlerts),
+		"top_apps":      rank(apps),
+		"top_protocols": rank(protos),
+	}
+	writeJSON(cumulPath, out)
+}
+
 func indexCols(header []string) map[string]int {
 	m := map[string]int{}
 	for i, h := range header {

packages/secubox-toolbox/secubox_toolbox/api.py

@@ -2391,6 +2391,9 @@ def _build_report_charts(graph: dict) -> dict:
 # identity as the report's mac_hash). Fail-empty so the report renders before the
 # first capture window.
 _DPI_STATE_PATH = Path("/var/lib/secubox/dpi/state.json")
+# #705 — prefer the 7d cumulative rollup (so the report shows history, not just
+# the last 60s window); fall back to the live state.json.
+_DPI_CUMUL_PATH = Path("/var/lib/secubox/dpi/cumulative.json")
 _DPI_CAT_EMOJI = {
     "cloud": "☁️", "filehost": "📦", "messaging": "💬", "ai": "🤖",
     "media": "🎬", "game": "🎮", "social": "👥", "adult": "🔞",
@@ -2421,10 +2424,14 @@ def _dpi_stats(mac_hash: str | None) -> dict:
     the secubox-dpi collector state. Returns {me, all}, each with categories /
     protocols / alerts / destinations donuts (+ summary counters)."""
     import json
-    try:
-        st = json.loads(_DPI_STATE_PATH.read_text()) if _DPI_STATE_PATH.exists() else {}
-    except Exception:
-        st = {}
+    st = {}
+    for p in (_DPI_CUMUL_PATH, _DPI_STATE_PATH):  # #705 cumulative first, live fallback
+        try:
+            if p.exists():
+                st = json.loads(p.read_text())
+                break
+        except Exception:
+            continue
     devices = st.get("devices") or []
 
     def cats(bycat: dict) -> list:
@@ -2479,6 +2486,43 @@ def _dpi_stats(mac_hash: str | None) -> dict:
     return {"me": me_stats, "all": all_stats}
 
 
+def _build_pdf_donuts(mac_hash: str | None, data: dict) -> list:
+    """#703 — assemble the 4 device-stat donuts for the PDF (mitm/certs/ads/dpi)
+    from LIVE sources (DPI collector + ad-block SQLite); the events table is
+    frozen post-#662 so we never read it. Each donut's segments carry pct +
+    cumulative start/end (via _dpi_donut)."""
+    me = (data.get("dpi_exfil") or {}).get("me") or {}
+    # ads — per-device blocked ad hosts (Go adstats → SQLite)
+    try:
+        ads = store.ad_client_stats(mac_hash, hours=24, top=6)
+    except Exception:
+        ads = {"total": 0, "top_hosts": []}
+    ads_segs = _dpi_donut([{"label": h.get("host", "?"), "emoji": "🚫",
+                            "count": int(h.get("hits", 0) or 0)} for h in ads.get("top_hosts", [])])
+    # certs — TLS-trust split from the protocol mix (what we could decrypt)
+    tls = quic = other = 0
+    for p in (me.get("protocols") or []):
+        nm = (p.get("label") or "").lower()
+        c = int(p.get("count", 0) or 0)
+        if "quic" in nm or "http3" in nm:
+            quic += c
+        elif "tls" in nm or "ssl" in nm or "https" in nm:
+            tls += c
+        else:
+            other += c
+    certs_segs = _dpi_donut([
+        {"label": "Inspecté (TLS)", "count": tls},
+        {"label": "Opaque (QUIC)", "count": quic},
+        {"label": "Autre", "count": other},
+    ])
+    return [
+        {"title": "🛰️ DPI — catégories", "hole": "DPI", "segments": me.get("categories") or []},
+        {"title": "🔍 MITM — protocoles", "hole": "proto", "segments": me.get("protocols") or []},
+        {"title": "🔒 Certs — confiance TLS", "hole": "certs", "segments": certs_segs},
+        {"title": "🚫 Pubs bloquées", "hole": str(ads.get("total", 0)), "segments": ads_segs},
+    ]
+
+
 # NOTE: route order matters in FastAPI — specific routes (/report/me,
 # /report/me/html) MUST be declared BEFORE the catch-all /report/{token},
 # otherwise FastAPI matches /report/me with token="me" and returns 404.
@@ -2563,6 +2607,7 @@ async def report_me(request: Request) -> Response:
     session = _aggregate_session(mac_hash)
     data = reports.build_report_data(mac_hash, session)
     data["dpi_exfil"] = _dpi_stats(mac_hash)  # #701 — DPI parity with the HTML report
+    data["pdf_donuts"] = _build_pdf_donuts(mac_hash, data)  # #703 — visual donuts
     pdf_bytes = reports.render_pdf(data)
     fname = f"gondwana-toolbox-{mac_hash[:8]}.pdf"
     return Response(

packages/secubox-toolbox/secubox_toolbox/reports.py

@@ -218,6 +218,9 @@ def render_pdf(report: dict) -> bytes:
             _bullet(pdf, f"{a.get('emoji', '?')} {a.get('app', '?')} ({a.get('category', '?')}) - {a.get('count', 0)} connexions", font_size=8)
         pdf.ln(2)
 
+    # ── DPI device donut charts (mitm/certs/ads/dpi) — #703 ──
+    _pdf_donut_grid(pdf, report.get("pdf_donuts") or [])
+
     # ── DPI / EXFILTRATION (R3 per-device + overall) — #701 (parity with HTML) ──
     dexf = report.get("dpi_exfil") or {}
     dme = dexf.get("me") or {}
@@ -645,6 +648,76 @@ def _bullet(pdf, text: str, font_size: int = 9) -> None:
     pdf.multi_cell(_page_w(pdf), 5, "  - " + safe)
 
 
+# #703 — visual donut charts in the PDF (fpdf2 solid_arc sectors + white hole).
+# RGB mirror of the HTML report palette.
+_PDF_DONUT_PALETTE = [
+    (0, 221, 68), (158, 118, 255), (255, 136, 102),
+    (102, 187, 255), (255, 179, 71), (255, 68, 102),
+]
+
+
+def _pdf_donut(pdf, x: float, y: float, w: float, title: str, hole: str, segs: list) -> None:
+    """Draw one donut (pie sectors + white centre hole) + legend inside a cell of
+    width w at (x, y). segs carry cumulative start/end percents (from _dpi_donut)."""
+    fam = getattr(pdf, "_secubox_family", "Helvetica")
+    pdf.set_xy(x, y)
+    pdf.set_font(fam, "B", 9)
+    pdf.set_text_color(0, 90, 64)
+    pdf.cell(w, 5, _ascii_safe(title)[:30], ln=False)
+    cx, cy, r, rh = x + 15, y + 23, 12.5, 8.0
+    if segs:
+        for i, s in enumerate(segs):
+            pdf.set_fill_color(*_PDF_DONUT_PALETTE[i % len(_PDF_DONUT_PALETTE)])
+            a0 = 90.0 - float(s.get("start", 0)) * 3.6
+            a1 = 90.0 - float(s.get("end", 0)) * 3.6
+            try:
+                pdf.solid_arc(cx, cy, r, a0, a1, clockwise=True, style="F")
+            except Exception:
+                pass
+        # centre hole (page is white)
+        pdf.set_fill_color(255, 255, 255)
+        pdf.ellipse(cx - rh, cy - rh, 2 * rh, 2 * rh, style="F")
+        if hole:
+            pdf.set_xy(cx - rh, cy - 2)
+            pdf.set_font(fam, "", 6)
+            pdf.set_text_color(110, 110, 110)
+            pdf.cell(2 * rh, 4, _ascii_safe(hole)[:8], align="C")
+        # legend (right of the donut)
+        ly = y + 8
+        for i, s in enumerate(segs[:6]):
+            pdf.set_fill_color(*_PDF_DONUT_PALETTE[i % len(_PDF_DONUT_PALETTE)])
+            pdf.rect(x + 33, ly + 0.6, 2.4, 2.4, style="F")
+            pdf.set_xy(x + 37, ly)
+            pdf.set_font(fam, "", 7)
+            pdf.set_text_color(40, 40, 40)
+            pdf.cell(w - 38, 3.5,
+                     _ascii_safe(f"{s.get('label', '?')[:16]}  {s.get('pct', 0)}%"), ln=False)
+            ly += 4
+    else:
+        pdf.set_xy(x, y + 20)
+        pdf.set_font(fam, "", 8)
+        pdf.set_text_color(120, 120, 120)
+        pdf.cell(w, 5, "Pas de donnees", ln=False)
+    pdf.set_text_color(0)
+
+
+def _pdf_donut_grid(pdf, donuts: list) -> None:
+    """Render up to 4 donuts in a 2x2 grid."""
+    if not donuts:
+        return
+    _section(pdf, "📊 STATS DE TON APPAREIL (graphiques)")
+    y0 = pdf.get_y() + 2
+    col_w = _page_w(pdf) / 2.0
+    row_h = 42.0
+    shown = donuts[:4]
+    for i, d in enumerate(shown):
+        col, row = i % 2, i // 2
+        _pdf_donut(pdf, pdf.l_margin + col * col_w, y0 + row * row_h, col_w - 4,
+                   d.get("title", ""), d.get("hole", ""), d.get("segments") or [])
+    rows = (len(shown) + 1) // 2
+    pdf.set_y(y0 + rows * row_h + 2)
+
+
 def _render_text_fallback(report: dict) -> str:
     """Plain text fallback when fpdf2 isn't installed."""
     lines = [