mirror of
https://github.com/CyberMind-FR/secubox-deb.git
synced 2026-07-01 17:17:14 +00:00
Compare commits
No commits in common. "9eb2d68b9232986e9ad53410ac333a497b37a195" and "69d4f0bd5cc1fef118ee95b0eff81f85351028ec" have entirely different histories.
9eb2d68b92
...
69d4f0bd5c
|
|
@ -3,25 +3,6 @@
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 2026-06-20 — kbin Tor shipped + client releases + ad-block/mitm hardening
|
|
||||||
|
|
||||||
- **#683 MERGED (PR #684)** — kbin Tor egress quick-switch (switch + nft owner-match
|
|
||||||
tunnel, own-services exemption, reconciler+timer), dashboard/landing/banner metrics
|
|
||||||
fixes, 🧅 indicators (banner/webext/APK), APK persistent WG identity, landing+report
|
|
||||||
**redesign** (verdict gauge + donut/bars + collapsible details). Live on gk2; Tor armed.
|
|
||||||
- **Client releases served from kbin**: `android-v0.4.0` (Latest) + `webext-v0.1.5`
|
|
||||||
published by CI; pinned webext tag bumped; board fetch-helpers pull them →
|
|
||||||
/wg/toolbox.apk (0.4.0) + /wg/toolbox.xpi (0.1.5). toolbox 2.7.12.
|
|
||||||
- **#685 ad-learner hardened (2.7.13)** — NEVER_LEARN guard (Google/CDN/fonts/captcha/
|
|
||||||
auth/payment), AD_MIN_SITES 1→2, prune existing. Root cause of euronews breakage:
|
|
||||||
the learner had 204'd `www.google.com` → broke reCAPTCHA/consent. Also allowlisted
|
|
||||||
www.google.com/.fr live.
|
|
||||||
- **mitm-wg stream_large_bodies=1m (2.7.14)** — large binary downloads (APK, CA) were
|
|
||||||
corrupted ONLY through the R3 tunnel (HTTP/2 buffer/reframe); now passed verbatim.
|
|
||||||
- **OPEN [#686]** — android-toolbox non-root flow broken (CA auto-install needs root,
|
|
||||||
WG handoff → Play Store, tunnel not detected). Needs on-device dev/testing; rooted-vs-
|
|
||||||
non-rooted decision pending. #685 signing was a red herring (corrupt = mitm buffering).
|
|
||||||
|
|
||||||
## 2026-06-19 — kbin Tor egress quick-switch implemented DARK (#683, ToolBoX 2.7.1)
|
## 2026-06-19 — kbin Tor egress quick-switch implemented DARK (#683, ToolBoX 2.7.1)
|
||||||
|
|
||||||
- **Switch + tunnel** for routing kbin surfing through Tor, shipped **default-OFF /
|
- **Switch + tunnel** for routing kbin surfing through Tor, shipped **default-OFF /
|
||||||
|
|
|
||||||
|
|
@ -75,18 +75,17 @@ ul{list-style:none;padding-left:.2rem}li{padding:.12rem 0;font-size:.82rem}li::b
|
||||||
<body>
|
<body>
|
||||||
|
|
||||||
{% set m = metrics or {} %}
|
{% set m = metrics or {} %}
|
||||||
{# #686 — summary + graphs come from the LIVE social graph (graph_stats), not the
|
{% set sc = risk_score|default(0) %}
|
||||||
frozen events table. #}
|
{% set rl = risk_label|default('LOW') %}
|
||||||
{% set gst = graph_stats or {} %}
|
|
||||||
{% set sc = exposure_score|default(0) %}
|
|
||||||
{% set ch = charts or {} %}
|
{% set ch = charts or {} %}
|
||||||
{% set gcol = 'var(--phos-hot)' if sc < 30 else ('var(--amber)' if sc < 70 else 'var(--red)') %}
|
{% set gcol = 'var(--phos-hot)' if sc < 30 else ('var(--amber)' if sc < 70 else 'var(--red)') %}
|
||||||
{% set palette = ['#00dd44','#9e76ff','#ff8866','#66bbff','#ffb347','#ff4466'] %}
|
{% set palette = ['#00dd44','#9e76ff','#ff8866','#66bbff','#ffb347','#ff4466'] %}
|
||||||
{% set n_trackers = gst.total_trackers|default(0) %}
|
{% set dpi_cls = dpi_classified or {} %}
|
||||||
{% set n_sites = gst.total_sites|default(0) %}
|
{% set cookies_p = cookies_providers or [] %}
|
||||||
{% set n_countries = gst.total_countries|default(0) %}
|
{% set geo_h = geo_top_hosts or [] %}
|
||||||
{% set n_antibot = gst.antibot_sites|default(0) %}
|
{% set n_apps = (dpi_cls.top_apps|default([])|selectattr('app','ne','?')|list|length) %}
|
||||||
{% set n_opgrade = gst.opgrade_sites|default(0) %}
|
{% set n_trackers = (cookies_p|map(attribute='count')|sum) %}
|
||||||
|
{% set n_countries = (geo_h|map(attribute='country')|reject('equalto','')|list|unique|list|length) %}
|
||||||
{% set _avatar = avatar_analysis or {} %}
|
{% set _avatar = avatar_analysis or {} %}
|
||||||
|
|
||||||
<h1>👁️ VILLAGE3B <span style="font-size:.8rem;color:var(--dim);font-weight:400">· mon rapport</span></h1>
|
<h1>👁️ VILLAGE3B <span style="font-size:.8rem;color:var(--dim);font-weight:400">· mon rapport</span></h1>
|
||||||
|
|
@ -110,21 +109,22 @@ ul{list-style:none;padding-left:.2rem}li{padding:.12rem 0;font-size:.82rem}li::b
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="verdict" style="color:{{ gcol }}">
|
<div class="verdict" style="color:{{ gcol }}">
|
||||||
{% if sc < 30 %}🟢 Exposition faible{% elif sc < 70 %}🟡 Exposition modérée{% else %}🔴 Exposition élevée{% endif %}
|
{% if sc < 30 %}🟢 Tout va bien — {{ rl }}{% elif sc < 70 %}🟡 À surveiller — {{ rl }}{% else %}🔴 Attention — {{ rl }}{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<p class="help">Niveau d'exposition au pistage (traceurs croisés + acteurs opérateur/anti-bot). Plus c'est <b>bas</b>, mieux c'est.</p>
|
<p class="help">Score de risque de ton appareil. Plus il est <b>bas</b>, mieux tu es protégé.</p>
|
||||||
|
{% if risk_explanation %}<p style="font-size:.85rem;margin-top:.5rem">{{ risk_explanation }}</p>{% endif %}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{# ── KPI row (LIVE social graph) ── #}
|
{# ── KPI row ── #}
|
||||||
<div class="kpis">
|
<div class="kpis">
|
||||||
<div class="kpi"><div class="e">🍪</div><div class="n">{{ n_trackers }}</div><div class="l">traceurs</div></div>
|
<div class="kpi"><div class="e">🌐</div><div class="n">{{ m.connections|default(0) }}</div><div class="l">connexions</div></div>
|
||||||
<div class="kpi"><div class="e">🌐</div><div class="n">{{ n_sites }}</div><div class="l">sites</div></div>
|
<div class="kpi"><div class="e">📡</div><div class="n">{{ m.unique_hosts|default(0) }}</div><div class="l">hôtes</div></div>
|
||||||
|
<div class="kpi"><div class="e">🍪</div><div class="n">{{ n_trackers }}</div><div class="l">trackers</div></div>
|
||||||
<div class="kpi"><div class="e">🌍</div><div class="n">{{ n_countries }}</div><div class="l">pays</div></div>
|
<div class="kpi"><div class="e">🌍</div><div class="n">{{ n_countries }}</div><div class="l">pays</div></div>
|
||||||
<div class="kpi"><div class="e">🤖</div><div class="n">{{ n_antibot }}</div><div class="l">anti-bot</div></div>
|
<div class="kpi"><div class="e">📺</div><div class="n">{{ n_apps }}</div><div class="l">apps</div></div>
|
||||||
<div class="kpi"><div class="e">📡</div><div class="n">{{ n_opgrade }}</div><div class="l">opérateur</div></div>
|
<div class="kpi"><div class="e">🔒</div><div class="n">{{ m.tls_pinned|default(0) }}</div><div class="l">cert-pin</div></div>
|
||||||
<div class="kpi"><div class="e">🔗</div><div class="n">{{ (graph.edges|default([]))|length }}</div><div class="l">liens</div></div>
|
|
||||||
</div>
|
</div>
|
||||||
<p class="help" style="text-align:center;margin-bottom:1rem">{{ n_trackers }} traceurs te suivent à travers {{ n_sites }} sites, depuis {{ n_countries }} pays.{% if n_opgrade %} Dont {{ n_opgrade }} de qualité opérateur.{% endif %}</p>
|
<p class="help" style="text-align:center;margin-bottom:1rem">Ton appareil a contacté {{ m.unique_hosts|default(0) }} serveurs dans {{ n_countries }} pays, avec {{ n_trackers }} traceurs repérés.</p>
|
||||||
|
|
||||||
{# ── GRAPHS ── #}
|
{# ── GRAPHS ── #}
|
||||||
<div class="card">
|
<div class="card">
|
||||||
|
|
@ -158,18 +158,18 @@ ul{list-style:none;padding-left:.2rem}li{padding:.12rem 0;font-size:.82rem}li::b
|
||||||
{% else %}<div class="empty">Pas encore de données géo</div>{% endif %}
|
{% else %}<div class="empty">Pas encore de données géo</div>{% endif %}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{# top tracked sites bars #}
|
{# apps bars #}
|
||||||
<div style="grid-column:1/-1">
|
<div style="grid-column:1/-1">
|
||||||
<div style="font-size:.82rem;color:var(--dim);margin-bottom:.4rem">🌐 Où tu es le plus pisté (traceurs par site)</div>
|
<div style="font-size:.82rem;color:var(--dim);margin-bottom:.4rem">📺 Quelles apps / services</div>
|
||||||
{% if ch.sites %}
|
{% if ch.apps %}
|
||||||
{% for a in ch.sites %}
|
{% for a in ch.apps %}
|
||||||
<div class="bar-row"><span class="bar-lbl">{{ a.label[:22] }}</span><span class="bar-track"><span class="bar-fill" style="width:{{ a.pct }}%;background:linear-gradient(90deg,var(--violet),#c9b6ff)"></span></span><span class="bar-val" style="color:var(--violet)">{{ a.count }}</span></div>
|
<div class="bar-row"><span class="bar-lbl">{{ a.emoji }} {{ a.label[:16] }}</span><span class="bar-track"><span class="bar-fill" style="width:{{ a.pct }}%;background:linear-gradient(90deg,var(--violet),#c9b6ff)"></span></span><span class="bar-val" style="color:var(--violet)">{{ a.count }}</span></div>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
{% else %}<div class="empty">Pas encore de sites pistés</div>{% endif %}
|
{% else %}<div class="empty">Aucune app classifiée</div>{% endif %}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
<p class="help">Les traceurs suivent ta navigation entre sites. « opérateur » = traceurs de niveau opérateur télécom (les plus intrusifs).</p>
|
<p class="help">Les traceurs suivent ta navigation entre sites. Les apps cert-pinning (🔒) refusent l'analyse — c'est bon signe.</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{# ── LEVEL SWITCHER (action) ── #}
|
{# ── LEVEL SWITCHER (action) ── #}
|
||||||
|
|
|
||||||
|
|
@ -1,53 +1,3 @@
|
||||||
secubox-toolbox (2.7.16-1~bookworm1) bookworm; urgency=medium
|
|
||||||
|
|
||||||
* fix: restore banner on heavy sites (leparisien.fr). The #685 stream_large_bodies=1m
|
|
||||||
streamed large HTML too (streamed bodies cannot be banner-injected). Replaced
|
|
||||||
by the content-aware stream_binaries addon: streams only large NON-HTML
|
|
||||||
(APK/XPI/video/octet-stream/big downloads) verbatim, HTML always buffered so
|
|
||||||
inject_banner + ad_ghost work.
|
|
||||||
|
|
||||||
-- Gerald KERMA <devel@cybermind.fr> Sat, 20 Jun 2026 13:40:00 +0200
|
|
||||||
|
|
||||||
secubox-toolbox (2.7.15-1~bookworm1) bookworm; urgency=medium
|
|
||||||
|
|
||||||
* fix(#686): /report/me/html reads the LIVE social graph (social.fetch_graph)
|
|
||||||
instead of the frozen events table (#662 cutover) — report was all-zeros even
|
|
||||||
when /social + webext showed data. Summary gauge = exposure score; KPIs
|
|
||||||
(traceurs/sites/pays/anti-bot/opérateur/liens) + graphs (trackers donut,
|
|
||||||
countries bars, top-pisté-sites bars) all from the live graph.
|
|
||||||
|
|
||||||
-- Gerald KERMA <devel@cybermind.fr> Sat, 20 Jun 2026 13:00:00 +0200
|
|
||||||
|
|
||||||
secubox-toolbox (2.7.14-1~bookworm1) bookworm; urgency=medium
|
|
||||||
|
|
||||||
* fix(#685): mitm-wg now streams large bodies (stream_large_bodies=1m) so big
|
|
||||||
binary downloads (APK, CA cert) pass through the R3 forging path verbatim
|
|
||||||
instead of being buffered/reframed over HTTP/2 — fixes "apk corrupt" /
|
|
||||||
"certificat vide" seen ONLY with the WG tunnel up. No addon touches non-HTML
|
|
||||||
bodies, so streaming is byte-transparent.
|
|
||||||
|
|
||||||
-- Gerald KERMA <devel@cybermind.fr> Sat, 20 Jun 2026 12:00:00 +0200
|
|
||||||
|
|
||||||
secubox-toolbox (2.7.13-1~bookworm1) bookworm; urgency=medium
|
|
||||||
|
|
||||||
* fix(#685): harden the ad-learner so it never 204s functional infra. The
|
|
||||||
aggressive promotion (AD_MIN_SITES=1) had hard-blocked www.google.com →
|
|
||||||
broke reCAPTCHA/consent on news sites (euronews). Now: a NEVER_LEARN guard
|
|
||||||
(Google/CDN/fonts/captcha/auth/payment registrables, matched on host +
|
|
||||||
registrable, env-extendable via SECUBOX_NEVER_LEARN), AD_MIN_SITES default
|
|
||||||
1 → 2, and existing never-learn/allowlisted entries are PRUNED from
|
|
||||||
learned-trackers.txt on each run.
|
|
||||||
|
|
||||||
-- Gerald KERMA <devel@cybermind.fr> Sat, 20 Jun 2026 11:00:00 +0200
|
|
||||||
|
|
||||||
secubox-toolbox (2.7.12-1~bookworm1) bookworm; urgency=medium
|
|
||||||
|
|
||||||
* chore: serve the new clients from kbin — bump pinned webext release tag
|
|
||||||
v0.1.4 → v0.1.5 (/wg/toolbox.xpi fallback + secubox-toolbox-fetch-xpi). The
|
|
||||||
APK serve path already pulls /releases/latest (now android-v0.4.0).
|
|
||||||
|
|
||||||
-- Gerald KERMA <devel@cybermind.fr> Sat, 20 Jun 2026 09:00:00 +0200
|
|
||||||
|
|
||||||
secubox-toolbox (2.7.11-1~bookworm1) bookworm; urgency=medium
|
secubox-toolbox (2.7.11-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
* feat: landing (kbin.gk2) restyled to match the new report — system font,
|
* feat: landing (kbin.gk2) restyled to match the new report — system font,
|
||||||
|
|
|
||||||
|
|
@ -1,53 +0,0 @@
|
||||||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
|
||||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
|
||||||
# Source-Disclosed License — All rights reserved except as expressly granted.
|
|
||||||
# See LICENCE-CMSD-1.0.md for terms.
|
|
||||||
#
|
|
||||||
# SecuBox-Deb :: toolbox :: stream large BINARY responses (#686)
|
|
||||||
#
|
|
||||||
# Replaces the content-agnostic `--set stream_large_bodies=1m`, which streamed
|
|
||||||
# EVERY body >1MB — including large HTML (leparisien.fr) — and streamed bodies
|
|
||||||
# can't be banner-injected → "plus de banner". Here we stream only large
|
|
||||||
# NON-HTML responses (APK / XPI / video / octet-stream / big downloads) so they
|
|
||||||
# pass through the HTTP/2 forging path VERBATIM (the buffer+reframe corrupted the
|
|
||||||
# 14MB APK over the R3 tunnel), while HTML is always buffered so inject_banner /
|
|
||||||
# ad_ghost still work.
|
|
||||||
from mitmproxy import http
|
|
||||||
|
|
||||||
_THRESHOLD = 1_000_000 # 1 MB
|
|
||||||
# Always-stream binary content-types regardless of declared length (covers
|
|
||||||
# chunked downloads with no Content-Length).
|
|
||||||
_BIN_CT = (
|
|
||||||
"application/vnd.android.package-archive", # .apk
|
|
||||||
"application/x-xpinstall", # .xpi
|
|
||||||
"application/octet-stream",
|
|
||||||
"application/zip",
|
|
||||||
"application/pdf",
|
|
||||||
"video/",
|
|
||||||
"audio/",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class StreamBinaries:
|
|
||||||
def responseheaders(self, flow: http.HTTPFlow) -> None:
|
|
||||||
try:
|
|
||||||
r = flow.response
|
|
||||||
if r is None:
|
|
||||||
return
|
|
||||||
ct = (r.headers.get("content-type", "") or "").lower()
|
|
||||||
if "text/html" in ct:
|
|
||||||
return # NEVER stream HTML — banner + ad_ghost need the body
|
|
||||||
if any(b in ct for b in _BIN_CT):
|
|
||||||
r.stream = True
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
cl = int(r.headers.get("content-length", "0") or "0")
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
cl = 0
|
|
||||||
if cl >= _THRESHOLD:
|
|
||||||
r.stream = True
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
addons = [StreamBinaries()]
|
|
||||||
|
|
@ -32,35 +32,10 @@ SPLICE_MIN_HITS = int(os.environ.get("SECUBOX_SPLICE_MIN_HITS", "20"))
|
||||||
SPLICE_MAX = 2000
|
SPLICE_MAX = 2000
|
||||||
MIN_SITES = 2 # cross-site threshold for operator-grade trackers
|
MIN_SITES = 2 # cross-site threshold for operator-grade trackers
|
||||||
MAX_ENTRIES = 8000
|
MAX_ENTRIES = 8000
|
||||||
# #656 — ad-candidate promotion. #685 hardening: require >= 2 distinct sites
|
# #656 — ad-candidate promotion (aggressive: 1 distinct site by default).
|
||||||
# (was 1 — a single-site host got hard-blocked, e.g. www.google.com → broke
|
AD_MIN_SITES = int(os.environ.get("SECUBOX_AD_MIN_SITES", "1"))
|
||||||
# reCAPTCHA/consent on euronews). Env-overridable.
|
|
||||||
AD_MIN_SITES = int(os.environ.get("SECUBOX_AD_MIN_SITES", "2"))
|
|
||||||
AD_ALLOWLIST = os.environ.get("SECUBOX_AD_ALLOWLIST",
|
AD_ALLOWLIST = os.environ.get("SECUBOX_AD_ALLOWLIST",
|
||||||
"/var/lib/secubox/toolbox/ad-allowlist.txt")
|
"/var/lib/secubox/toolbox/ad-allowlist.txt")
|
||||||
|
|
||||||
# #685 — NEVER-LEARN guard: registrables that host FUNCTIONAL content (CDNs,
|
|
||||||
# fonts, captcha, auth, OS/payment services). The learner must NEVER 204 these —
|
|
||||||
# blocking them breaks sites (www.google.com reCAPTCHA/consent broke euronews).
|
|
||||||
# Checked against the host AND its registrable; existing entries are also pruned.
|
|
||||||
_NEVER_LEARN_SEED = {
|
|
||||||
"google.com", "gstatic.com", "googleapis.com", "googleusercontent.com",
|
|
||||||
"googlevideo.com", "ytimg.com", "ggpht.com", "youtube.com", "recaptcha.net",
|
|
||||||
"apple.com", "icloud.com", "mzstatic.com", "cdn-apple.com", "cloudflare.com",
|
|
||||||
"jsdelivr.net", "jquery.com", "bootstrapcdn.com", "unpkg.com", "cdnjs.com",
|
|
||||||
"akamaihd.net", "akamai.net", "fastly.net", "edgekey.net", "edgesuite.net",
|
|
||||||
"microsoft.com", "office.com", "live.com", "windows.net", "azureedge.net",
|
|
||||||
"msftauth.net", "paypal.com", "paypalobjects.com", "stripe.com",
|
|
||||||
}
|
|
||||||
NEVER_LEARN = _NEVER_LEARN_SEED | {
|
|
||||||
d.strip().lower()
|
|
||||||
for d in os.environ.get("SECUBOX_NEVER_LEARN", "").split(",") if d.strip()
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _never_learn(host: str) -> bool:
|
|
||||||
h = (host or "").lower().strip(".")
|
|
||||||
return bool(h) and (h in NEVER_LEARN or (registrable(h) or h) in NEVER_LEARN)
|
|
||||||
COOKIE_XSITE_TOP_N = int(os.environ.get("SECUBOX_COOKIE_XSITE_TOP_N", "5"))
|
COOKIE_XSITE_TOP_N = int(os.environ.get("SECUBOX_COOKIE_XSITE_TOP_N", "5"))
|
||||||
|
|
||||||
sys.path.insert(0, os.environ.get("SECUBOX_TOOLBOX_LIB", "/usr/lib/secubox/toolbox"))
|
sys.path.insert(0, os.environ.get("SECUBOX_TOOLBOX_LIB", "/usr/lib/secubox/toolbox"))
|
||||||
|
|
@ -216,16 +191,13 @@ def _ad_feed() -> int:
|
||||||
continue
|
continue
|
||||||
if reg in self_doms or any(h == d or h.endswith("." + d) for d in self_doms):
|
if reg in self_doms or any(h == d or h.endswith("." + d) for d in self_doms):
|
||||||
continue
|
continue
|
||||||
# #685 — never hard-block functional infra (CDN/fonts/captcha/auth).
|
|
||||||
if _never_learn(h):
|
|
||||||
continue
|
|
||||||
# #658 — promote the EXACT host, NOT the registrable: blocking a tracker
|
# #658 — promote the EXACT host, NOT the registrable: blocking a tracker
|
||||||
# subdomain (analytics.tiktok.com) must never block the parent site
|
# subdomain (analytics.tiktok.com) must never block the parent site
|
||||||
# (tiktok.com). Dedicated ad hosts are already registrable-level.
|
# (tiktok.com). Dedicated ad hosts are already registrable-level.
|
||||||
promoted.add(h)
|
promoted.add(h)
|
||||||
# MERGE with existing learned-trackers.txt (union, dedup, cap). #685: also
|
if not promoted:
|
||||||
# PRUNE any existing never-learn / allowlisted entries already on disk, so a
|
return 0
|
||||||
# previously mis-learned host (e.g. www.google.com) is cleaned on the next run.
|
# MERGE with existing learned-trackers.txt (union, dedup, cap).
|
||||||
existing: set = set()
|
existing: set = set()
|
||||||
try:
|
try:
|
||||||
if os.path.exists(OUT):
|
if os.path.exists(OUT):
|
||||||
|
|
@ -236,11 +208,7 @@ def _ad_feed() -> int:
|
||||||
existing.add(ln)
|
existing.add(ln)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
sys.stderr.write(f"autolearn: ad merge read failed: {e}\n")
|
sys.stderr.write(f"autolearn: ad merge read failed: {e}\n")
|
||||||
pruned = {e for e in existing
|
merged = sorted(existing | promoted)[:MAX_ENTRIES]
|
||||||
if _never_learn(e) or e in allow or (registrable(e) or e) in allow}
|
|
||||||
if not promoted and not pruned:
|
|
||||||
return 0
|
|
||||||
merged = sorted((existing - pruned) | promoted)[:MAX_ENTRIES]
|
|
||||||
try:
|
try:
|
||||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||||||
tmp = OUT + ".tmp"
|
tmp = OUT + ".tmp"
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ DEST_DIR="/var/lib/secubox/toolbox/webext"
|
||||||
DEST="${DEST_DIR}/secubox-toolbox-webext.xpi"
|
DEST="${DEST_DIR}/secubox-toolbox-webext.xpi"
|
||||||
# Tag-pinned (not /latest/): the webext release is make_latest:false so it
|
# Tag-pinned (not /latest/): the webext release is make_latest:false so it
|
||||||
# doesn't steal "latest" from the Android APK release. Bump on new webext-v*.
|
# doesn't steal "latest" from the Android APK release. Bump on new webext-v*.
|
||||||
RELEASE_URL="https://github.com/CyberMind-FR/secubox-deb/releases/download/webext-v0.1.5/secubox-toolbox-webext.xpi"
|
RELEASE_URL="https://github.com/CyberMind-FR/secubox-deb/releases/download/webext-v0.1.4/secubox-toolbox-webext.xpi"
|
||||||
|
|
||||||
log() { logger -t "$MODULE" -- "$*" 2>/dev/null || echo "[$MODULE] $*" >&2; }
|
log() { logger -t "$MODULE" -- "$*" 2>/dev/null || echo "[$MODULE] $*" >&2; }
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -87,10 +87,6 @@ ARGS=(
|
||||||
# upstream in mitmproxy 10.4 ; with mitmproxy 11+ we can safely
|
# upstream in mitmproxy 10.4 ; with mitmproxy 11+ we can safely
|
||||||
# re-enable keep-alive. Halves TCP handshakes towards busy CDNs.
|
# re-enable keep-alive. Halves TCP handshakes towards busy CDNs.
|
||||||
--set keep_host_header=true
|
--set keep_host_header=true
|
||||||
# #686 — large-binary streaming is now content-aware via the stream_binaries
|
|
||||||
# addon (streams APK/XPI/video/large NON-HTML verbatim) instead of the blunt
|
|
||||||
# `stream_large_bodies=1m`, which also streamed large HTML and killed banner
|
|
||||||
# injection on heavy sites (leparisien.fr).
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if [ -n "$IGNORE_REGEX" ]; then
|
if [ -n "$IGNORE_REGEX" ]; then
|
||||||
|
|
@ -119,7 +115,7 @@ fi
|
||||||
# ad_ghost (#566) runs right after protective_mode: for R3+/R4 it 204s known
|
# ad_ghost (#566) runs right after protective_mode: for R3+/R4 it 204s known
|
||||||
# ad/tracker hosts (bandwidth save) at request time and injects ad-hiding CSS
|
# ad/tracker hosts (bandwidth save) at request time and injects ad-hiding CSS
|
||||||
# on HTML responses. Gated by the modular filter config (toolbox WebUI).
|
# on HTML responses. Gated by the modular filter config (toolbox WebUI).
|
||||||
for addon in stream_binaries tls_splice inject_xff utiq_defense protective_mode privacy_guard ad_ghost media_cache local_store social_graph inject_banner dpi cookies avatar ja4 soc_relay cert_pin_detect media_stats; do
|
for addon in tls_splice inject_xff utiq_defense protective_mode privacy_guard ad_ghost media_cache local_store social_graph inject_banner dpi cookies avatar ja4 soc_relay cert_pin_detect media_stats; do
|
||||||
ARGS+=(-s "$ADDON_DIR/${addon}.py")
|
ARGS+=(-s "$ADDON_DIR/${addon}.py")
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1632,7 +1632,7 @@ async def wg_toolbox_apk() -> Response:
|
||||||
_WEBEXT_XPI = Path("/var/lib/secubox/toolbox/webext/secubox-toolbox-webext.xpi")
|
_WEBEXT_XPI = Path("/var/lib/secubox/toolbox/webext/secubox-toolbox-webext.xpi")
|
||||||
_WEBEXT_XPI_RELEASE = (
|
_WEBEXT_XPI_RELEASE = (
|
||||||
"https://github.com/CyberMind-FR/secubox-deb/releases/download/"
|
"https://github.com/CyberMind-FR/secubox-deb/releases/download/"
|
||||||
"webext-v0.1.5/secubox-toolbox-webext.xpi"
|
"webext-v0.1.4/secubox-toolbox-webext.xpi"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -2342,12 +2342,11 @@ def _classify_apps(hosts: set[str]) -> list[str]:
|
||||||
return apps
|
return apps
|
||||||
|
|
||||||
|
|
||||||
def _build_report_charts(graph: dict) -> dict:
|
def _build_report_charts(session: dict) -> dict:
|
||||||
"""Graph-ready aggregates for the report, from the LIVE social graph
|
"""Graph-ready aggregates for the simplified report (trackers donut,
|
||||||
(social.fetch_graph). The events table froze at the #662 cutover, so the
|
countries bars, apps bars). Defensive / fail-empty. Each list item has
|
||||||
report reads the SAME source as /social + the webext (was the bug: it read
|
{label, emoji/flag, count, pct}; trackers also carry cumulative start/end
|
||||||
the dead events → all zeros). Returns trackers donut + countries bars + sites
|
for a CSS conic-gradient donut."""
|
||||||
bars; trackers also carry cumulative start/end for the CSS conic-gradient."""
|
|
||||||
def _top_pct(items: list, n: int = 6) -> list:
|
def _top_pct(items: list, n: int = 6) -> list:
|
||||||
items = [it for it in items if it.get("count")]
|
items = [it for it in items if it.get("count")]
|
||||||
items.sort(key=lambda x: x["count"], reverse=True)
|
items.sort(key=lambda x: x["count"], reverse=True)
|
||||||
|
|
@ -2357,33 +2356,30 @@ def _build_report_charts(graph: dict) -> dict:
|
||||||
it["pct"] = round(100 * it["count"] / total)
|
it["pct"] = round(100 * it["count"] / total)
|
||||||
return items
|
return items
|
||||||
|
|
||||||
g = graph or {}
|
cp = session.get("cookies_providers") or []
|
||||||
nodes = g.get("nodes") or []
|
|
||||||
|
|
||||||
trackers = _top_pct([
|
trackers = _top_pct([
|
||||||
{"label": (n.get("domain") or n.get("id") or "?"), "emoji": "🍪",
|
{"label": p.get("provider", "?"), "emoji": p.get("emoji", "🍪"),
|
||||||
"count": int(n.get("hits", 0) or 0)} for n in nodes])
|
"count": int(p.get("count", 0) or 0)} for p in cp])
|
||||||
cum = 0
|
cum = 0
|
||||||
for it in trackers:
|
for it in trackers:
|
||||||
it["start"] = cum
|
it["start"] = cum
|
||||||
cum += it["pct"]
|
cum += it["pct"]
|
||||||
it["end"] = cum
|
it["end"] = cum
|
||||||
|
|
||||||
|
by_country: dict = {}
|
||||||
|
for h in (session.get("geo_top_hosts") or []):
|
||||||
|
key = (h.get("flag") or "🏴", h.get("country") or "?")
|
||||||
|
by_country[key] = by_country.get(key, 0) + int(h.get("count", 0) or 0)
|
||||||
countries = _top_pct([
|
countries = _top_pct([
|
||||||
{"flag": c.get("flag") or "🏴", "label": (c.get("country_iso") or "?"),
|
{"flag": k[0], "label": k[1], "count": v} for k, v in by_country.items()])
|
||||||
"count": int(c.get("hits", 0) or 0)} for c in (g.get("by_country") or [])])
|
|
||||||
|
|
||||||
# top tracked sites = number of DISTINCT trackers reaching each first-party
|
dc = session.get("dpi_classified") or {}
|
||||||
# site (from each node's sites list) — "where you're tracked most".
|
apps = _top_pct([
|
||||||
site_trk: dict = {}
|
{"label": a.get("app", "?"), "emoji": a.get("emoji", "📦"),
|
||||||
for n in nodes:
|
"count": int(a.get("count", 0) or 0)}
|
||||||
for s in (n.get("sites") or []):
|
for a in (dc.get("top_apps") or []) if a.get("app") not in (None, "", "?")])
|
||||||
if s:
|
|
||||||
site_trk[s] = site_trk.get(s, 0) + 1
|
|
||||||
sites = _top_pct([{"label": s, "emoji": "🌐", "count": c}
|
|
||||||
for s, c in site_trk.items()])
|
|
||||||
|
|
||||||
return {"trackers": trackers, "countries": countries, "sites": sites}
|
return {"trackers": trackers, "countries": countries, "apps": apps}
|
||||||
|
|
||||||
|
|
||||||
# NOTE: route order matters in FastAPI — specific routes (/report/me,
|
# NOTE: route order matters in FastAPI — specific routes (/report/me,
|
||||||
|
|
@ -2412,21 +2408,6 @@ async def report_me_html(request: Request) -> HTMLResponse:
|
||||||
)
|
)
|
||||||
ip = _client_ip(request) or (request.client.host if request.client else "?")
|
ip = _client_ip(request) or (request.client.host if request.client else "?")
|
||||||
session = _aggregate_session(mac_hash)
|
session = _aggregate_session(mac_hash)
|
||||||
# #686 — the events table froze at the #662 cutover, so the report's numbers
|
|
||||||
# came out all-zero. The LIVE per-client data is the social graph (same source
|
|
||||||
# /social + the webext use). Pull it (7d) and drive the summary + graphs off it.
|
|
||||||
try:
|
|
||||||
from . import social as _social
|
|
||||||
graph = _social.fetch_graph(mac_hash, since_seconds=7 * 86400)
|
|
||||||
except Exception:
|
|
||||||
graph = {"stats": {}, "nodes": [], "by_country": []}
|
|
||||||
gs = graph.get("stats") or {}
|
|
||||||
# Honest exposure indicator (0-100) from the live graph: tracker breadth +
|
|
||||||
# operator-grade / anti-bot presence. Not a "compromise" score (events dead).
|
|
||||||
exposure_score = min(100, int(
|
|
||||||
(gs.get("total_trackers", 0) or 0) * 1.5
|
|
||||||
+ (gs.get("opgrade_sites", 0) or 0) * 12
|
|
||||||
+ (gs.get("antibot_sites", 0) or 0) * 8))
|
|
||||||
# Phase 3 (#492) : pass query args + force no-cache so iPhone Safari
|
# Phase 3 (#492) : pass query args + force no-cache so iPhone Safari
|
||||||
# actually fetches the new template.
|
# actually fetches the new template.
|
||||||
# Phase 6 (#496) : also pass wg_enabled so dashboard R3 link renders
|
# Phase 6 (#496) : also pass wg_enabled so dashboard R3 link renders
|
||||||
|
|
@ -2439,8 +2420,7 @@ async def report_me_html(request: Request) -> HTMLResponse:
|
||||||
current_level=store.get_client_level(mac_hash) if mac_hash else "r1",
|
current_level=store.get_client_level(mac_hash) if mac_hash else "r1",
|
||||||
wg_enabled=wg_enabled,
|
wg_enabled=wg_enabled,
|
||||||
cumulative=cumulative,
|
cumulative=cumulative,
|
||||||
graph=graph, graph_stats=gs, exposure_score=exposure_score,
|
charts=_build_report_charts(session),
|
||||||
charts=_build_report_charts(graph),
|
|
||||||
**session,
|
**session,
|
||||||
)
|
)
|
||||||
return HTMLResponse(html, headers={
|
return HTMLResponse(html, headers={
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user