diff --git a/.claude/HISTORY.md b/.claude/HISTORY.md index e15b685f..b2695836 100644 --- a/.claude/HISTORY.md +++ b/.claude/HISTORY.md @@ -5187,3 +5187,18 @@ git checkout HEAD -- index.html - ↻ Auto-rotate button: continuous rotation animation - **Default Colorset**: RGB (simple red/green/blue) - **Deployed**: https://wall.maegia.tv/ + +### 2026-03-15 + +- **Dual-Stream DPI Architecture (Phase 1 Complete)** + - New `secubox-dpi-dual` package: parallel MITM + Passive TAP deep packet inspection + - Architecture: DUAL-STREAM-DPI.md comprehensive spec document + - TAP Stream: tc mirred port mirroring → dummy interface → netifyd/nDPI analysis + - Flow Collector: Stats aggregation from netifyd, writes `/tmp/secubox/dpi-flows.json` + - Correlation Engine: Matches MITM WAF events + TAP flow data, CrowdSec integration + - CLI: `dpi-dualctl` start/stop/status/flows/threats/mirror + - Procd service: manages flow-collector + correlator instances + - MITM Double Buffer: `dpi_buffer.py` mitmproxy addon (Phase 2 prep) + - UCI config: `/etc/config/dpi-dual` with dual/mitm-only/tap-only modes + - Files: mirror-setup.sh, dpi-flow-collector, dpi-correlator, dpi-dualctl, init.d/dpi-dual, dpi_buffer.py + diff --git a/.claude/WIP.md b/.claude/WIP.md index 263e8652..cbcc2c1f 100644 --- a/.claude/WIP.md +++ b/.claude/WIP.md @@ -579,6 +579,38 @@ _Last updated: 2026-03-15 (Wall Colorsets)_ (No active tasks) +### 2026-03-15 + +- **Dual-Stream DPI Architecture (Phase 1 Complete)** + - New `secubox-dpi-dual` package implementing parallel MITM + Passive TAP DPI + - Architecture doc: `package/secubox/DUAL-STREAM-DPI.md` + - **TAP Stream (Passive)**: + - `mirror-setup.sh`: tc mirred port mirroring (ingress + egress) + - Creates dummy TAP interface for netifyd analysis + - Software and hardware TAP mode support + - **Flow Collector**: + - `dpi-flow-collector`: Aggregates netifyd flow statistics + - Writes stats to `/tmp/secubox/dpi-flows.json` + - Interface stats from /sys/class/net + - Configurable flow retention cleanup + - **Correlation Engine**: + - `dpi-correlator`: Matches MITM + TAP stream events + - Watches CrowdSec decisions and WAF alerts + - Enriches threats with context from both streams + - Output: `/tmp/secubox/correlated-threats.json` + - **CLI Tool**: + - `dpi-dualctl`: start/stop/restart/status/flows/threats/mirror + - Shows unified status of both streams + - **Procd Service**: + - `init.d/dpi-dual`: Manages flow-collector and correlator instances + - Auto-starts based on UCI mode setting (dual/mitm-only/tap-only) + - **MITM Double Buffer (Phase 2 prep)**: + - `dpi_buffer.py`: mitmproxy addon for async analysis + - Ring buffer with configurable size (1000 requests default) + - Heuristic threat scoring (path traversal, XSS, SQLi, LFI patterns) + - Writes threats to `/tmp/secubox/waf-alerts.json` + - **UCI Config**: `/etc/config/dpi-dual` with global, mitm, tap, correlation sections + --- ## Next Up diff --git a/package/secubox/secubox-dpi-dual/Makefile b/package/secubox/secubox-dpi-dual/Makefile new file mode 100644 index 00000000..f7149c52 --- /dev/null +++ b/package/secubox/secubox-dpi-dual/Makefile @@ -0,0 +1,51 @@ +include $(TOPDIR)/rules.mk + +PKG_NAME:=secubox-dpi-dual +PKG_VERSION:=1.0.0 +PKG_RELEASE:=1 + +PKG_MAINTAINER:=SecuBox +PKG_LICENSE:=GPL-3.0 + +include $(INCLUDE_DIR)/package.mk + +define Package/secubox-dpi-dual + SECTION:=secubox + CATEGORY:=SecuBox + SUBMENU:=Security + TITLE:=Dual-Stream DPI (MITM + Passive TAP) + DEPENDS:=+netifyd +iproute2-tc +jsonfilter +coreutils-stat + PKGARCH:=all +endef + +define Package/secubox-dpi-dual/description + Dual-stream Deep Packet Inspection architecture: + - Stream 1 (MITM): HAProxy + mitmproxy with double buffer + - Stream 2 (TAP): tc mirred + netifyd passive analysis + - Correlation engine for unified threat analytics +endef + +define Package/secubox-dpi-dual/conffiles +/etc/config/dpi-dual +endef + +define Package/secubox-dpi-dual/install + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_CONF) ./files/etc/config/dpi-dual $(1)/etc/config/ + + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/etc/init.d/dpi-dual $(1)/etc/init.d/ + + $(INSTALL_DIR) $(1)/usr/sbin + $(INSTALL_BIN) ./files/usr/sbin/dpi-dualctl $(1)/usr/sbin/ + $(INSTALL_BIN) ./files/usr/sbin/dpi-flow-collector $(1)/usr/sbin/ + $(INSTALL_BIN) ./files/usr/sbin/dpi-correlator $(1)/usr/sbin/ + + $(INSTALL_DIR) $(1)/usr/lib/dpi-dual + $(INSTALL_BIN) ./files/usr/lib/dpi-dual/mirror-setup.sh $(1)/usr/lib/dpi-dual/ + + $(INSTALL_DIR) $(1)/srv/mitmproxy/addons + $(INSTALL_DATA) ./files/srv/mitmproxy/addons/dpi_buffer.py $(1)/srv/mitmproxy/addons/ +endef + +$(eval $(call BuildPackage,secubox-dpi-dual)) diff --git a/package/secubox/secubox-dpi-dual/files/etc/config/dpi-dual b/package/secubox/secubox-dpi-dual/files/etc/config/dpi-dual new file mode 100644 index 00000000..e1368e35 --- /dev/null +++ b/package/secubox/secubox-dpi-dual/files/etc/config/dpi-dual @@ -0,0 +1,27 @@ +config global 'settings' + option enabled '1' + option mode 'dual' + option correlation '1' + option stats_dir '/tmp/secubox' + option flow_dir '/tmp/dpi-flows' + +config mitm 'mitm' + option enabled '1' + option buffer_size '1000' + option async_analysis '1' + option replay_on_alert '1' + option buffer_dir '/tmp/dpi-buffer' + +config tap 'tap' + option enabled '1' + option interface 'tap0' + option mirror_source 'eth0' + option mirror_mode 'software' + option flow_retention '300' + option netifyd_instance 'tap' + +config correlation 'correlation' + option enabled '1' + option window '60' + option output '/tmp/secubox/correlated-threats.json' + option watch_crowdsec '1' diff --git a/package/secubox/secubox-dpi-dual/files/etc/init.d/dpi-dual b/package/secubox/secubox-dpi-dual/files/etc/init.d/dpi-dual new file mode 100644 index 00000000..5557dae1 --- /dev/null +++ b/package/secubox/secubox-dpi-dual/files/etc/init.d/dpi-dual @@ -0,0 +1,91 @@ +#!/bin/sh /etc/rc.common +# DPI Dual-Stream procd service +# Part of secubox-dpi-dual package + +START=95 +STOP=10 +USE_PROCD=1 + +NAME="dpi-dual" +PROG="/usr/sbin/dpi-dualctl" + +validate_section() { + uci_load_validate dpi-dual global "$1" "$2" \ + 'enabled:bool:1' \ + 'mode:string:dual' \ + 'correlation:bool:1' \ + 'stats_dir:string:/tmp/secubox' \ + 'flow_dir:string:/tmp/dpi-flows' +} + +start_service() { + local enabled mode + + config_load dpi-dual + config_get enabled settings enabled "1" + config_get mode settings mode "dual" + + [ "$enabled" != "1" ] && { + echo "DPI Dual-Stream is disabled" + return 0 + } + + echo "Starting DPI Dual-Stream (mode: $mode)..." + + # Create directories + local stats_dir flow_dir + config_get stats_dir settings stats_dir "/tmp/secubox" + config_get flow_dir settings flow_dir "/tmp/dpi-flows" + mkdir -p "$stats_dir" "$flow_dir" + + # Start TAP stream if enabled + case "$mode" in + dual|tap-only) + /usr/lib/dpi-dual/mirror-setup.sh start + + # Start flow collector as procd service + procd_open_instance flow-collector + procd_set_param command /usr/sbin/dpi-flow-collector start + procd_set_param respawn + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_close_instance + ;; + esac + + # Start correlator if enabled + local correlation + config_get correlation settings correlation "1" + if [ "$correlation" = "1" ]; then + procd_open_instance correlator + procd_set_param command /usr/sbin/dpi-correlator start + procd_set_param respawn + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_close_instance + fi + + echo "DPI Dual-Stream started" +} + +stop_service() { + echo "Stopping DPI Dual-Stream..." + + # Stop mirror + /usr/lib/dpi-dual/mirror-setup.sh stop 2>/dev/null + + echo "DPI Dual-Stream stopped" +} + +reload_service() { + stop_service + start_service +} + +service_triggers() { + procd_add_reload_trigger "dpi-dual" +} + +status() { + /usr/sbin/dpi-dualctl status +} diff --git a/package/secubox/secubox-dpi-dual/files/srv/mitmproxy/addons/dpi_buffer.py b/package/secubox/secubox-dpi-dual/files/srv/mitmproxy/addons/dpi_buffer.py new file mode 100644 index 00000000..6538e080 --- /dev/null +++ b/package/secubox/secubox-dpi-dual/files/srv/mitmproxy/addons/dpi_buffer.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +""" +DPI Double Buffer Addon for mitmproxy +Part of secubox-dpi-dual package + +Implements the double-buffer pattern: +- Buffer A: Live path, minimal latency (default mitmproxy behavior) +- Buffer B: Copy for deep analysis, async processing + +This addon queues requests for asynchronous analysis without +blocking the live traffic path. +""" + +import json +import time +import hashlib +import asyncio +from pathlib import Path +from collections import deque +from typing import Optional, Dict, Any +from mitmproxy import http, ctx + + +class DPIBuffer: + """Double-buffer for request analysis without blocking live traffic.""" + + def __init__(self): + self.buffer_size = 1000 + self.buffer: deque = deque(maxlen=self.buffer_size) + self.buffer_dir = Path("/tmp/dpi-buffer") + self.stats_file = Path("/tmp/secubox/dpi-buffer.json") + self.analysis_enabled = True + self.request_count = 0 + self.threat_count = 0 + + # Ensure directories exist + self.buffer_dir.mkdir(parents=True, exist_ok=True) + self.stats_file.parent.mkdir(parents=True, exist_ok=True) + + def load(self, loader): + """Load configuration from mitmproxy options.""" + loader.add_option( + name="dpi_buffer_size", + typespec=int, + default=1000, + help="Size of the request buffer for async analysis", + ) + loader.add_option( + name="dpi_async_analysis", + typespec=bool, + default=True, + help="Enable asynchronous request analysis", + ) + + def configure(self, updated): + """Apply configuration updates.""" + if "dpi_buffer_size" in updated: + self.buffer_size = ctx.options.dpi_buffer_size + # Resize buffer + new_buffer = deque(self.buffer, maxlen=self.buffer_size) + self.buffer = new_buffer + + if "dpi_async_analysis" in updated: + self.analysis_enabled = ctx.options.dpi_async_analysis + + def request(self, flow: http.HTTPFlow): + """ + Handle incoming request. + Buffer A: Forward immediately (default mitmproxy behavior) + Buffer B: Queue for async analysis + """ + self.request_count += 1 + + # Build entry for Buffer B (async analysis) + entry = self._build_entry(flow) + self.buffer.append(entry) + + # Queue for async analysis if enabled + if self.analysis_enabled: + asyncio.create_task(self._async_analyze(entry)) + + # Update stats periodically (every 10 requests) + if self.request_count % 10 == 0: + self._write_stats() + + def response(self, flow: http.HTTPFlow): + """Handle response - update buffer entry with response info.""" + if not flow.request.timestamp_start: + return + + # Find and update the corresponding entry + req_hash = self._request_hash(flow) + for entry in self.buffer: + if entry.get("req_hash") == req_hash: + entry["response"] = { + "status": flow.response.status_code if flow.response else None, + "content_length": len(flow.response.content) if flow.response and flow.response.content else 0, + "content_type": flow.response.headers.get("content-type", "") if flow.response else "", + } + break + + def _build_entry(self, flow: http.HTTPFlow) -> Dict[str, Any]: + """Build a buffer entry from a flow.""" + content_hash = None + if flow.request.content: + content_hash = hashlib.md5(flow.request.content).hexdigest() + + client_ip = "unknown" + if flow.client_conn and flow.client_conn.peername: + client_ip = flow.client_conn.peername[0] + + return { + "ts": flow.request.timestamp_start, + "req_hash": self._request_hash(flow), + "method": flow.request.method, + "host": flow.request.host, + "port": flow.request.port, + "path": flow.request.path, + "headers": dict(flow.request.headers), + "content_hash": content_hash, + "content_length": len(flow.request.content) if flow.request.content else 0, + "client_ip": client_ip, + "analyzed": False, + "threat_score": 0, + } + + def _request_hash(self, flow: http.HTTPFlow) -> str: + """Generate a unique hash for a request.""" + key = f"{flow.request.timestamp_start}:{flow.request.host}:{flow.request.path}" + return hashlib.md5(key.encode()).hexdigest()[:16] + + async def _async_analyze(self, entry: Dict[str, Any]): + """ + Async analysis pipeline - runs without blocking live traffic. + + Analysis steps: + 1. Pattern matching against known threat signatures + 2. Anomaly scoring based on request characteristics + 3. Rate limiting detection + 4. Write results to analysis log + """ + try: + threat_score = 0 + + # Simple heuristic analysis (placeholder for more sophisticated detection) + # Check for common attack patterns in path + suspicious_patterns = [ + "../", "..\\", # Path traversal + " 1000000: + threat_score += 10 # Large file upload + + # Update entry with analysis results + entry["analyzed"] = True + entry["threat_score"] = min(threat_score, 100) + + # Track threats + if threat_score > 30: + self.threat_count += 1 + await self._log_threat(entry) + + except Exception as e: + ctx.log.error(f"DPI Buffer analysis error: {e}") + + async def _log_threat(self, entry: Dict[str, Any]): + """Log a detected threat to the alerts file.""" + alert_file = Path("/tmp/secubox/waf-alerts.json") + try: + alerts = [] + if alert_file.exists(): + alerts = json.loads(alert_file.read_text()) + + alert_id = len(alerts) + 1 + alerts.append({ + "id": alert_id, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), + "client_ip": entry.get("client_ip"), + "host": entry.get("host"), + "path": entry.get("path"), + "method": entry.get("method"), + "threat_score": entry.get("threat_score"), + "rule": "dpi_buffer_analysis", + }) + + # Keep last 1000 alerts + alerts = alerts[-1000:] + alert_file.write_text(json.dumps(alerts, indent=2)) + + except Exception as e: + ctx.log.error(f"Failed to log threat: {e}") + + def _write_stats(self): + """Write buffer statistics to stats file.""" + try: + stats = { + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), + "entries": len(self.buffer), + "max_size": self.buffer_size, + "requests_total": self.request_count, + "threats_detected": self.threat_count, + "analysis_enabled": self.analysis_enabled, + } + self.stats_file.write_text(json.dumps(stats, indent=2)) + except Exception as e: + ctx.log.error(f"Failed to write stats: {e}") + + def get_context(self, client_ip: str, window_sec: int = 60) -> list: + """ + Get recent requests from the same IP for context on alerts. + Used by the correlation engine to gather context around threat events. + """ + now = time.time() + return [ + e for e in self.buffer + if e.get("client_ip") == client_ip + and now - e.get("ts", 0) < window_sec + ] + + +# Mitmproxy addon instance +addons = [DPIBuffer()] diff --git a/package/secubox/secubox-dpi-dual/files/usr/lib/dpi-dual/mirror-setup.sh b/package/secubox/secubox-dpi-dual/files/usr/lib/dpi-dual/mirror-setup.sh new file mode 100644 index 00000000..5811c3fe --- /dev/null +++ b/package/secubox/secubox-dpi-dual/files/usr/lib/dpi-dual/mirror-setup.sh @@ -0,0 +1,155 @@ +#!/bin/sh +# Mirror setup for passive DPI using tc mirred +# Creates a TAP interface and mirrors traffic from source interface + +. /lib/functions.sh + +config_load dpi-dual + +TAP_IF="" +MIRROR_SRC="" +MIRROR_MODE="" + +load_config() { + config_get TAP_IF tap interface "tap0" + config_get MIRROR_SRC tap mirror_source "eth0" + config_get MIRROR_MODE tap mirror_mode "software" +} + +create_tap_interface() { + echo "Creating TAP interface: $TAP_IF" + + # Remove if exists + ip link show "$TAP_IF" >/dev/null 2>&1 && ip link del "$TAP_IF" + + # Create dummy interface for receiving mirrored packets + ip link add name "$TAP_IF" type dummy + ip link set "$TAP_IF" up + + # Set promiscuous mode + ip link set "$TAP_IF" promisc on + + echo "TAP interface $TAP_IF created" +} + +setup_mirror_ingress() { + echo "Setting up ingress mirror: $MIRROR_SRC -> $TAP_IF" + + # Add ingress qdisc if not exists + tc qdisc show dev "$MIRROR_SRC" | grep -q "ingress" || \ + tc qdisc add dev "$MIRROR_SRC" handle ffff: ingress + + # Mirror all ingress traffic to TAP + tc filter add dev "$MIRROR_SRC" parent ffff: protocol all prio 1 \ + u32 match u32 0 0 \ + action mirred egress mirror dev "$TAP_IF" + + echo "Ingress mirror configured" +} + +setup_mirror_egress() { + echo "Setting up egress mirror: $MIRROR_SRC -> $TAP_IF" + + # Add root qdisc for egress if not exists + tc qdisc show dev "$MIRROR_SRC" | grep -q "prio" || \ + tc qdisc add dev "$MIRROR_SRC" handle 1: root prio + + # Mirror all egress traffic to TAP + tc filter add dev "$MIRROR_SRC" parent 1: protocol all prio 1 \ + u32 match u32 0 0 \ + action mirred egress mirror dev "$TAP_IF" + + echo "Egress mirror configured" +} + +cleanup_mirror() { + echo "Cleaning up mirror configuration for $MIRROR_SRC" + + # Remove ingress qdisc + tc qdisc del dev "$MIRROR_SRC" handle ffff: ingress 2>/dev/null + + # Remove root qdisc (careful - this removes all tc config) + tc qdisc del dev "$MIRROR_SRC" root 2>/dev/null + + echo "Mirror cleanup done" +} + +remove_tap_interface() { + echo "Removing TAP interface: $TAP_IF" + ip link del "$TAP_IF" 2>/dev/null + echo "TAP interface removed" +} + +status() { + echo "=== TAP Interface ===" + if ip link show "$TAP_IF" >/dev/null 2>&1; then + ip -s link show "$TAP_IF" + else + echo "TAP interface $TAP_IF not found" + fi + + echo "" + echo "=== Mirror Source: $MIRROR_SRC ===" + echo "Ingress qdisc:" + tc qdisc show dev "$MIRROR_SRC" | grep ingress || echo " (none)" + echo "Ingress filters:" + tc filter show dev "$MIRROR_SRC" parent ffff: 2>/dev/null || echo " (none)" + + echo "" + echo "Egress qdisc:" + tc qdisc show dev "$MIRROR_SRC" | grep -v ingress || echo " (none)" + echo "Egress filters:" + tc filter show dev "$MIRROR_SRC" parent 1: 2>/dev/null || echo " (none)" +} + +start() { + load_config + + if [ "$MIRROR_MODE" = "hardware" ]; then + echo "Hardware TAP mode - configure switch port mirroring manually" + echo "TAP interface: $TAP_IF" + return 0 + fi + + # Software mirroring with tc mirred + create_tap_interface + setup_mirror_ingress + setup_mirror_egress + + echo "" + echo "Mirror setup complete:" + echo " Source: $MIRROR_SRC" + echo " TAP: $TAP_IF" + echo " Mode: $MIRROR_MODE" +} + +stop() { + load_config + + cleanup_mirror + remove_tap_interface + + echo "Mirror stopped" +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + stop + sleep 1 + start + ;; + status) + load_config + status + ;; + *) + echo "Usage: $0 {start|stop|restart|status}" + exit 1 + ;; +esac diff --git a/package/secubox/secubox-dpi-dual/files/usr/sbin/dpi-correlator b/package/secubox/secubox-dpi-dual/files/usr/sbin/dpi-correlator new file mode 100644 index 00000000..5e2e4044 --- /dev/null +++ b/package/secubox/secubox-dpi-dual/files/usr/sbin/dpi-correlator @@ -0,0 +1,184 @@ +#!/bin/sh +# DPI Correlator - Matches events from MITM and TAP streams +# Part of secubox-dpi-dual package + +. /lib/functions.sh + +config_load dpi-dual + +MITM_LOG="" +DPI_FLOWS="" +CORRELATED="" +WINDOW="" +WATCH_CROWDSEC="" + +load_config() { + config_get MITM_LOG mitm log_file "/var/log/mitmproxy/access.log" + config_get DPI_FLOWS settings flow_dir "/tmp/dpi-flows" + config_get CORRELATED correlation output "/tmp/secubox/correlated-threats.json" + config_get WINDOW correlation window "60" + config_get WATCH_CROWDSEC correlation watch_crowdsec "1" +} + +init_files() { + local corr_dir + corr_dir=$(dirname "$CORRELATED") + mkdir -p "$corr_dir" + [ ! -f "$CORRELATED" ] && echo "[]" > "$CORRELATED" +} + +# Get recent MITM requests from an IP +get_mitm_context() { + local ip="$1" + local count="${2:-10}" + + if [ -f "$MITM_LOG" ]; then + grep "$ip" "$MITM_LOG" 2>/dev/null | tail -"$count" | \ + awk -F'\t' '{printf "{\"ts\":\"%s\",\"method\":\"%s\",\"host\":\"%s\",\"path\":\"%s\"},", $1, $2, $3, $4}' | \ + sed 's/,$//' + fi +} + +# Get DPI flow info for an IP +get_dpi_context() { + local ip="$1" + + if [ -d "$DPI_FLOWS" ]; then + find "$DPI_FLOWS" -name "*.json" -exec grep -l "$ip" {} \; 2>/dev/null | \ + head -5 | xargs cat 2>/dev/null | \ + jsonfilter -e '@' 2>/dev/null | \ + tr '\n' ',' | sed 's/,$//' + fi +} + +# Correlate a threat event +correlate_threat() { + local ip="$1" + local timestamp="$2" + local reason="${3:-unknown}" + + local mitm_ctx dpi_ctx + mitm_ctx=$(get_mitm_context "$ip") + dpi_ctx=$(get_dpi_context "$ip") + + # Build correlation entry + local entry + entry=$(cat << EOF +{ + "ip": "$ip", + "timestamp": "$timestamp", + "reason": "$reason", + "mitm_context": [$mitm_ctx], + "dpi_context": [$dpi_ctx], + "correlated_at": "$(date -Iseconds)" +} +EOF +) + + # Append to correlated file (keep last 1000 entries) + local tmp_file="/tmp/correlated_$$.json" + if [ -f "$CORRELATED" ]; then + # Read existing, add new, keep last 1000 + (cat "$CORRELATED" 2>/dev/null | jsonfilter -e '@[*]' 2>/dev/null; echo "$entry") | \ + tail -1000 | \ + awk 'BEGIN{print "["} {if(NR>1)print ","; print} END{print "]"}' > "$tmp_file" + mv "$tmp_file" "$CORRELATED" + else + echo "[$entry]" > "$CORRELATED" + fi + + echo "Correlated threat: $ip ($reason)" +} + +# Watch CrowdSec for new decisions +watch_crowdsec_decisions() { + local cs_db="/var/lib/crowdsec/data/crowdsec.db" + local last_check="/tmp/dpi-correlator-lastcheck" + + [ ! -f "$cs_db" ] && return + + # Get timestamp of last check + local last_ts=0 + [ -f "$last_check" ] && last_ts=$(cat "$last_check") + + # Query new decisions (simplified - just check recent bans) + if command -v cscli >/dev/null 2>&1; then + cscli decisions list -o json 2>/dev/null | \ + jsonfilter -e '@[*].value' 2>/dev/null | \ + while read -r ip; do + [ -n "$ip" ] && correlate_threat "$ip" "$(date -Iseconds)" "crowdsec_ban" + done + fi + + # Update last check timestamp + date +%s > "$last_check" +} + +# Watch for WAF alerts from mitmproxy +watch_waf_alerts() { + local waf_alerts="/tmp/secubox/waf-alerts.json" + local last_alert="/tmp/dpi-correlator-lastalert" + + [ ! -f "$waf_alerts" ] && return + + local last_id=0 + [ -f "$last_alert" ] && last_id=$(cat "$last_alert") + + # Process new alerts + local current_id + current_id=$(jsonfilter -i "$waf_alerts" -e '@[-1].id' 2>/dev/null || echo 0) + + if [ "$current_id" -gt "$last_id" ]; then + # Get new alerts + jsonfilter -i "$waf_alerts" -e '@[*]' 2>/dev/null | while read -r alert; do + local alert_id ip reason + alert_id=$(echo "$alert" | jsonfilter -e '@.id' 2>/dev/null) + [ "$alert_id" -le "$last_id" ] && continue + + ip=$(echo "$alert" | jsonfilter -e '@.client_ip' 2>/dev/null) + reason=$(echo "$alert" | jsonfilter -e '@.rule' 2>/dev/null || echo "waf_alert") + + [ -n "$ip" ] && correlate_threat "$ip" "$(date -Iseconds)" "$reason" + done + + echo "$current_id" > "$last_alert" + fi +} + +run_correlator() { + load_config + init_files + + echo "DPI Correlator started" + echo " Window: ${WINDOW}s" + echo " Output: $CORRELATED" + echo " Watch CrowdSec: $WATCH_CROWDSEC" + + while true; do + [ "$WATCH_CROWDSEC" = "1" ] && watch_crowdsec_decisions + watch_waf_alerts + sleep 5 + done +} + +case "$1" in + start) + run_correlator + ;; + correlate) + # Manual correlation: dpi-correlator correlate [reason] + load_config + init_files + [ -n "$2" ] && correlate_threat "$2" "$(date -Iseconds)" "${3:-manual}" + ;; + status) + load_config + echo "Correlated threats: $(wc -l < "$CORRELATED" 2>/dev/null || echo 0)" + echo "Last 5 correlations:" + tail -5 "$CORRELATED" 2>/dev/null | jsonfilter -e '@[*]' 2>/dev/null || echo " (none)" + ;; + *) + echo "Usage: $0 {start|correlate [reason]|status}" + exit 1 + ;; +esac diff --git a/package/secubox/secubox-dpi-dual/files/usr/sbin/dpi-dualctl b/package/secubox/secubox-dpi-dual/files/usr/sbin/dpi-dualctl new file mode 100644 index 00000000..b6531668 --- /dev/null +++ b/package/secubox/secubox-dpi-dual/files/usr/sbin/dpi-dualctl @@ -0,0 +1,235 @@ +#!/bin/sh +# DPI Dual Control - CLI for dual-stream DPI management +# Part of secubox-dpi-dual package + +. /lib/functions.sh + +config_load dpi-dual + +STATS_DIR="" +FLOW_DIR="" + +load_config() { + config_get STATS_DIR settings stats_dir "/tmp/secubox" + config_get FLOW_DIR settings flow_dir "/tmp/dpi-flows" +} + +cmd_start() { + echo "Starting DPI Dual-Stream..." + + # Check mode + local mode + config_get mode settings mode "dual" + + case "$mode" in + dual|tap-only) + echo "Starting TAP stream..." + /usr/lib/dpi-dual/mirror-setup.sh start + + # Restart netifyd to pick up TAP interface + if /etc/init.d/netifyd enabled 2>/dev/null; then + /etc/init.d/netifyd restart + fi + + # Start flow collector + start-stop-daemon -S -b -x /usr/sbin/dpi-flow-collector -- start + echo "TAP stream started" + ;; + esac + + case "$mode" in + dual|mitm-only) + echo "MITM stream managed by mitmproxy service" + ;; + esac + + # Start correlator if enabled + local correlation + config_get correlation settings correlation "1" + if [ "$correlation" = "1" ]; then + echo "Starting correlator..." + start-stop-daemon -S -b -x /usr/sbin/dpi-correlator -- start + echo "Correlator started" + fi + + echo "DPI Dual-Stream started (mode: $mode)" +} + +cmd_stop() { + echo "Stopping DPI Dual-Stream..." + + # Stop correlator + killall dpi-correlator 2>/dev/null + + # Stop flow collector + killall dpi-flow-collector 2>/dev/null + + # Stop mirror + /usr/lib/dpi-dual/mirror-setup.sh stop + + echo "DPI Dual-Stream stopped" +} + +cmd_status() { + load_config + + local mode + config_get mode settings mode "dual" + + echo "=== DPI Dual-Stream Status ===" + echo "Mode: $mode" + echo "" + + echo "=== MITM Stream ===" + if pgrep mitmproxy >/dev/null 2>&1; then + echo "Status: RUNNING" + pgrep -a mitmproxy | head -1 + else + echo "Status: STOPPED" + fi + + local buffer_file="$STATS_DIR/dpi-buffer.json" + if [ -f "$buffer_file" ]; then + local entries + entries=$(jsonfilter -i "$buffer_file" -e '@.entries' 2>/dev/null || echo 0) + echo "Buffer entries: $entries" + else + echo "Buffer: not available" + fi + echo "" + + echo "=== TAP Stream ===" + local tap_if + config_get tap_if tap interface "tap0" + + if ip link show "$tap_if" >/dev/null 2>&1; then + echo "TAP Interface: $tap_if (UP)" + ip -s link show "$tap_if" 2>/dev/null | grep -E "RX:|TX:" | head -2 + else + echo "TAP Interface: $tap_if (DOWN)" + fi + + if pgrep netifyd >/dev/null 2>&1; then + echo "netifyd: RUNNING" + else + echo "netifyd: STOPPED" + fi + + if pgrep dpi-flow-collector >/dev/null 2>&1; then + echo "Flow Collector: RUNNING" + else + echo "Flow Collector: STOPPED" + fi + + local flows_file="$STATS_DIR/dpi-flows.json" + if [ -f "$flows_file" ]; then + local flows_1min + flows_1min=$(jsonfilter -i "$flows_file" -e '@.flows_1min' 2>/dev/null || echo 0) + echo "Flows (1min): $flows_1min" + fi + echo "" + + echo "=== Correlation Engine ===" + if pgrep dpi-correlator >/dev/null 2>&1; then + echo "Status: RUNNING" + else + echo "Status: STOPPED" + fi + + local corr_file + config_get corr_file correlation output "/tmp/secubox/correlated-threats.json" + if [ -f "$corr_file" ]; then + local threats + threats=$(wc -l < "$corr_file" 2>/dev/null || echo 0) + echo "Threats correlated: $threats" + else + echo "Threats correlated: 0" + fi +} + +cmd_flows() { + load_config + local flows_file="$STATS_DIR/dpi-flows.json" + + if [ -f "$flows_file" ]; then + cat "$flows_file" + else + echo '{"error":"No flow data available"}' + fi +} + +cmd_threats() { + local count="${1:-20}" + local corr_file + config_get corr_file correlation output "/tmp/secubox/correlated-threats.json" + + if [ -f "$corr_file" ]; then + tail -"$count" "$corr_file" + else + echo '[]' + fi +} + +cmd_mirror() { + /usr/lib/dpi-dual/mirror-setup.sh "$@" +} + +cmd_help() { + cat << EOF +DPI Dual-Stream Control + +Usage: $0 [args] + +Commands: + start Start all DPI streams (according to mode) + stop Stop all DPI streams + restart Restart all DPI streams + status Show status of all streams + flows Show current flow statistics (JSON) + threats [N] Show last N correlated threats (default: 20) + mirror Control mirror setup (start|stop|status) + help Show this help + +Configuration: /etc/config/dpi-dual + +Modes: + dual - Both MITM and TAP streams active + mitm-only - Only MITM stream (HAProxy + mitmproxy) + tap-only - Only passive TAP stream (netifyd) + +EOF +} + +case "$1" in + start) + cmd_start + ;; + stop) + cmd_stop + ;; + restart) + cmd_stop + sleep 1 + cmd_start + ;; + status) + cmd_status + ;; + flows) + cmd_flows + ;; + threats) + cmd_threats "$2" + ;; + mirror) + shift + cmd_mirror "$@" + ;; + help|--help|-h) + cmd_help + ;; + *) + cmd_help + exit 1 + ;; +esac diff --git a/package/secubox/secubox-dpi-dual/files/usr/sbin/dpi-flow-collector b/package/secubox/secubox-dpi-dual/files/usr/sbin/dpi-flow-collector new file mode 100644 index 00000000..b0037d81 --- /dev/null +++ b/package/secubox/secubox-dpi-dual/files/usr/sbin/dpi-flow-collector @@ -0,0 +1,111 @@ +#!/bin/sh +# DPI Flow Collector - Aggregates netifyd flow statistics +# Part of secubox-dpi-dual package + +. /lib/functions.sh + +config_load dpi-dual + +FLOW_DIR="" +STATS_DIR="" +RETENTION="" + +load_config() { + config_get FLOW_DIR settings flow_dir "/tmp/dpi-flows" + config_get STATS_DIR settings stats_dir "/tmp/secubox" + config_get RETENTION tap flow_retention "300" +} + +init_dirs() { + mkdir -p "$FLOW_DIR" + mkdir -p "$STATS_DIR" +} + +collect_flows() { + local stats_file="$STATS_DIR/dpi-flows.json" + local netifyd_socket="/var/run/netifyd/netifyd.sock" + + # Count recent flow files + local total_flows=0 + if [ -d "$FLOW_DIR" ]; then + total_flows=$(find "$FLOW_DIR" -name "*.json" -mmin -1 2>/dev/null | wc -l) + fi + + # Get protocol distribution from netifyd status + local protocols="{}" + if [ -S "$netifyd_socket" ]; then + # Try to get stats from netifyd socket + local proto_data + proto_data=$(echo '{"type":"get_stats"}' | nc -U "$netifyd_socket" 2>/dev/null | head -1) + if [ -n "$proto_data" ]; then + protocols=$(echo "$proto_data" | jsonfilter -e '@.protocols' 2>/dev/null || echo '{}') + fi + fi + + # Get interface stats from /proc + local tap_if + config_get tap_if tap interface "tap0" + + local rx_bytes=0 tx_bytes=0 rx_packets=0 tx_packets=0 + if [ -d "/sys/class/net/$tap_if/statistics" ]; then + rx_bytes=$(cat "/sys/class/net/$tap_if/statistics/rx_bytes" 2>/dev/null || echo 0) + tx_bytes=$(cat "/sys/class/net/$tap_if/statistics/tx_bytes" 2>/dev/null || echo 0) + rx_packets=$(cat "/sys/class/net/$tap_if/statistics/rx_packets" 2>/dev/null || echo 0) + tx_packets=$(cat "/sys/class/net/$tap_if/statistics/tx_packets" 2>/dev/null || echo 0) + fi + + # Write stats JSON + cat > "$stats_file" << EOF +{ + "timestamp": "$(date -Iseconds)", + "flows_1min": $total_flows, + "tap_interface": "$tap_if", + "rx_bytes": $rx_bytes, + "tx_bytes": $tx_bytes, + "rx_packets": $rx_packets, + "tx_packets": $tx_packets, + "protocols": $protocols, + "mode": "passive_tap" +} +EOF +} + +cleanup_old_flows() { + # Remove flow files older than retention period + local retention_min=$((RETENTION / 60)) + [ "$retention_min" -lt 1 ] && retention_min=1 + + find "$FLOW_DIR" -name "*.json" -mmin +"$retention_min" -delete 2>/dev/null +} + +run_collector() { + load_config + init_dirs + + echo "DPI Flow Collector started" + echo " Flow dir: $FLOW_DIR" + echo " Stats dir: $STATS_DIR" + echo " Retention: ${RETENTION}s" + + while true; do + collect_flows + cleanup_old_flows + sleep 10 + done +} + +case "$1" in + start) + run_collector + ;; + once) + load_config + init_dirs + collect_flows + cleanup_old_flows + ;; + *) + echo "Usage: $0 {start|once}" + exit 1 + ;; +esac