feat(dpi): Phase 4 - Integration, documentation and widget

- Add settings.js LuCI view for full UCI configuration - Add widget.js embeddable component for other dashboards - Add comprehensive README.md with architecture diagram - Add luci-app-dpi-dual entry to SecuBox catalog - Update menu.d to include Settings tab Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-03-15 12:31:47 +01:00 · 2026-03-15 12:31:47 +01:00 · b767f4dc30
commit b767f4dc30
parent 7ff9ee3805
5 changed files with 645 additions and 0 deletions
--- a/package/secubox/luci-app-dpi-dual/htdocs/luci-static/resources/dpi-dual/widget.js
+++ b/package/secubox/luci-app-dpi-dual/htdocs/luci-static/resources/dpi-dual/widget.js
@ -0,0 +1,154 @@
+'use strict';
+'require rpc';
+'require dom';
+
+/**
+ * DPI Dual-Stream Widget
+ * Embeddable status widget for use in other dashboards
+ *
+ * Usage:
+ *   'require dpi-dual/widget';
+ *   widget.render().then(function(el) { container.appendChild(el); });
+ */
+
+var callStatus = rpc.declare({
+    object: 'luci.dpi-dual',
+    method: 'status',
+    expect: {}
+});
+
+var callGetCorrelationStats = rpc.declare({
+    object: 'luci.dpi-dual',
+    method: 'get_correlation_stats',
+    expect: {}
+});
+
+function formatBytes(bytes) {
+    if (bytes === 0) return '0 B';
+    var k = 1024;
+    var sizes = ['B', 'KB', 'MB', 'GB'];
+    var i = Math.floor(Math.log(bytes) / Math.log(k));
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i];
+}
+
+function createLED(running, label) {
+    var color = running ? '#00d4aa' : '#ff4d4d';
+    return E('div', { 'style': 'display:flex;align-items:center;gap:6px;' }, [
+        E('span', {
+            'style': 'width:10px;height:10px;border-radius:50%;background:' + color +
+                     ';box-shadow:0 0 6px ' + color + ';'
+        }),
+        E('span', { 'style': 'font-size:0.85rem;color:#e0e0e0;' }, label)
+    ]);
+}
+
+function createMetric(label, value, color) {
+    return E('div', {
+        'style': 'text-align:center;min-width:60px;'
+    }, [
+        E('div', {
+            'style': 'font-size:1.2rem;font-weight:700;color:' + (color || '#00d4aa') + ';font-family:monospace;'
+        }, String(value)),
+        E('div', {
+            'style': 'font-size:0.65rem;color:#808090;text-transform:uppercase;'
+        }, label)
+    ]);
+}
+
+return {
+    /**
+     * Render a compact DPI status widget
+     * @returns {Promise<Element>} Widget element
+     */
+    render: function() {
+        return Promise.all([
+            callStatus().catch(function() { return {}; }),
+            callGetCorrelationStats().catch(function() { return {}; })
+        ]).then(function(data) {
+            var status = data[0] || {};
+            var stats = data[1] || {};
+
+            var mitm = status.mitm_stream || {};
+            var tap = status.tap_stream || {};
+            var corr = status.correlation || {};
+
+            return E('div', {
+                'class': 'dpi-dual-widget',
+                'style': 'background:#12121a;border-radius:10px;padding:1rem;border-left:4px solid #00a0ff;'
+            }, [
+                // Header
+                E('div', {
+                    'style': 'display:flex;justify-content:space-between;align-items:center;margin-bottom:0.8rem;'
+                }, [
+                    E('div', { 'style': 'display:flex;align-items:center;gap:8px;' }, [
+                        E('span', { 'style': 'font-size:1.2rem;' }, '📡'),
+                        E('span', { 'style': 'font-weight:600;color:#fff;' }, 'DPI Dual-Stream')
+                    ]),
+                    E('a', {
+                        'href': L.url('admin/secubox/dpi-dual'),
+                        'style': 'color:#00a0ff;font-size:0.8rem;text-decoration:none;'
+                    }, 'Details →')
+                ]),
+
+                // Status LEDs
+                E('div', {
+                    'style': 'display:flex;gap:1rem;margin-bottom:0.8rem;flex-wrap:wrap;'
+                }, [
+                    createLED(mitm.running, 'MITM'),
+                    createLED(tap.running, 'TAP'),
+                    createLED(corr.running, 'Correlation')
+                ]),
+
+                // Metrics
+                E('div', {
+                    'style': 'display:flex;gap:1rem;flex-wrap:wrap;background:#1a1a24;padding:0.6rem;border-radius:6px;'
+                }, [
+                    createMetric('Buffer', mitm.buffer_entries || 0, '#00d4aa'),
+                    createMetric('Threats', mitm.threats_detected || 0, '#ffa500'),
+                    createMetric('Flows', tap.flows_1min || 0, '#00a0ff'),
+                    createMetric('Correlated', stats.total_correlations || 0, '#ff6b6b'),
+                    createMetric('Banned', stats.banned_ips || 0, '#ff4d4d')
+                ])
+            ]);
+        });
+    },
+
+    /**
+     * Render a minimal status line
+     * @returns {Promise<Element>} Status line element
+     */
+    renderCompact: function() {
+        return callStatus().then(function(status) {
+            var mitm = status.mitm_stream || {};
+            var tap = status.tap_stream || {};
+
+            var allRunning = mitm.running && tap.running;
+            var color = allRunning ? '#00d4aa' : '#ffa500';
+
+            return E('span', {
+                'style': 'display:inline-flex;align-items:center;gap:6px;'
+            }, [
+                E('span', {
+                    'style': 'width:8px;height:8px;border-radius:50%;background:' + color + ';'
+                }),
+                E('span', {}, 'DPI: ' + (mitm.threats_detected || 0) + ' threats')
+            ]);
+        });
+    },
+
+    /**
+     * Get raw status data
+     * @returns {Promise<Object>} Status object
+     */
+    getStatus: function() {
+        return callStatus();
+    },
+
+    /**
+     * Get correlation statistics
+     * @returns {Promise<Object>} Stats object
+     */
+    getStats: function() {
+        return callGetCorrelationStats();
+    }
+};
--- a/package/secubox/luci-app-dpi-dual/htdocs/luci-static/resources/view/dpi-dual/settings.js
+++ b/package/secubox/luci-app-dpi-dual/htdocs/luci-static/resources/view/dpi-dual/settings.js
@ -0,0 +1,198 @@
+'use strict';
+'require view';
+'require form';
+'require uci';
+'require rpc';
+'require ui';
+
+var callStatus = rpc.declare({
+    object: 'luci.dpi-dual',
+    method: 'status',
+    expect: {}
+});
+
+return view.extend({
+    load: function() {
+        return Promise.all([
+            uci.load('dpi-dual'),
+            callStatus().catch(function() { return {}; })
+        ]);
+    },
+
+    render: function(data) {
+        var status = data[1] || {};
+        var m, s, o;
+
+        m = new form.Map('dpi-dual', 'DPI Dual-Stream Settings',
+            'Configure the dual-stream deep packet inspection architecture. ' +
+            'MITM stream provides active inspection with SSL termination, ' +
+            'while TAP stream offers passive analysis with zero latency impact.');
+
+        // Status section (read-only)
+        s = m.section(form.NamedSection, 'settings', 'global', 'Current Status');
+        s.anonymous = true;
+
+        o = s.option(form.DummyValue, '_status', 'Service Status');
+        o.rawhtml = true;
+        o.cfgvalue = function() {
+            var mitm = status.mitm_stream || {};
+            var tap = status.tap_stream || {};
+            var corr = status.correlation || {};
+
+            var mitmColor = mitm.running ? '#00d4aa' : '#ff4d4d';
+            var tapColor = tap.running ? '#00d4aa' : '#ff4d4d';
+            var corrColor = corr.running ? '#00d4aa' : '#ff4d4d';
+
+            return '<div style="display:flex;gap:2rem;flex-wrap:wrap;">' +
+                '<div><span style="color:' + mitmColor + ';font-weight:600;">●</span> MITM: ' +
+                    (mitm.running ? 'Running' : 'Stopped') + '</div>' +
+                '<div><span style="color:' + tapColor + ';font-weight:600;">●</span> TAP: ' +
+                    (tap.running ? 'Running' : 'Stopped') + '</div>' +
+                '<div><span style="color:' + corrColor + ';font-weight:600;">●</span> Correlation: ' +
+                    (corr.running ? 'Running' : 'Stopped') + '</div>' +
+                '</div>';
+        };
+
+        // Global settings
+        s = m.section(form.NamedSection, 'settings', 'global', 'Global Settings');
+        s.anonymous = true;
+        s.addremove = false;
+
+        o = s.option(form.Flag, 'enabled', 'Enable DPI',
+            'Master switch for the dual-stream DPI system');
+        o.default = '1';
+        o.rmempty = false;
+
+        o = s.option(form.ListValue, 'mode', 'Operating Mode',
+            'Select which streams to activate');
+        o.value('dual', 'Dual Stream (MITM + TAP)');
+        o.value('mitm-only', 'MITM Only (Active inspection)');
+        o.value('tap-only', 'TAP Only (Passive analysis)');
+        o.default = 'dual';
+
+        o = s.option(form.Flag, 'correlation', 'Enable Correlation',
+            'Match events from both streams for unified threat analytics');
+        o.default = '1';
+
+        o = s.option(form.Value, 'stats_dir', 'Stats Directory',
+            'Directory for statistics and cache files');
+        o.default = '/tmp/secubox';
+        o.placeholder = '/tmp/secubox';
+
+        o = s.option(form.Value, 'flow_dir', 'Flow Directory',
+            'Directory for netifyd flow files');
+        o.default = '/tmp/dpi-flows';
+        o.placeholder = '/tmp/dpi-flows';
+
+        // MITM settings
+        s = m.section(form.NamedSection, 'mitm', 'mitm', 'MITM Stream Settings');
+        s.anonymous = true;
+        s.addremove = false;
+
+        o = s.option(form.Flag, 'enabled', 'Enable MITM Stream',
+            'Active inspection with SSL termination via mitmproxy');
+        o.default = '1';
+
+        o = s.option(form.Value, 'buffer_size', 'Buffer Size',
+            'Number of requests to keep in the double buffer');
+        o.default = '1000';
+        o.datatype = 'uinteger';
+        o.placeholder = '1000';
+
+        o = s.option(form.Flag, 'async_analysis', 'Async Analysis',
+            'Enable asynchronous threat analysis (non-blocking)');
+        o.default = '1';
+
+        o = s.option(form.Flag, 'replay_on_alert', 'Replay on Alert',
+            'Replay buffered requests when CrowdSec alert is triggered');
+        o.default = '1';
+
+        o = s.option(form.Value, 'buffer_dir', 'Buffer Directory',
+            'Directory for request buffer files');
+        o.default = '/tmp/dpi-buffer';
+        o.placeholder = '/tmp/dpi-buffer';
+
+        // TAP settings
+        s = m.section(form.NamedSection, 'tap', 'tap', 'TAP Stream Settings');
+        s.anonymous = true;
+        s.addremove = false;
+
+        o = s.option(form.Flag, 'enabled', 'Enable TAP Stream',
+            'Passive traffic analysis via port mirroring');
+        o.default = '1';
+
+        o = s.option(form.Value, 'interface', 'TAP Interface',
+            'Virtual interface for mirrored traffic');
+        o.default = 'tap0';
+        o.placeholder = 'tap0';
+
+        o = s.option(form.Value, 'mirror_source', 'Mirror Source',
+            'Interface to mirror traffic from (usually WAN)');
+        o.default = 'eth0';
+        o.placeholder = 'eth0';
+
+        o = s.option(form.ListValue, 'mirror_mode', 'Mirror Mode',
+            'How to capture traffic for analysis');
+        o.value('software', 'Software (tc mirred)');
+        o.value('hardware', 'Hardware (switch port mirror)');
+        o.default = 'software';
+
+        o = s.option(form.Value, 'flow_retention', 'Flow Retention (seconds)',
+            'How long to keep flow files before cleanup');
+        o.default = '300';
+        o.datatype = 'uinteger';
+        o.placeholder = '300';
+
+        o = s.option(form.Value, 'netifyd_instance', 'Netifyd Instance',
+            'Name of the netifyd instance for this TAP');
+        o.default = 'tap';
+        o.placeholder = 'tap';
+
+        // Correlation settings
+        s = m.section(form.NamedSection, 'correlation', 'correlation', 'Correlation Engine Settings');
+        s.anonymous = true;
+        s.addremove = false;
+
+        o = s.option(form.Flag, 'enabled', 'Enable Correlation',
+            'Match events across MITM and TAP streams');
+        o.default = '1';
+
+        o = s.option(form.Value, 'window', 'Correlation Window (seconds)',
+            'Time window for matching related events');
+        o.default = '60';
+        o.datatype = 'uinteger';
+        o.placeholder = '60';
+
+        o = s.option(form.Flag, 'watch_crowdsec', 'Watch CrowdSec',
+            'Monitor CrowdSec decisions for correlation');
+        o.default = '1';
+
+        o = s.option(form.Flag, 'auto_ban', 'Auto-Ban',
+            'Automatically ban IPs exceeding reputation threshold');
+        o.default = '0';
+
+        o = s.option(form.Value, 'auto_ban_threshold', 'Auto-Ban Threshold',
+            'Reputation score threshold for automatic banning (0-100)');
+        o.default = '80';
+        o.datatype = 'range(0,100)';
+        o.placeholder = '80';
+        o.depends('auto_ban', '1');
+
+        o = s.option(form.Flag, 'notifications', 'Enable Notifications',
+            'Send notifications for high-severity threats');
+        o.default = '1';
+
+        o = s.option(form.Value, 'reputation_decay', 'Reputation Decay',
+            'Points to decay from reputation score per hour');
+        o.default = '5';
+        o.datatype = 'uinteger';
+        o.placeholder = '5';
+
+        o = s.option(form.Value, 'output', 'Correlation Output',
+            'File path for correlated threats log');
+        o.default = '/tmp/secubox/correlated-threats.json';
+        o.placeholder = '/tmp/secubox/correlated-threats.json';
+
+        return m.render();
+    }
+});
--- a/package/secubox/luci-app-dpi-dual/root/usr/share/luci/menu.d/luci-app-dpi-dual.json
+++ b/package/secubox/luci-app-dpi-dual/root/usr/share/luci/menu.d/luci-app-dpi-dual.json
@ -26,5 +26,13 @@
 			"type": "view",
 			"path": "dpi-dual/timeline"
 		}
+	},
+	"admin/secubox/dpi-dual/settings": {
+		"title": "Settings",
+		"order": 3,
+		"action": {
+			"type": "view",
+			"path": "dpi-dual/settings"
+		}
 	}
 }
--- a/package/secubox/secubox-core/root/usr/share/secubox/catalog.json
+++ b/package/secubox/secubox-core/root/usr/share/secubox/catalog.json
@ -2636,6 +2636,87 @@
          }
        ]
      }
+    },
+    {
+      "id": "luci-app-dpi-dual",
+      "name": "DPI Dual-Stream",
+      "version": "1.0.0",
+      "category": "security",
+      "runtime": "native",
+      "description": "Dual-stream Deep Packet Inspection with parallel MITM active inspection and passive TAP analysis for comprehensive network security",
+      "author": "CyberMind.fr",
+      "license": "GPL-3.0",
+      "icon": "📡",
+      "featured": true,
+      "featured_priority": 4,
+      "featured_category": "security",
+      "featured_reason": "Advanced DPI with MITM double buffer and passive TAP analysis",
+      "badges": ["new", "recommended"],
+      "tags": [
+        "dpi",
+        "deep-packet-inspection",
+        "mitm",
+        "tap",
+        "security",
+        "waf",
+        "correlation",
+        "threat-detection"
+      ],
+      "packages": {
+        "required": [
+          "luci-app-dpi-dual",
+          "secubox-dpi-dual",
+          "netifyd",
+          "iproute2-tc"
+        ]
+      },
+      "capabilities": [
+        "deep-packet-inspection",
+        "threat-correlation",
+        "ip-reputation",
+        "waf-integration"
+      ],
+      "requirements": {
+        "min_ram_mb": 256,
+        "min_storage_mb": 50
+      },
+      "status": "stable",
+      "notes": "Combines active MITM inspection (HAProxy + mitmproxy) with passive TAP analysis (tc mirred + netifyd). Features IP reputation, auto-ban, CrowdSec integration.",
+      "pkg_version": "1.0.0-1",
+      "app_version": "1.0.0",
+      "changelog": {
+        "1.0.0": {
+          "date": "2026-03-15",
+          "changes": [
+            "Initial release",
+            "MITM double buffer with threat pattern detection",
+            "TAP stream with tc mirred port mirroring",
+            "Correlation engine with IP reputation",
+            "LuCI dashboard with timeline view",
+            "CrowdSec integration and auto-ban"
+          ]
+        }
+      },
+      "widget": {
+        "enabled": true,
+        "template": "security-widget",
+        "refresh_interval": 10,
+        "metrics": [
+          {
+            "id": "threats_detected",
+            "label": "Threats Detected",
+            "type": "counter",
+            "source": "ubus",
+            "method": "luci.dpi-dual.status"
+          },
+          {
+            "id": "correlated_events",
+            "label": "Correlated Events",
+            "type": "counter",
+            "source": "ubus"
+          }
+        ]
+      }
    }
  ],
  "featured_sections": {
--- a/package/secubox/secubox-dpi-dual/README.md
+++ b/package/secubox/secubox-dpi-dual/README.md
@ -0,0 +1,204 @@
+# SecuBox DPI Dual-Stream
+
+Dual-stream Deep Packet Inspection architecture combining active MITM inspection with passive TAP analysis for comprehensive network security.
+
+## Architecture
+
+```
+                    ┌─────────────────────────────────────┐
+                    │            WAN INTERFACE            │
+                    └─────────────────┬───────────────────┘
+                                      │
+           ┌──────────────────────────┼──────────────────────────┐
+           │                          │                          │
+           ▼                          ▼                          │
+ ┌─────────────────────┐    ┌─────────────────────┐              │
+ │   STREAM 1: MITM    │    │  STREAM 2: TAP/DPI  │              │
+ │   (Active Path)     │    │  (Passive Mirror)   │              │
+ └─────────┬───────────┘    └─────────┬───────────┘              │
+           │                          │                          │
+           ▼                          ▼                          │
+ ┌─────────────────────┐    ┌─────────────────────┐              │
+ │  HAProxy + MITM     │    │   tc mirred/TAP     │              │
+ │  (SSL Termination)  │    │  (Port Mirroring)   │              │
+ └─────────┬───────────┘    └─────────┬───────────┘              │
+           │                          │                          │
+           ▼                          ▼                          │
+ ┌─────────────────────┐    ┌─────────────────────┐              │
+ │   Double Buffer     │    │     netifyd         │              │
+ │  (Async Analysis)   │    │   (nDPI Engine)     │              │
+ └─────────┬───────────┘    └─────────┬───────────┘              │
+           │                          │                          │
+           └──────────────┬───────────┘                          │
+                          │                                      │
+                          ▼                                      │
+           ┌─────────────────────────────────────┐               │
+           │        CORRELATION ENGINE           │               │
+           │  (IP Reputation + Context Match)    │               │
+           └─────────────────────────────────────┘               │
+```
+
+## Features
+
+### Stream 1: MITM (Active Inspection)
+- Full content inspection with SSL/TLS termination
+- WAF rule enforcement via mitmproxy
+- Double-buffered request analysis
+- Threat pattern detection (XSS, SQLi, LFI, RCE, SSRF, Path Traversal)
+- Scanner detection (sqlmap, nikto, nmap, etc.)
+- Optional request blocking for high-score threats
+
+### Stream 2: TAP (Passive Analysis)
+- Zero latency impact on live traffic
+- Protocol identification via nDPI (300+ protocols)
+- Flow statistics and bandwidth analysis
+- Works with encrypted traffic (metadata analysis)
+- Software (tc mirred) or hardware port mirroring
+
+### Correlation Engine
+- IP reputation tracking with score decay
+- Event matching across both streams
+- CrowdSec integration (decision watching, auto-ban)
+- Full context gathering (MITM requests, WAF alerts, DPI flows)
+- High-severity threat notifications
+
+## Installation
+
+```bash
+opkg update
+opkg install secubox-dpi-dual luci-app-dpi-dual
+```
+
+## CLI Usage
+
+```bash
+# Start/Stop/Restart
+dpi-dualctl start
+dpi-dualctl stop
+dpi-dualctl restart
+
+# Check status
+dpi-dualctl status
+
+# View flow statistics
+dpi-dualctl flows
+
+# View recent threats
+dpi-dualctl threats 20
+
+# Mirror control
+dpi-dualctl mirror status
+dpi-dualctl mirror start
+dpi-dualctl mirror stop
+```
+
+### Correlator Commands
+
+```bash
+# Manual correlation
+dpi-correlator correlate 192.168.1.100 waf_alert "suspicious_request" 75
+
+# Get IP reputation
+dpi-correlator reputation 192.168.1.100
+
+# Get full context for IP
+dpi-correlator context 192.168.1.100
+
+# Search correlations
+dpi-correlator search 192.168.1.100 50
+
+# Show stats
+dpi-correlator stats
+```
+
+## Configuration
+
+Edit `/etc/config/dpi-dual`:
+
+```
+config global 'settings'
+    option enabled '1'
+    option mode 'dual'           # dual|mitm-only|tap-only
+    option correlation '1'
+
+config mitm 'mitm'
+    option enabled '1'
+    option buffer_size '1000'    # requests in double buffer
+    option async_analysis '1'
+
+config tap 'tap'
+    option enabled '1'
+    option interface 'tap0'
+    option mirror_source 'eth0'
+    option mirror_mode 'software' # software|hardware
+
+config correlation 'correlation'
+    option enabled '1'
+    option watch_crowdsec '1'
+    option auto_ban '0'
+    option auto_ban_threshold '80'
+    option notifications '1'
+```
+
+## LuCI Dashboard
+
+Navigate to **SecuBox → DPI Dual-Stream**:
+
+- **Overview**: Stream status, metrics, threats table
+- **Correlation Timeline**: Event cards with IP context
+- **Settings**: Full configuration interface
+
+## Files
+
+| File | Purpose |
+|------|---------|
+| `/usr/sbin/dpi-dualctl` | Main CLI tool |
+| `/usr/sbin/dpi-flow-collector` | Flow aggregation service |
+| `/usr/sbin/dpi-correlator` | Correlation engine |
+| `/usr/lib/dpi-dual/mirror-setup.sh` | tc mirred port mirroring |
+| `/usr/lib/dpi-dual/correlation-lib.sh` | Shared correlation functions |
+| `/srv/mitmproxy/addons/dpi_buffer.py` | mitmproxy double buffer addon |
+| `/etc/config/dpi-dual` | UCI configuration |
+| `/etc/init.d/dpi-dual` | procd service |
+
+## Output Files
+
+| File | Content |
+|------|---------|
+| `/tmp/secubox/dpi-flows.json` | Flow statistics from TAP stream |
+| `/tmp/secubox/dpi-buffer.json` | Buffer statistics from MITM |
+| `/tmp/secubox/waf-alerts.json` | WAF threat alerts |
+| `/tmp/secubox/correlated-threats.json` | Correlated threat log (JSONL) |
+| `/tmp/secubox/ip-reputation.json` | IP reputation database |
+| `/tmp/secubox/notifications.json` | High-severity threat notifications |
+
+## Dependencies
+
+- `netifyd` - nDPI-based flow analyzer
+- `iproute2-tc` - Traffic control for port mirroring
+- `jsonfilter` - JSON parsing (libubox)
+- `coreutils-stat` - File statistics
+
+## Performance
+
+| Aspect | MITM Stream | TAP Stream |
+|--------|-------------|------------|
+| Latency | +5-20ms | 0ms |
+| CPU | High (SSL, WAF) | Low (nDPI) |
+| Memory | Buffer dependent | Minimal |
+| Visibility | Full content | Metadata only |
+
+## Security Notes
+
+1. **TAP stream is read-only** — cannot block, only observe
+2. **MITM stream requires CA trust** — users must accept certificate
+3. **Buffer data is sensitive** — limited retention, auto-cleanup
+4. **Correlation logs contain PII** — follow data protection regulations
+
+## License
+
+GPL-3.0
+
+## Author
+
+SecuBox Team <secubox@gk2.net>