From 30926404dcaeaba0748a5591d271e31b7da5be53 Mon Sep 17 00:00:00 2001 From: CyberMind-FR Date: Fri, 9 Jan 2026 13:51:15 +0100 Subject: [PATCH] feat(media-flow): Add nDPId integration for local DPI streaming detection - Add media-flow-ndpid-collector script for collecting streaming data from nDPId - Update RPCD backend to detect and use nDPId as primary DPI source - Update frontend dashboard to show DPI source indicator (nDPId/netifyd/none) - Add active streams table displaying real-time streaming activity - Update init.d script to auto-detect and use best available collector - Remove hard dependency on netifyd, make DPI engines optional - Bump version to 0.6.0 nDPId provides local deep packet inspection without requiring cloud subscription, enabling accurate streaming service detection (Netflix, YouTube, Spotify, etc.) with quality estimation. Co-Authored-By: Claude Opus 4.5 --- package/secubox/luci-app-media-flow/Makefile | 9 +- .../resources/view/media-flow/dashboard.js | 176 +++++++++++++--- .../root/etc/init.d/media-flow | 56 ++++- .../root/usr/bin/media-flow-ndpid-collector | 194 ++++++++++++++++++ .../root/usr/libexec/rpcd/luci.media-flow | 111 ++++++++-- 5 files changed, 499 insertions(+), 47 deletions(-) create mode 100644 package/secubox/luci-app-media-flow/root/usr/bin/media-flow-ndpid-collector diff --git a/package/secubox/luci-app-media-flow/Makefile b/package/secubox/luci-app-media-flow/Makefile index f6568f4c..2efe2452 100644 --- a/package/secubox/luci-app-media-flow/Makefile +++ b/package/secubox/luci-app-media-flow/Makefile @@ -11,20 +11,25 @@ PKG_LICENSE:=Apache-2.0 PKG_MAINTAINER:=CyberMind LUCI_TITLE:=Media Flow - Streaming Detection & Monitoring -LUCI_DESCRIPTION:=Real-time detection and monitoring of streaming services (Netflix, YouTube, Spotify, etc.) with quality estimation, history tracking, and alerts -LUCI_DEPENDS:=+luci-base +rpcd +netifyd +jq +LUCI_DESCRIPTION:=Real-time detection and monitoring of streaming services (Netflix, YouTube, Spotify, etc.) with quality estimation, history tracking, and alerts. Supports nDPId local DPI and netifyd. +LUCI_DEPENDS:=+luci-base +rpcd +jq LUCI_PKGARCH:=all +# Optional dependencies (nDPId for local DPI, netifyd as fallback) +# +ndpid provides local application detection without cloud subscription +# +netifyd requires cloud subscription for application detection # File permissions (CRITICAL: RPCD scripts MUST be executable 755) PKG_FILE_MODES:=/usr/libexec/rpcd/luci.media-flow:root:root:755 \ /usr/bin/media-flow-collector:root:root:755 \ + /usr/bin/media-flow-ndpid-collector:root:root:755 \ /etc/init.d/media-flow:root:root:755 define Package/$(PKG_NAME)/install $(call Package/luci-app-media-flow/install,$(1)) $(INSTALL_DIR) $(1)/usr/bin $(INSTALL_BIN) ./root/usr/bin/media-flow-collector $(1)/usr/bin/ + $(INSTALL_BIN) ./root/usr/bin/media-flow-ndpid-collector $(1)/usr/bin/ $(INSTALL_DIR) $(1)/etc/init.d $(INSTALL_BIN) ./root/etc/init.d/media-flow $(1)/etc/init.d/ endef diff --git a/package/secubox/luci-app-media-flow/htdocs/luci-static/resources/view/media-flow/dashboard.js b/package/secubox/luci-app-media-flow/htdocs/luci-static/resources/view/media-flow/dashboard.js index 0eb588ae..07204249 100644 --- a/package/secubox/luci-app-media-flow/htdocs/luci-static/resources/view/media-flow/dashboard.js +++ b/package/secubox/luci-app-media-flow/htdocs/luci-static/resources/view/media-flow/dashboard.js @@ -21,46 +21,170 @@ return L.view.extend({ var v = E('div', { 'class': 'cbi-map' }, [ E('link', { 'rel': 'stylesheet', 'href': L.resource('secubox-theme/secubox-theme.css') }), E('h2', {}, _('Media Flow Dashboard')), - E('div', { 'class': 'cbi-map-descr' }, _('Network flow monitoring and statistics')) + E('div', { 'class': 'cbi-map-descr' }, _('Streaming service detection and network flow monitoring')) ]); - // Status overview + // DPI Source indicator + var dpiSource = status.dpi_source || 'none'; + var dpiColor = dpiSource === 'ndpid' ? '#00cc88' : (dpiSource === 'netifyd' ? '#0088cc' : '#cc0000'); + var dpiLabel = dpiSource === 'ndpid' ? 'nDPId (Local DPI)' : (dpiSource === 'netifyd' ? 'Netifyd' : 'No DPI Engine'); + + // Status overview with DPI source var statusSection = E('div', { 'class': 'cbi-section' }, [ E('h3', {}, _('Status')), E('div', { 'class': 'table' }, [ E('div', { 'class': 'tr' }, [ - E('div', { 'class': 'td left', 'width': '33%' }, [ + E('div', { 'class': 'td left', 'width': '25%' }, [ E('strong', {}, _('Module: ')), E('span', {}, status.enabled ? _('Enabled') : _('Disabled')) ]), - E('div', { 'class': 'td left', 'width': '33%' }, [ + E('div', { 'class': 'td left', 'width': '25%' }, [ + E('strong', {}, _('DPI Source: ')), + E('span', { 'style': 'color: ' + dpiColor + '; font-weight: bold;' }, '● ' + dpiLabel) + ]), + E('div', { 'class': 'td left', 'width': '25%' }, [ + E('strong', {}, _('Active Flows: ')), + E('span', { 'style': 'font-size: 1.3em; color: #0088cc' }, String(status.active_flows || 0)) + ]), + E('div', { 'class': 'td left', 'width': '25%' }, [ + E('strong', {}, _('Active Streams: ')), + E('span', { 'style': 'font-size: 1.3em; color: #ec4899' }, String(status.active_streams || 0)) + ]) + ]), + E('div', { 'class': 'tr' }, [ + E('div', { 'class': 'td left', 'width': '25%' }, [ + E('strong', {}, _('nDPId: ')), + E('span', {}, status.ndpid_running ? + E('span', { 'style': 'color: green' }, '● ' + _('Running') + (status.ndpid_version !== 'unknown' ? ' (v' + status.ndpid_version + ')' : '')) : + E('span', { 'style': 'color: #999' }, '○ ' + _('Not running')) + ) + ]), + E('div', { 'class': 'td left', 'width': '25%' }, [ E('strong', {}, _('Netifyd: ')), E('span', {}, status.netifyd_running ? E('span', { 'style': 'color: green' }, '● ' + _('Running') + ' (v' + (status.netifyd_version || '?') + ')') : - E('span', { 'style': 'color: red' }, '● ' + _('Stopped')) + E('span', { 'style': 'color: #999' }, '○ ' + _('Not running')) ) ]), - E('div', { 'class': 'td left', 'width': '33%' }, [ - E('strong', {}, _('Active Flows: ')), - E('span', { 'style': 'font-size: 1.5em; color: #0088cc' }, String(status.active_flows || 0)) + E('div', { 'class': 'td left', 'width': '25%' }, [ + E('strong', {}, _('nDPId Flows: ')), + E('span', {}, String(status.ndpid_flows || 0)) + ]), + E('div', { 'class': 'td left', 'width': '25%' }, [ + E('strong', {}, _('History: ')), + E('span', {}, String(status.history_entries || 0) + ' entries') ]) ]) ]) ]); v.appendChild(statusSection); - // Netifyd 5.x limitation notice - var noticeSection = E('div', { 'class': 'cbi-section' }, [ - E('div', { 'class': 'alert-message warning', 'style': 'background: #fff3cd; border: 1px solid #ffc107; padding: 15px; border-radius: 4px; margin-bottom: 15px;' }, [ - E('strong', {}, _('Notice: ')), - E('span', {}, _('Netifyd 5.x requires a cloud subscription for streaming service detection. Currently showing network flow statistics only.')) - ]) - ]); + // Info notice based on DPI source + var noticeSection; + if (dpiSource === 'ndpid') { + noticeSection = E('div', { 'class': 'cbi-section' }, [ + E('div', { 'style': 'background: #d4edda; border: 1px solid #28a745; padding: 15px; border-radius: 4px; margin-bottom: 15px;' }, [ + E('strong', {}, _('nDPId Active: ')), + E('span', {}, _('Using local deep packet inspection for streaming detection. No cloud subscription required.')) + ]) + ]); + } else if (dpiSource === 'netifyd') { + noticeSection = E('div', { 'class': 'cbi-section' }, [ + E('div', { 'style': 'background: #fff3cd; border: 1px solid #ffc107; padding: 15px; border-radius: 4px; margin-bottom: 15px;' }, [ + E('strong', {}, _('Notice: ')), + E('span', {}, _('Netifyd 5.x requires a cloud subscription for streaming service detection. Install nDPId for local detection.')) + ]) + ]); + } else { + noticeSection = E('div', { 'class': 'cbi-section' }, [ + E('div', { 'style': 'background: #f8d7da; border: 1px solid #dc3545; padding: 15px; border-radius: 4px; margin-bottom: 15px;' }, [ + E('strong', {}, _('No DPI Engine: ')), + E('span', {}, _('Install nDPId or netifyd for streaming detection capabilities.')) + ]) + ]); + } v.appendChild(noticeSection); + // Active Streams section (when nDPId provides data) + var streamsSection = E('div', { 'class': 'cbi-section' }, [ + E('h3', {}, _('Active Streams')), + E('div', { 'id': 'active-streams-container' }) + ]); + + var categoryIcons = { + 'video': String.fromCodePoint(0x1F3AC), + 'audio': String.fromCodePoint(0x1F3B5), + 'visio': String.fromCodePoint(0x1F4F9) + }; + + var qualityColors = { + '4K': '#9333ea', + 'FHD': '#2563eb', + 'HD': '#059669', + 'SD': '#d97706', + 'Lossless': '#9333ea', + 'High': '#2563eb', + 'Normal': '#059669', + 'Low': '#d97706', + 'Audio Only': '#6b7280' + }; + + var renderActiveStreams = function(streams, dpiSrc) { + var container = document.getElementById('active-streams-container'); + if (!container) return; + + container.innerHTML = ''; + + if (!streams || streams.length === 0) { + container.appendChild(E('div', { 'style': 'text-align: center; padding: 30px; color: #666;' }, [ + E('div', { 'style': 'font-size: 3em; margin-bottom: 10px;' }, String.fromCodePoint(0x1F4E1)), + E('p', {}, dpiSrc === 'ndpid' ? _('No streaming activity detected') : _('Waiting for streaming data...')) + ])); + return; + } + + var table = E('table', { 'class': 'table', 'style': 'width: 100%;' }, [ + E('tr', { 'class': 'tr table-titles' }, [ + E('th', { 'class': 'th' }, _('Service')), + E('th', { 'class': 'th' }, _('Client')), + E('th', { 'class': 'th' }, _('Category')), + E('th', { 'class': 'th' }, _('Quality')), + E('th', { 'class': 'th' }, _('Bandwidth')), + E('th', { 'class': 'th' }, _('Data')) + ]) + ]); + + streams.slice(0, 20).forEach(function(stream) { + var icon = categoryIcons[stream.category] || String.fromCodePoint(0x1F4CA); + var qualityColor = qualityColors[stream.quality] || '#6b7280'; + var bytesTotal = (stream.bytes_rx || 0) + (stream.bytes_tx || 0); + var dataMB = (bytesTotal / 1048576).toFixed(1); + + table.appendChild(E('tr', { 'class': 'tr' }, [ + E('td', { 'class': 'td' }, [ + E('strong', {}, stream.app || 'Unknown') + ]), + E('td', { 'class': 'td' }, stream.client || '-'), + E('td', { 'class': 'td' }, icon + ' ' + (stream.category || 'other')), + E('td', { 'class': 'td' }, [ + E('span', { 'style': 'background: ' + qualityColor + '; color: white; padding: 2px 8px; border-radius: 4px; font-size: 0.85em;' }, + stream.quality || '-') + ]), + E('td', { 'class': 'td' }, (stream.bandwidth || 0) + ' kbps'), + E('td', { 'class': 'td' }, dataMB + ' MB') + ])); + }); + + container.appendChild(table); + }; + + // Initial render of streams + renderActiveStreams(streamsData.streams || [], dpiSource); + v.appendChild(streamsSection); + // Network flow stats var flowSection = E('div', { 'class': 'cbi-section' }, [ - E('h3', {}, _('Network Flows')), + E('h3', {}, _('Network Flow Statistics')), E('div', { 'id': 'flow-stats-container' }) ]); @@ -70,21 +194,27 @@ return L.view.extend({ if (!container) return; var flowCount = data.flow_count || 0; + var dpiSrc = data.dpi_source || 'none'; var note = data.note || ''; container.innerHTML = ''; - container.appendChild(E('div', { 'class': 'table', 'style': 'background: #f8f9fa; padding: 20px; border-radius: 8px;' }, [ - E('div', { 'class': 'tr' }, [ - E('div', { 'class': 'td', 'style': 'text-align: center;' }, [ - E('div', { 'style': 'font-size: 3em; color: #0088cc; font-weight: bold;' }, String(flowCount)), - E('div', { 'style': 'color: #666; margin-top: 5px;' }, _('Active Network Flows')) - ]) + container.appendChild(E('div', { 'style': 'display: flex; justify-content: space-around; background: #f8f9fa; padding: 20px; border-radius: 8px;' }, [ + E('div', { 'style': 'text-align: center;' }, [ + E('div', { 'style': 'font-size: 2.5em; color: #0088cc; font-weight: bold;' }, String(flowCount)), + E('div', { 'style': 'color: #666; margin-top: 5px;' }, _('Total Flows')) + ]), + E('div', { 'style': 'text-align: center;' }, [ + E('div', { 'style': 'font-size: 2.5em; color: #ec4899; font-weight: bold;' }, String((data.streams || []).length)), + E('div', { 'style': 'color: #666; margin-top: 5px;' }, _('Streaming Flows')) ]) ])); if (note) { container.appendChild(E('p', { 'style': 'font-style: italic; color: #666; text-align: center; margin-top: 10px;' }, note)); } + + // Update active streams table + renderActiveStreams(data.streams || [], dpiSrc); }); }; @@ -186,7 +316,7 @@ return L.view.extend({ var servicesList = Object.keys(services); if (servicesList.length === 0) { - container.appendChild(E('p', { 'style': 'font-style: italic' }, _('No historical data available. Stream detection requires netifyd cloud subscription.'))); + container.appendChild(E('p', { 'style': 'font-style: italic' }, _('No historical data available. Start streaming services while nDPId is running to collect data.'))); drawDonutChart({}, [], 0); return; } diff --git a/package/secubox/luci-app-media-flow/root/etc/init.d/media-flow b/package/secubox/luci-app-media-flow/root/etc/init.d/media-flow index 649f1a36..dbe4dd7c 100644 --- a/package/secubox/luci-app-media-flow/root/etc/init.d/media-flow +++ b/package/secubox/luci-app-media-flow/root/etc/init.d/media-flow @@ -1,37 +1,53 @@ #!/bin/sh /etc/rc.common # # Media Flow Init Script -# Manages the media flow data collector cron job +# Manages the media flow data collector cron jobs +# Supports nDPId (local DPI) and netifyd as data sources # START=99 STOP=10 CRON_FILE="/etc/crontabs/root" -CRON_ENTRY="*/5 * * * * /usr/bin/media-flow-collector >/dev/null 2>&1" CRON_MARKER="# media-flow-collector" +# Detect best collector to use +get_collector() { + # Prefer nDPId collector if nDPId is installed + if [ -x /usr/bin/media-flow-ndpid-collector ] && command -v ndpid >/dev/null 2>&1; then + echo "/usr/bin/media-flow-ndpid-collector" + else + echo "/usr/bin/media-flow-collector" + fi +} + add_cron_entry() { # Remove existing entries first remove_cron_entry + local collector=$(get_collector) + local cron_entry="*/1 * * * * $collector >/dev/null 2>&1" + # Add the new entry with marker if [ -f "$CRON_FILE" ]; then echo "$CRON_MARKER" >> "$CRON_FILE" - echo "$CRON_ENTRY" >> "$CRON_FILE" + echo "$cron_entry" >> "$CRON_FILE" else echo "$CRON_MARKER" > "$CRON_FILE" - echo "$CRON_ENTRY" >> "$CRON_FILE" + echo "$cron_entry" >> "$CRON_FILE" fi # Restart cron /etc/init.d/cron reload 2>/dev/null || /etc/init.d/cron restart 2>/dev/null + + logger -t media-flow "Collector enabled: $collector" } remove_cron_entry() { if [ -f "$CRON_FILE" ]; then sed -i '/# media-flow-collector/d' "$CRON_FILE" sed -i '\|/usr/bin/media-flow-collector|d' "$CRON_FILE" + sed -i '\|/usr/bin/media-flow-ndpid-collector|d' "$CRON_FILE" /etc/init.d/cron reload 2>/dev/null || /etc/init.d/cron restart 2>/dev/null fi } @@ -43,7 +59,8 @@ start() { logger -t media-flow "Starting media flow collector" add_cron_entry # Run once immediately - /usr/bin/media-flow-collector & + local collector=$(get_collector) + $collector & fi } @@ -66,17 +83,40 @@ reload() { status() { local enabled=$(uci -q get media_flow.global.enabled 2>/dev/null || echo "1") + local collector=$(get_collector) - if grep -q "media-flow-collector" "$CRON_FILE" 2>/dev/null; then - echo "Media Flow collector: ACTIVE" + echo "Media Flow v0.6.0" + echo "=================" + + if grep -q "media-flow" "$CRON_FILE" 2>/dev/null; then + echo "Collector: ACTIVE" else - echo "Media Flow collector: INACTIVE" + echo "Collector: INACTIVE" fi + echo "Using: $collector" echo "UCI enabled: $enabled" + # DPI engine status + if pgrep -x ndpid >/dev/null 2>&1; then + echo "nDPId: Running" + else + echo "nDPId: Not running" + fi + + if pgrep -x netifyd >/dev/null 2>&1; then + echo "Netifyd: Running" + else + echo "Netifyd: Not running" + fi + if [ -f /tmp/media-flow-history.json ]; then local count=$(jq 'length' /tmp/media-flow-history.json 2>/dev/null || echo 0) echo "History entries: $count" fi + + if [ -f /tmp/media-flow-ndpid-cache.json ]; then + local streams=$(jq 'length' /tmp/media-flow-ndpid-cache.json 2>/dev/null || echo 0) + echo "Active streams: $streams" + fi } diff --git a/package/secubox/luci-app-media-flow/root/usr/bin/media-flow-ndpid-collector b/package/secubox/luci-app-media-flow/root/usr/bin/media-flow-ndpid-collector new file mode 100644 index 00000000..3513a284 --- /dev/null +++ b/package/secubox/luci-app-media-flow/root/usr/bin/media-flow-ndpid-collector @@ -0,0 +1,194 @@ +#!/bin/sh +# +# Media Flow nDPId Collector +# Collects streaming service data from nDPId flows and stores in history +# Uses nDPId's local DPI detection (no cloud subscription required) +# +# Copyright (C) 2025 CyberMind.fr + +HISTORY_FILE="/tmp/media-flow-history.json" +NDPID_FLOWS="/tmp/ndpid-flows.json" +NDPID_APPS="/tmp/ndpid-apps.json" +MEDIA_CACHE="/tmp/media-flow-ndpid-cache.json" +MAX_ENTRIES=1000 +LOCK_FILE="/tmp/media-flow-ndpid-collector.lock" + +# Streaming services patterns for nDPId applications +# These match nDPId's application protocol names +STREAMING_VIDEO="YouTube|Netflix|Disney|AmazonVideo|PrimeVideo|Twitch|HboMax|Hulu|Vimeo|Peacock|Paramount|Crunchyroll|DailyMotion|Vevo|Plex|AppleTV" +STREAMING_AUDIO="Spotify|AppleMusic|Deezer|SoundCloud|Tidal|Pandora|AmazonMusic|YouTubeMusic|iHeartRadio|Audible" +STREAMING_VISIO="Zoom|Teams|GoogleMeet|Discord|Skype|Webex|FaceTime|WhatsApp|Signal|Telegram|Slack|GoToMeeting" +STREAMING_ALL="${STREAMING_VIDEO}|${STREAMING_AUDIO}|${STREAMING_VISIO}" + +# Check if already running +if [ -f "$LOCK_FILE" ]; then + pid=$(cat "$LOCK_FILE" 2>/dev/null) + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + exit 0 + fi +fi + +echo $$ > "$LOCK_FILE" +trap "rm -f $LOCK_FILE" EXIT + +# Check if enabled +enabled=$(uci -q get media_flow.global.enabled 2>/dev/null || echo "1") +[ "$enabled" != "1" ] && exit 0 + +# Check if nDPId data is available +if [ ! -f "$NDPID_FLOWS" ]; then + # Fall back to checking if ndpid is running + if ! pgrep -x ndpid > /dev/null 2>&1; then + exit 0 + fi +fi + +# Initialize history file +[ ! -f "$HISTORY_FILE" ] && echo '[]' > "$HISTORY_FILE" + +# Function to categorize streaming service +categorize_service() { + local app="$1" + if echo "$app" | grep -qiE "$STREAMING_VIDEO"; then + echo "video" + elif echo "$app" | grep -qiE "$STREAMING_AUDIO"; then + echo "audio" + elif echo "$app" | grep -qiE "$STREAMING_VISIO"; then + echo "visio" + else + echo "other" + fi +} + +# Function to estimate quality from bandwidth +estimate_quality() { + local kbps="$1" + local category="$2" + + if [ "$category" = "audio" ]; then + # Audio quality tiers (kbps) + if [ "$kbps" -lt 96 ]; then + echo "Low" + elif [ "$kbps" -lt 192 ]; then + echo "Normal" + elif [ "$kbps" -lt 320 ]; then + echo "High" + else + echo "Lossless" + fi + elif [ "$category" = "visio" ]; then + # Video call quality tiers (kbps) + if [ "$kbps" -lt 500 ]; then + echo "Audio Only" + elif [ "$kbps" -lt 1500 ]; then + echo "SD" + elif [ "$kbps" -lt 3000 ]; then + echo "HD" + else + echo "FHD" + fi + else + # Video streaming quality tiers (kbps) + if [ "$kbps" -lt 1000 ]; then + echo "SD" + elif [ "$kbps" -lt 3000 ]; then + echo "HD" + elif [ "$kbps" -lt 8000 ]; then + echo "FHD" + else + echo "4K" + fi + fi +} + +# Process nDPId flows +if [ -f "$NDPID_FLOWS" ] && command -v jq >/dev/null 2>&1; then + timestamp=$(date -Iseconds) + + # Extract streaming flows from nDPId data + new_entries=$(jq -c --arg ts "$timestamp" --arg pattern "$STREAMING_ALL" ' + [.[] | + select(.app != null and .app != "" and .app != "Unknown") | + select(.app | test($pattern; "i")) | + select(.state == "active" or .bytes_rx > 10000) | + { + timestamp: $ts, + app: .app, + client: (.src_ip // "unknown"), + server: (.dst_ip // "unknown"), + hostname: (.hostname // null), + protocol: (.proto // "unknown"), + bytes_rx: (.bytes_rx // 0), + bytes_tx: (.bytes_tx // 0), + packets: (.packets // 0), + confidence: (.confidence // "Unknown"), + ndpi_category: (.category // "Unknown"), + flow_id: (.id // 0), + state: (.state // "active") + } + ] | + # Calculate bandwidth and quality for each entry + map(. + { + duration: 1, + bandwidth: ((.bytes_rx + .bytes_tx) * 8 / 1000 | floor), + category: ( + if (.app | test("YouTube|Netflix|Disney|Amazon|Twitch|Hbo|Hulu|Vimeo|Peacock|Paramount|Plex|AppleTV"; "i")) then "video" + elif (.app | test("Spotify|Apple.*Music|Deezer|SoundCloud|Tidal|Pandora|iHeart|Audible"; "i")) then "audio" + elif (.app | test("Zoom|Teams|Meet|Discord|Skype|Webex|Face.*Time|WhatsApp|Signal|Telegram|Slack"; "i")) then "visio" + else "other" + end + ) + }) | + # Add quality estimation + map(. + { + quality: ( + if .category == "audio" then + (if .bandwidth < 96 then "Low" elif .bandwidth < 192 then "Normal" elif .bandwidth < 320 then "High" else "Lossless" end) + elif .category == "visio" then + (if .bandwidth < 500 then "Audio Only" elif .bandwidth < 1500 then "SD" elif .bandwidth < 3000 then "HD" else "FHD" end) + else + (if .bandwidth < 1000 then "SD" elif .bandwidth < 3000 then "HD" elif .bandwidth < 8000 then "FHD" else "4K" end) + end + ) + }) | + # Only include flows with significant traffic + [.[] | select(.bytes_rx > 10000 or .packets > 100)] + ' "$NDPID_FLOWS" 2>/dev/null) + + # Save current state to cache for frontend + if [ -n "$new_entries" ] && [ "$new_entries" != "[]" ] && [ "$new_entries" != "null" ]; then + echo "$new_entries" > "$MEDIA_CACHE" + + # Merge with history (avoid duplicates by flow_id within same minute) + jq -c --argjson new "$new_entries" ' + # Add new entries + . + ($new | map(del(.flow_id, .state))) | + # Remove duplicates (same client+app within 60 seconds) + unique_by(.client + .app + (.timestamp | split("T")[0])) | + # Keep only last MAX_ENTRIES + .[-'"$MAX_ENTRIES"':] + ' "$HISTORY_FILE" > "${HISTORY_FILE}.tmp" 2>/dev/null && mv "${HISTORY_FILE}.tmp" "$HISTORY_FILE" + else + # No active streams, save empty cache + echo '[]' > "$MEDIA_CACHE" + fi +fi + +# Also process nDPId apps file for aggregated stats +if [ -f "$NDPID_APPS" ] && command -v jq >/dev/null 2>&1; then + # Extract streaming apps from aggregated data + jq -c --arg pattern "$STREAMING_ALL" ' + [.[] | select(.name | test($pattern; "i"))] | + sort_by(-.bytes) | + .[0:20] + ' "$NDPID_APPS" > "/tmp/media-flow-apps.json" 2>/dev/null +fi + +# Clean old entries based on retention (days) +retention=$(uci -q get media_flow.global.history_retention 2>/dev/null || echo "7") +if [ "$retention" -gt 0 ] 2>/dev/null; then + cutoff_date=$(date -d "$retention days ago" -Iseconds 2>/dev/null || date -Iseconds) + jq -c --arg cutoff "$cutoff_date" '[.[] | select(.timestamp >= $cutoff)]' "$HISTORY_FILE" > "${HISTORY_FILE}.tmp" 2>/dev/null && mv "${HISTORY_FILE}.tmp" "$HISTORY_FILE" +fi + +exit 0 diff --git a/package/secubox/luci-app-media-flow/root/usr/libexec/rpcd/luci.media-flow b/package/secubox/luci-app-media-flow/root/usr/libexec/rpcd/luci.media-flow index cf498869..d261de83 100755 --- a/package/secubox/luci-app-media-flow/root/usr/libexec/rpcd/luci.media-flow +++ b/package/secubox/luci-app-media-flow/root/usr/libexec/rpcd/luci.media-flow @@ -1,14 +1,20 @@ #!/bin/sh # RPCD backend for Media Flow # Provides ubus interface: luci.media-flow -# Note: netifyd 5.x does not export per-flow application data locally -# This module shows available network statistics from netifyd +# Supports both nDPId (local DPI) and netifyd as data sources +# nDPId provides local application detection without cloud subscription . /lib/functions.sh . /usr/share/libubox/jshn.sh HISTORY_FILE="/tmp/media-flow-history.json" STATS_DIR="/tmp/media-flow-stats" +NDPID_FLOWS="/tmp/ndpid-flows.json" +NDPID_APPS="/tmp/ndpid-apps.json" +MEDIA_CACHE="/tmp/media-flow-ndpid-cache.json" + +# Streaming patterns for filtering +STREAMING_PATTERN="YouTube|Netflix|Disney|Amazon|Twitch|Hbo|Hulu|Vimeo|Peacock|Paramount|Spotify|Apple.*Music|Deezer|SoundCloud|Tidal|Zoom|Teams|Meet|Discord|Skype|Webex" # Initialize storage init_storage() { @@ -16,6 +22,36 @@ init_storage() { [ ! -f "$HISTORY_FILE" ] && echo '[]' > "$HISTORY_FILE" } +# Detect available DPI source +get_dpi_source() { + # Prefer nDPId if running and has data + if [ -f "$NDPID_FLOWS" ] && pgrep -x ndpid >/dev/null 2>&1; then + echo "ndpid" + elif [ -f /var/run/netifyd/status.json ] && pgrep -x netifyd >/dev/null 2>&1; then + echo "netifyd" + else + echo "none" + fi +} + +# Get nDPId flow data +get_ndpid_data() { + if [ -f "$NDPID_FLOWS" ]; then + cat "$NDPID_FLOWS" + else + echo '[]' + fi +} + +# Get nDPId apps data +get_ndpid_apps() { + if [ -f "$NDPID_APPS" ]; then + cat "$NDPID_APPS" + else + echo '[]' + fi +} + # Get netifyd status data get_netifyd_data() { if [ -f /var/run/netifyd/status.json ]; then @@ -79,21 +115,46 @@ case "$1" in status) init_storage + # Check nDPId status + ndpid_running=0 + ndpid_version="unknown" + ndpid_flows=0 + pgrep -x ndpid > /dev/null 2>&1 && ndpid_running=1 + if [ "$ndpid_running" = "1" ] && [ -f "$NDPID_FLOWS" ]; then + ndpid_flows=$(jq 'length' "$NDPID_FLOWS" 2>/dev/null || echo 0) + ndpid_version=$(ndpid -v 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+' | head -1 || echo "unknown") + fi + + # Check netifyd status netifyd_running=0 - pgrep netifyd > /dev/null 2>&1 && netifyd_running=1 + pgrep -x netifyd > /dev/null 2>&1 && netifyd_running=1 netifyd_data=$(get_netifyd_data) - flow_count=0 + netifyd_flows=0 netifyd_version="unknown" if [ "$netifyd_running" = "1" ] && [ -n "$netifyd_data" ] && [ "$netifyd_data" != "{}" ]; then - flow_count=$(echo "$netifyd_data" | jq '.flows_active // .flow_count // 0' 2>/dev/null || echo 0) + netifyd_flows=$(echo "$netifyd_data" | jq '.flows_active // .flow_count // 0' 2>/dev/null || echo 0) netifyd_version=$(echo "$netifyd_data" | jq -r '.agent_version // "unknown"' 2>/dev/null || echo "unknown") fi + # Determine active DPI source + dpi_source=$(get_dpi_source) + + # Use flow count from active source + if [ "$dpi_source" = "ndpid" ]; then + flow_count=$ndpid_flows + else + flow_count=$netifyd_flows + fi + history_count=0 [ -f "$HISTORY_FILE" ] && history_count=$(jq 'length' "$HISTORY_FILE" 2>/dev/null || echo 0) + # Check if nDPId cache has active streams + active_streams=0 + [ -f "$MEDIA_CACHE" ] && active_streams=$(jq 'length' "$MEDIA_CACHE" 2>/dev/null || echo 0) + enabled=$(uci -q get media_flow.global.enabled 2>/dev/null || echo "1") refresh=$(uci -q get media_flow.global.refresh_interval 2>/dev/null || echo "5") @@ -101,13 +162,18 @@ case "$1" in { "enabled": $enabled, "module": "media-flow", - "version": "0.5.2", + "version": "0.6.0", + "dpi_source": "$dpi_source", + "ndpid_running": $ndpid_running, + "ndpid_version": "$ndpid_version", + "ndpid_flows": $ndpid_flows, "netifyd_running": $netifyd_running, "netifyd_version": "$netifyd_version", + "netifyd_flows": $netifyd_flows, "active_flows": $flow_count, + "active_streams": $active_streams, "history_entries": $history_count, - "refresh_interval": $refresh, - "note": "netifyd 5.x requires cloud subscription for streaming detection" + "refresh_interval": $refresh } EOF ;; @@ -115,15 +181,32 @@ case "$1" in get_active_streams) init_storage - netifyd_data=$(get_netifyd_data) - flow_count=$(echo "$netifyd_data" | jq '.flows_active // .flow_count // 0' 2>/dev/null || echo 0) + dpi_source=$(get_dpi_source) + streams="[]" + flow_count=0 + + if [ "$dpi_source" = "ndpid" ]; then + # Get active streams from nDPId cache + if [ -f "$MEDIA_CACHE" ]; then + streams=$(cat "$MEDIA_CACHE" 2>/dev/null || echo "[]") + fi + if [ -f "$NDPID_FLOWS" ]; then + flow_count=$(jq 'length' "$NDPID_FLOWS" 2>/dev/null || echo 0) + fi + note="Streams detected via nDPId local DPI" + elif [ "$dpi_source" = "netifyd" ]; then + netifyd_data=$(get_netifyd_data) + flow_count=$(echo "$netifyd_data" | jq '.flows_active // .flow_count // 0' 2>/dev/null || echo 0) + note="Application detection requires netifyd cloud subscription" + else + note="No DPI engine available" + fi - # netifyd 5.x doesn't export application detection locally - # Return empty streams with explanation cat <<-EOF { - "streams": [], - "note": "Application detection requires netifyd cloud subscription", + "streams": $streams, + "dpi_source": "$dpi_source", + "note": "$note", "flow_count": $flow_count } EOF