From 4b1e0f3405d1c5cfb46f6e944273142f6c4cf78c Mon Sep 17 00:00:00 2001 From: CyberMind-FR Date: Sun, 11 Jan 2026 07:29:48 +0100 Subject: [PATCH] feat: Add realtime acquisition statistics to CrowdSec metrics view New features: - New RPCD method: acquisition_metrics for detailed stats - Realtime metrics display with 10-second polling - Visual stat cards: lines read, parsed, unparsed, buckets - Parse rate progress bar with color coding - Active acquisition sources badges - Rate calculation (events/sec) between polls - Live update indicator with timestamp API changes: - Added getAcquisitionMetrics() to API layer - Added acquisition_metrics to ACL permissions Bumped version to 0.7.0-17 Co-Authored-By: Claude Opus 4.5 --- .../luci-app-crowdsec-dashboard/Makefile | 2 +- .../resources/crowdsec-dashboard/api.js | 9 +- .../view/crowdsec-dashboard/metrics.js | 152 +++++++++++++++++- .../usr/libexec/rpcd/luci.crowdsec-dashboard | 94 ++++++++++- .../acl.d/luci-app-crowdsec-dashboard.json | 3 +- 5 files changed, 252 insertions(+), 8 deletions(-) diff --git a/package/secubox/luci-app-crowdsec-dashboard/Makefile b/package/secubox/luci-app-crowdsec-dashboard/Makefile index bc4dbc32..dc13866e 100644 --- a/package/secubox/luci-app-crowdsec-dashboard/Makefile +++ b/package/secubox/luci-app-crowdsec-dashboard/Makefile @@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=luci-app-crowdsec-dashboard PKG_VERSION:=0.7.0 -PKG_RELEASE:=16 +PKG_RELEASE:=17 PKG_ARCH:=all PKG_LICENSE:=Apache-2.0 diff --git a/package/secubox/luci-app-crowdsec-dashboard/htdocs/luci-static/resources/crowdsec-dashboard/api.js b/package/secubox/luci-app-crowdsec-dashboard/htdocs/luci-static/resources/crowdsec-dashboard/api.js index 7a3a81dc..2ee47d21 100644 --- a/package/secubox/luci-app-crowdsec-dashboard/htdocs/luci-static/resources/crowdsec-dashboard/api.js +++ b/package/secubox/luci-app-crowdsec-dashboard/htdocs/luci-static/resources/crowdsec-dashboard/api.js @@ -9,7 +9,7 @@ * CrowdSec Core: 1.7.4+ */ -// Version: 0.6.0 +// Version: 0.7.0 var callStatus = rpc.declare({ object: 'luci.crowdsec-dashboard', @@ -244,6 +244,12 @@ var callAcquisitionConfig = rpc.declare({ expect: { } }); +var callAcquisitionMetrics = rpc.declare({ + object: 'luci.crowdsec-dashboard', + method: 'acquisition_metrics', + expect: { } +}); + function formatDuration(seconds) { if (!seconds) return 'N/A'; if (seconds < 60) return seconds + 's'; @@ -377,6 +383,7 @@ return baseclass.extend({ // Acquisition Methods configureAcquisition: callConfigureAcquisition, getAcquisitionConfig: callAcquisitionConfig, + getAcquisitionMetrics: callAcquisitionMetrics, formatDuration: formatDuration, formatDate: formatDate, diff --git a/package/secubox/luci-app-crowdsec-dashboard/htdocs/luci-static/resources/view/crowdsec-dashboard/metrics.js b/package/secubox/luci-app-crowdsec-dashboard/htdocs/luci-static/resources/view/crowdsec-dashboard/metrics.js index 95d2e409..e8f25457 100644 --- a/package/secubox/luci-app-crowdsec-dashboard/htdocs/luci-static/resources/view/crowdsec-dashboard/metrics.js +++ b/package/secubox/luci-app-crowdsec-dashboard/htdocs/luci-static/resources/view/crowdsec-dashboard/metrics.js @@ -21,6 +21,9 @@ return view.extend({ bouncers: [], machines: [], hub: {}, + acquisitionMetrics: {}, + previousAcquisitionMetrics: null, + acquisitionRates: {}, load: function() { this.csApi = api; @@ -30,14 +33,16 @@ return view.extend({ this.csApi.getBouncers(), this.csApi.getMachines(), this.csApi.getHub(), - this.csApi.getMetricsConfig() + this.csApi.getMetricsConfig(), + this.csApi.getAcquisitionMetrics() ]).then(function(results) { return { metrics: results[0], bouncers: results[1], machines: results[2], hub: results[3], - metricsConfig: results[4] + metricsConfig: results[4], + acquisitionMetrics: results[5] }; }); }, @@ -255,6 +260,118 @@ return view.extend({ return E('div', { 'class': 'cyber-acquisition-list' }, items); }, + renderRealtimeAcquisitionMetrics: function() { + var self = this; + var acqMetrics = this.acquisitionMetrics; + + if (!acqMetrics || !acqMetrics.available) { + return E('div', { 'class': 'cyber-empty', 'style': 'text-align: center; padding: 2rem; color: var(--cyber-text-muted, #666);' }, [ + E('div', { 'style': 'font-size: 2rem; margin-bottom: 0.5rem;' }, '📊'), + E('p', {}, acqMetrics && acqMetrics.error ? acqMetrics.error : _('Realtime metrics not available')) + ]); + } + + var totalRead = acqMetrics.total_lines_read || 0; + var totalParsed = acqMetrics.total_lines_parsed || 0; + var totalUnparsed = acqMetrics.total_lines_unparsed || 0; + var totalBuckets = acqMetrics.total_buckets || 0; + var parseRate = acqMetrics.parse_rate || 0; + var activeFiles = acqMetrics.active_files || []; + + // Calculate rates if we have previous data + var readRate = this.acquisitionRates.readRate || 0; + var parsedRate = this.acquisitionRates.parsedRate || 0; + + // Create stats cards grid + var statsGrid = E('div', { 'class': 'cyber-realtime-stats', 'style': 'display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 1rem; margin-bottom: 1.5rem;' }, [ + // Lines Read Card + E('div', { 'class': 'cyber-stat-card', 'style': 'background: var(--cyber-card-bg, rgba(30,30,40,0.8)); border: 1px solid var(--cyber-border, rgba(255,255,255,0.1)); border-radius: 8px; padding: 1rem; text-align: center;' }, [ + E('div', { 'style': 'font-size: 0.75rem; color: var(--cyber-text-muted, #666); margin-bottom: 0.25rem; text-transform: uppercase;' }, _('Lines Read')), + E('div', { 'class': 'cyber-stat-value', 'style': 'font-size: 1.5rem; font-weight: 700; color: var(--cyber-accent-primary, #667eea);' }, this.formatNumber(totalRead)), + readRate > 0 ? E('div', { 'style': 'font-size: 0.7rem; color: var(--cyber-success, #00d4aa); margin-top: 0.25rem;' }, '+' + readRate + '/s') : null + ]), + // Lines Parsed Card + E('div', { 'class': 'cyber-stat-card', 'style': 'background: var(--cyber-card-bg, rgba(30,30,40,0.8)); border: 1px solid var(--cyber-border, rgba(255,255,255,0.1)); border-radius: 8px; padding: 1rem; text-align: center;' }, [ + E('div', { 'style': 'font-size: 0.75rem; color: var(--cyber-text-muted, #666); margin-bottom: 0.25rem; text-transform: uppercase;' }, _('Parsed')), + E('div', { 'class': 'cyber-stat-value', 'style': 'font-size: 1.5rem; font-weight: 700; color: var(--cyber-success, #00d4aa);' }, this.formatNumber(totalParsed)), + parsedRate > 0 ? E('div', { 'style': 'font-size: 0.7rem; color: var(--cyber-success, #00d4aa); margin-top: 0.25rem;' }, '+' + parsedRate + '/s') : null + ]), + // Parse Rate Card with progress bar + E('div', { 'class': 'cyber-stat-card', 'style': 'background: var(--cyber-card-bg, rgba(30,30,40,0.8)); border: 1px solid var(--cyber-border, rgba(255,255,255,0.1)); border-radius: 8px; padding: 1rem; text-align: center;' }, [ + E('div', { 'style': 'font-size: 0.75rem; color: var(--cyber-text-muted, #666); margin-bottom: 0.25rem; text-transform: uppercase;' }, _('Parse Rate')), + E('div', { 'class': 'cyber-stat-value', 'style': 'font-size: 1.5rem; font-weight: 700; color: ' + (parseRate >= 80 ? 'var(--cyber-success, #00d4aa)' : parseRate >= 50 ? 'var(--cyber-warning, #ffa500)' : 'var(--cyber-danger, #ff4757)') + ';' }, parseRate + '%'), + E('div', { 'class': 'cyber-progress', 'style': 'height: 4px; background: var(--cyber-border, rgba(255,255,255,0.1)); border-radius: 2px; margin-top: 0.5rem; overflow: hidden;' }, [ + E('div', { 'class': 'cyber-progress-bar', 'style': 'width: ' + parseRate + '%; height: 100%; background: ' + (parseRate >= 80 ? 'var(--cyber-success, #00d4aa)' : parseRate >= 50 ? 'var(--cyber-warning, #ffa500)' : 'var(--cyber-danger, #ff4757)') + '; transition: width 0.3s ease;' }) + ]) + ]), + // Buckets Card + E('div', { 'class': 'cyber-stat-card', 'style': 'background: var(--cyber-card-bg, rgba(30,30,40,0.8)); border: 1px solid var(--cyber-border, rgba(255,255,255,0.1)); border-radius: 8px; padding: 1rem; text-align: center;' }, [ + E('div', { 'style': 'font-size: 0.75rem; color: var(--cyber-text-muted, #666); margin-bottom: 0.25rem; text-transform: uppercase;' }, _('Buckets')), + E('div', { 'class': 'cyber-stat-value', 'style': 'font-size: 1.5rem; font-weight: 700; color: var(--cyber-warning, #ffa500);' }, this.formatNumber(totalBuckets)) + ]), + // Unparsed Card + E('div', { 'class': 'cyber-stat-card', 'style': 'background: var(--cyber-card-bg, rgba(30,30,40,0.8)); border: 1px solid var(--cyber-border, rgba(255,255,255,0.1)); border-radius: 8px; padding: 1rem; text-align: center;' }, [ + E('div', { 'style': 'font-size: 0.75rem; color: var(--cyber-text-muted, #666); margin-bottom: 0.25rem; text-transform: uppercase;' }, _('Unparsed')), + E('div', { 'class': 'cyber-stat-value', 'style': 'font-size: 1.5rem; font-weight: 700; color: ' + (totalUnparsed > 0 ? 'var(--cyber-danger, #ff4757)' : 'var(--cyber-text-muted, #666)') + ';' }, this.formatNumber(totalUnparsed)) + ]) + ]); + + // Active sources list + var sourcesList = E('div', { 'class': 'cyber-sources-list', 'style': 'margin-top: 1rem;' }, [ + E('div', { 'style': 'font-size: 0.85rem; font-weight: 600; color: var(--cyber-text-secondary, #a0a0b0); margin-bottom: 0.5rem;' }, _('Active Acquisition Sources')), + activeFiles.length > 0 ? + E('div', { 'style': 'display: flex; flex-wrap: wrap; gap: 0.5rem;' }, + activeFiles.map(function(file) { + return E('span', { + 'class': 'cyber-badge cyber-badge--info', + 'style': 'font-size: 0.75rem; padding: 0.25rem 0.5rem; background: var(--cyber-accent-primary, #667eea); color: white; border-radius: 4px;' + }, file); + }) + ) : + E('span', { 'style': 'color: var(--cyber-text-muted, #666); font-size: 0.85rem;' }, _('No active sources')) + ]); + + // Last update timestamp + var timestamp = acqMetrics.timestamp ? new Date(acqMetrics.timestamp * 1000).toLocaleTimeString() : 'N/A'; + var lastUpdate = E('div', { 'style': 'text-align: right; font-size: 0.75rem; color: var(--cyber-text-muted, #666); margin-top: 1rem;' }, [ + E('span', { 'class': 'cyber-pulse', 'style': 'display: inline-block; width: 8px; height: 8px; background: var(--cyber-success, #00d4aa); border-radius: 50%; margin-right: 0.5rem; animation: pulse 2s infinite;' }), + _('Last update: ') + timestamp + ]); + + return E('div', { 'class': 'cyber-realtime-acquisition', 'id': 'realtime-acquisition-container' }, [ + statsGrid, + sourcesList, + lastUpdate + ]); + }, + + formatNumber: function(num) { + if (num >= 1000000) return (num / 1000000).toFixed(1) + 'M'; + if (num >= 1000) return (num / 1000).toFixed(1) + 'K'; + return String(num); + }, + + updateAcquisitionRates: function(newMetrics) { + if (!this.previousAcquisitionMetrics || !newMetrics) { + this.previousAcquisitionMetrics = newMetrics; + return; + } + + var prevTimestamp = this.previousAcquisitionMetrics.timestamp || 0; + var newTimestamp = newMetrics.timestamp || 0; + var timeDiff = newTimestamp - prevTimestamp; + + if (timeDiff > 0) { + var readDiff = (newMetrics.total_lines_read || 0) - (this.previousAcquisitionMetrics.total_lines_read || 0); + var parsedDiff = (newMetrics.total_lines_parsed || 0) - (this.previousAcquisitionMetrics.total_lines_parsed || 0); + + this.acquisitionRates.readRate = Math.round(readDiff / timeDiff); + this.acquisitionRates.parsedRate = Math.round(parsedDiff / timeDiff); + } + + this.previousAcquisitionMetrics = newMetrics; + }, + renderMetricsConfig: function(metricsConfig) { var self = this; var enabled = metricsConfig && (metricsConfig.metrics_enabled === true || metricsConfig.metrics_enabled === 1); @@ -329,6 +446,7 @@ return view.extend({ this.bouncers = data.bouncers || []; this.machines = data.machines || {}; this.hub = data.hub || {}; + this.acquisitionMetrics = data.acquisitionMetrics || {}; var metricsConfig = data.metricsConfig || {}; var view = E('div', { 'class': 'crowdsec-dashboard crowdsec-metrics' }, [ @@ -390,7 +508,7 @@ return view.extend({ E('div', { 'class': 'cyber-card-body' }, this.renderCollectionsList()) ]), - // Acquisition + // Acquisition - Per Source Details E('div', { 'class': 'cyber-card' }, [ E('div', { 'class': 'cyber-card-header' }, [ E('div', { 'class': 'cyber-card-title' }, [ @@ -402,6 +520,18 @@ return view.extend({ ]) ]), + // Realtime Acquisition Statistics (full width) + E('div', { 'class': 'cyber-card', 'style': 'margin-top: 1.5rem;' }, [ + E('div', { 'class': 'cyber-card-header' }, [ + E('div', { 'class': 'cyber-card-title' }, [ + E('span', { 'style': 'margin-right: 0.5rem;' }, '⚡'), + _('Realtime Acquisition Statistics') + ]), + E('span', { 'class': 'cyber-badge cyber-badge--info', 'style': 'font-size: 0.7rem;' }, _('Live')) + ]), + E('div', { 'class': 'cyber-card-body', 'id': 'realtime-acquisition-body' }, this.renderRealtimeAcquisitionMetrics()) + ]), + // Raw metrics sections E('div', { 'class': 'cyber-card', 'style': 'margin-top: 1.5rem;' }, [ E('div', { 'class': 'cyber-card-header' }, [ @@ -422,7 +552,7 @@ return view.extend({ ]) ]); - // Setup polling (every 60 seconds for metrics) + // Setup polling (every 60 seconds for general metrics) poll.add(function() { return Promise.all([ self.csApi.getMetrics(), @@ -435,6 +565,20 @@ return view.extend({ }); }, 60); + // Fast polling for realtime acquisition metrics (every 10 seconds) + poll.add(function() { + return self.csApi.getAcquisitionMetrics().then(function(result) { + self.updateAcquisitionRates(result); + self.acquisitionMetrics = result; + + // Update the realtime acquisition display + var container = document.getElementById('realtime-acquisition-body'); + if (container) { + dom.content(container, self.renderRealtimeAcquisitionMetrics()); + } + }); + }, 10); + return view; }, diff --git a/package/secubox/luci-app-crowdsec-dashboard/root/usr/libexec/rpcd/luci.crowdsec-dashboard b/package/secubox/luci-app-crowdsec-dashboard/root/usr/libexec/rpcd/luci.crowdsec-dashboard index be660cbd..2638339a 100755 --- a/package/secubox/luci-app-crowdsec-dashboard/root/usr/libexec/rpcd/luci.crowdsec-dashboard +++ b/package/secubox/luci-app-crowdsec-dashboard/root/usr/libexec/rpcd/luci.crowdsec-dashboard @@ -1369,6 +1369,95 @@ YAML json_dump } +# Get realtime acquisition metrics with rates +get_acquisition_metrics() { + check_cscli + json_init + + # Get raw metrics from cscli + local metrics_output + metrics_output=$(run_cscli metrics -o json 2>/dev/null) + + if [ -z "$metrics_output" ]; then + json_add_boolean "available" 0 + json_add_string "error" "Metrics not available" + json_dump + return + fi + + json_add_boolean "available" 1 + json_add_int "timestamp" "$(date +%s)" + + # Parse acquisition sources from metrics + # Store metrics in temp file for parsing + local tmp_file="/tmp/crowdsec_metrics.$$" + echo "$metrics_output" > "$tmp_file" + + # Extract acquisition metrics + json_add_array "sources" + + # Use jsonfilter to extract acquisition data + # Format: {"source": "file:/var/log/messages", "lines_read": 123, "lines_parsed": 100, ...} + local sources + sources=$(cat "$tmp_file" | jsonfilter -e '@.acquisition' 2>/dev/null) + + if [ -n "$sources" ]; then + # Parse each source + for source in $(echo "$metrics_output" | jsonfilter -e '@.acquisition.*' 2>/dev/null | head -20); do + json_add_object "" + json_add_string "source" "$source" + json_close_object + done + fi + + json_close_array + + # Get overall stats + local total_read=0 + local total_parsed=0 + local total_unparsed=0 + local total_buckets=0 + + # Parse acquisition stats using awk + if [ -f "$tmp_file" ]; then + # Extract line counts + total_read=$(cat "$tmp_file" | grep -o '"lines_read":[0-9]*' | grep -o '[0-9]*' | awk '{sum+=$1} END {print sum+0}') + total_parsed=$(cat "$tmp_file" | grep -o '"lines_parsed":[0-9]*' | grep -o '[0-9]*' | awk '{sum+=$1} END {print sum+0}') + total_unparsed=$(cat "$tmp_file" | grep -o '"lines_unparsed":[0-9]*' | grep -o '[0-9]*' | awk '{sum+=$1} END {print sum+0}') + total_buckets=$(cat "$tmp_file" | grep -o '"lines_poured_to_bucket":[0-9]*' | grep -o '[0-9]*' | awk '{sum+=$1} END {print sum+0}') + fi + + json_add_int "total_lines_read" "${total_read:-0}" + json_add_int "total_lines_parsed" "${total_parsed:-0}" + json_add_int "total_lines_unparsed" "${total_unparsed:-0}" + json_add_int "total_buckets" "${total_buckets:-0}" + + # Calculate parse rate + if [ "$total_read" -gt 0 ]; then + local parse_rate=$((total_parsed * 100 / total_read)) + json_add_int "parse_rate" "$parse_rate" + else + json_add_int "parse_rate" 0 + fi + + # Check active acquisition files + json_add_array "active_files" + local acquis_dir="/etc/crowdsec/acquis.d" + if [ -d "$acquis_dir" ]; then + for f in "$acquis_dir"/*.yaml; do + if [ -f "$f" ]; then + json_add_string "" "$(basename "$f" .yaml)" + fi + done + fi + json_close_array + + # Clean up + rm -f "$tmp_file" + + json_dump +} + # Get current acquisition configuration get_acquisition_config() { json_init @@ -1441,7 +1530,7 @@ service_control() { # Main dispatcher case "$1" in list) - echo '{"decisions":{},"alerts":{"limit":"number"},"metrics":{},"bouncers":{},"machines":{},"hub":{},"status":{},"ban":{"ip":"string","duration":"string","reason":"string"},"unban":{"ip":"string"},"stats":{},"seccubox_logs":{},"collect_debug":{},"waf_status":{},"metrics_config":{},"configure_metrics":{"enable":"string"},"collections":{},"install_collection":{"collection":"string"},"remove_collection":{"collection":"string"},"update_hub":{},"register_bouncer":{"bouncer_name":"string"},"delete_bouncer":{"bouncer_name":"string"},"firewall_bouncer_status":{},"control_firewall_bouncer":{"action":"string"},"firewall_bouncer_config":{},"update_firewall_bouncer_config":{"key":"string","value":"string"},"nftables_stats":{},"check_wizard_needed":{},"wizard_state":{},"repair_lapi":{},"reset_wizard":{},"console_status":{},"console_enroll":{"key":"string","name":"string"},"console_disable":{},"service_control":{"action":"string"},"configure_acquisition":{"syslog_enabled":"string","firewall_enabled":"string","ssh_enabled":"string","http_enabled":"string","syslog_path":"string"},"acquisition_config":{}}' + echo '{"decisions":{},"alerts":{"limit":"number"},"metrics":{},"bouncers":{},"machines":{},"hub":{},"status":{},"ban":{"ip":"string","duration":"string","reason":"string"},"unban":{"ip":"string"},"stats":{},"seccubox_logs":{},"collect_debug":{},"waf_status":{},"metrics_config":{},"configure_metrics":{"enable":"string"},"collections":{},"install_collection":{"collection":"string"},"remove_collection":{"collection":"string"},"update_hub":{},"register_bouncer":{"bouncer_name":"string"},"delete_bouncer":{"bouncer_name":"string"},"firewall_bouncer_status":{},"control_firewall_bouncer":{"action":"string"},"firewall_bouncer_config":{},"update_firewall_bouncer_config":{"key":"string","value":"string"},"nftables_stats":{},"check_wizard_needed":{},"wizard_state":{},"repair_lapi":{},"reset_wizard":{},"console_status":{},"console_enroll":{"key":"string","name":"string"},"console_disable":{},"service_control":{"action":"string"},"configure_acquisition":{"syslog_enabled":"string","firewall_enabled":"string","ssh_enabled":"string","http_enabled":"string","syslog_path":"string"},"acquisition_config":{},"acquisition_metrics":{}}' ;; call) case "$2" in @@ -1587,6 +1676,9 @@ case "$1" in acquisition_config) get_acquisition_config ;; + acquisition_metrics) + get_acquisition_metrics + ;; *) echo '{"error": "Unknown method"}' ;; diff --git a/package/secubox/luci-app-crowdsec-dashboard/root/usr/share/rpcd/acl.d/luci-app-crowdsec-dashboard.json b/package/secubox/luci-app-crowdsec-dashboard/root/usr/share/rpcd/acl.d/luci-app-crowdsec-dashboard.json index a4b143a4..7b7427ef 100644 --- a/package/secubox/luci-app-crowdsec-dashboard/root/usr/share/rpcd/acl.d/luci-app-crowdsec-dashboard.json +++ b/package/secubox/luci-app-crowdsec-dashboard/root/usr/share/rpcd/acl.d/luci-app-crowdsec-dashboard.json @@ -22,7 +22,8 @@ "check_wizard_needed", "wizard_state", "console_status", - "acquisition_config" + "acquisition_config", + "acquisition_metrics" ], "file": [ "read", "stat" ] },