From f4b9c910c533439f97fa2f9563e2e84723c594a3 Mon Sep 17 00:00:00 2001 From: CyberMind-FR Date: Mon, 2 Feb 2026 10:07:40 +0100 Subject: [PATCH] feat(mitmproxy): Add WAN protection mode for incoming traffic inspection Add WAF-like functionality to mitmproxy for protecting services exposed to the internet. Incoming WAN traffic is redirected through mitmproxy for threat detection before reaching backend services. Features: - WAN protection mode with nftables rules for incoming traffic - Enhanced bot scanner detection with 50+ scanner signatures - Behavioral detection for config/admin/backup/shell hunting - CrowdSec integration with new scenarios for bot scanners - LuCI interface for WAN protection configuration - DPI mirror mode support (secondary feature) New CrowdSec scenarios: - secubox/mitmproxy-botscan: Detect automated reconnaissance - secubox/mitmproxy-shell-hunter: Detect shell/backdoor hunting - secubox/mitmproxy-config-hunter: Detect credential file hunting - secubox/mitmproxy-suspicious-ua: Detect suspicious user agents Co-Authored-By: Claude Opus 4.5 --- .../resources/view/mitmproxy/settings.js | 118 ++++++++- .../root/usr/libexec/rpcd/luci.mitmproxy | 142 +++++++++- .../parsers/s01-parse/secubox-mitmproxy.yaml | 21 ++ .../scenarios/secubox-mitmproxy-threats.yaml | 64 +++++ .../files/etc/config/mitmproxy | 30 ++- .../files/usr/sbin/mitmproxyctl | 188 +++++++++++++- .../srv/mitmproxy/addons/secubox_analytics.py | 244 ++++++++++++++++-- 7 files changed, 763 insertions(+), 44 deletions(-) diff --git a/package/secubox/luci-app-mitmproxy/htdocs/luci-static/resources/view/mitmproxy/settings.js b/package/secubox/luci-app-mitmproxy/htdocs/luci-static/resources/view/mitmproxy/settings.js index 35631a08..dbd8e9cf 100644 --- a/package/secubox/luci-app-mitmproxy/htdocs/luci-static/resources/view/mitmproxy/settings.js +++ b/package/secubox/luci-app-mitmproxy/htdocs/luci-static/resources/view/mitmproxy/settings.js @@ -2,17 +2,21 @@ 'require view'; 'require form'; 'require uci'; +'require network'; return view.extend({ load: function() { - return uci.load('mitmproxy'); + return Promise.all([ + uci.load('mitmproxy'), + uci.load('network') + ]); }, render: function() { var m, s, o; m = new form.Map('mitmproxy', _('mitmproxy Settings'), - _('Configure the HTTPS intercepting proxy.')); + _('Configure the HTTPS intercepting proxy for traffic inspection and threat detection.')); // Main Settings s = m.section(form.TypedSection, 'mitmproxy', _('General')); @@ -23,10 +27,10 @@ return view.extend({ o = s.option(form.ListValue, 'mode', _('Mode')); o.value('regular', _('Regular Proxy')); - o.value('transparent', _('Transparent Proxy')); + o.value('transparent', _('Transparent Proxy (LAN)')); o.value('upstream', _('Upstream Proxy')); o.value('reverse', _('Reverse Proxy')); - o.default = 'transparent'; + o.default = 'regular'; o = s.option(form.Value, 'proxy_port', _('Proxy Port')); o.datatype = 'port'; @@ -51,28 +55,114 @@ return view.extend({ o = s.option(form.Flag, 'anticomp', _('Disable Compression')); - // Transparent Mode - s = m.section(form.TypedSection, 'transparent', _('Transparent Mode')); + // WAN Protection Mode + s = m.section(form.TypedSection, 'wan_protection', _('WAN Protection Mode')); s.anonymous = true; + s.description = _('Protect services exposed to the internet. Intercept incoming WAN traffic for threat detection (WAF mode). Detects bot scanners, attacks, and feeds to CrowdSec for automatic blocking.'); - o = s.option(form.Flag, 'enabled', _('Enable Transparent Redirect')); + o = s.option(form.Flag, 'enabled', _('Enable WAN Protection')); + o.description = _('Intercept incoming WAN traffic for threat analysis'); + o.default = '0'; - o = s.option(form.Value, 'interface', _('Interface')); + o = s.option(form.ListValue, 'wan_interface', _('WAN Interface')); + o.description = _('Network interface for incoming traffic'); + o.default = 'wan'; + o.depends('enabled', '1'); + // Add common WAN interface options + o.value('wan', _('wan')); + o.value('wan6', _('wan6')); + o.value('eth1', _('eth1')); + o.value('eth0', _('eth0')); + // Try to populate from network config + uci.sections('network', 'interface', function(iface) { + if (iface['.name'] && iface['.name'].match(/wan/i)) { + o.value(iface['.name'], iface['.name']); + } + }); + + o = s.option(form.Value, 'wan_http_port', _('WAN HTTP Port')); + o.datatype = 'port'; + o.default = '80'; + o.description = _('HTTP port to intercept on WAN (0 to disable)'); + o.depends('enabled', '1'); + + o = s.option(form.Value, 'wan_https_port', _('WAN HTTPS Port')); + o.datatype = 'port'; + o.default = '443'; + o.description = _('HTTPS port to intercept on WAN (0 to disable)'); + o.depends('enabled', '1'); + + o = s.option(form.Flag, 'crowdsec_feed', _('CrowdSec Integration')); + o.description = _('Feed detected threats to CrowdSec for automatic blocking'); + o.default = '1'; + o.depends('enabled', '1'); + + o = s.option(form.Flag, 'block_bots', _('Block Known Bots')); + o.description = _('Immediately block requests from known bot scanners (Nikto, SQLMap, etc.)'); + o.default = '0'; + o.depends('enabled', '1'); + + o = s.option(form.Value, 'rate_limit', _('Rate Limit')); + o.datatype = 'uinteger'; + o.default = '0'; + o.description = _('Max requests per IP per minute (0 to disable rate limiting)'); + o.depends('enabled', '1'); + + // LAN Transparent Mode + s = m.section(form.TypedSection, 'transparent', _('LAN Transparent Mode')); + s.anonymous = true; + s.description = _('Intercept outbound LAN traffic for inspection. Note: WAN Protection Mode is recommended for most use cases.'); + + o = s.option(form.Flag, 'enabled', _('Enable LAN Transparent Redirect')); + o.description = _('Redirect outbound LAN HTTP/HTTPS traffic through proxy'); + + o = s.option(form.Value, 'interface', _('LAN Interface')); o.default = 'br-lan'; o.depends('enabled', '1'); - // Filtering - s = m.section(form.TypedSection, 'filtering', _('Analytics')); + o = s.option(form.Flag, 'redirect_http', _('Redirect HTTP')); + o.default = '1'; + o.depends('enabled', '1'); + + o = s.option(form.Flag, 'redirect_https', _('Redirect HTTPS')); + o.default = '1'; + o.depends('enabled', '1'); + + // DPI Mirror Mode + s = m.section(form.TypedSection, 'dpi_mirror', _('DPI Mirror Mode')); + s.anonymous = true; + s.description = _('Mirror traffic to DPI engines (netifyd/ndpid) for deep packet inspection. This is a secondary feature for advanced network analysis.'); + + o = s.option(form.Flag, 'enabled', _('Enable DPI Mirror')); + o.description = _('Mirror traffic to DPI interface for analysis'); + o.default = '0'; + + o = s.option(form.Value, 'dpi_interface', _('DPI Interface')); + o.default = 'br-lan'; + o.description = _('Interface where DPI engines listen'); + o.depends('enabled', '1'); + + o = s.option(form.Flag, 'mirror_wan', _('Mirror WAN Traffic')); + o.default = '0'; + o.depends('enabled', '1'); + + o = s.option(form.Flag, 'mirror_lan', _('Mirror LAN Traffic')); + o.default = '0'; + o.depends('enabled', '1'); + + // Filtering/Analytics + s = m.section(form.TypedSection, 'filtering', _('Threat Analytics')); s.anonymous = true; - o = s.option(form.Flag, 'enabled', _('Enable Analytics')); - o.description = _('Enable threat detection addon'); + o = s.option(form.Flag, 'enabled', _('Enable Threat Analytics')); + o.description = _('Enable threat detection addon for attack analysis'); - o = s.option(form.Value, 'addon_script', _('Addon Script')); + o = s.option(form.Value, 'addon_script', _('Analytics Addon')); o.default = '/data/addons/secubox_analytics.py'; o.depends('enabled', '1'); - o = s.option(form.Flag, 'log_requests', _('Log Requests')); + o = s.option(form.Flag, 'log_requests', _('Log All Requests')); + o.description = _('Log all requests (not just threats) for analysis'); o.depends('enabled', '1'); // HAProxy Router diff --git a/package/secubox/luci-app-mitmproxy/root/usr/libexec/rpcd/luci.mitmproxy b/package/secubox/luci-app-mitmproxy/root/usr/libexec/rpcd/luci.mitmproxy index 49aa1824..f92c7773 100755 --- a/package/secubox/luci-app-mitmproxy/root/usr/libexec/rpcd/luci.mitmproxy +++ b/package/secubox/luci-app-mitmproxy/root/usr/libexec/rpcd/luci.mitmproxy @@ -20,6 +20,12 @@ get_status() { local haproxy_router_enabled=$(uci_get haproxy_router.enabled) local haproxy_listen_port=$(uci_get haproxy_router.listen_port) + # WAN Protection settings + local wan_protection_enabled=$(uci_get wan_protection.enabled) + local wan_interface=$(uci_get wan_protection.wan_interface) + local crowdsec_feed=$(uci_get wan_protection.crowdsec_feed) + local block_bots=$(uci_get wan_protection.block_bots) + # Check for LXC availability local lxc_available=0 command -v lxc-start >/dev/null 2>&1 && lxc_available=1 @@ -40,6 +46,12 @@ get_status() { nft list table inet mitmproxy >/dev/null 2>&1 && nft_active=1 fi + # Check nftables status for WAN protection mode + local nft_wan_active=0 + if [ "$wan_protection_enabled" = "1" ] && command -v nft >/dev/null 2>&1; then + nft list table inet mitmproxy_wan >/dev/null 2>&1 && nft_wan_active=1 + fi + # Get authentication token from container data path local token="" local token_file="${data_path:-/srv/mitmproxy}/.mitmproxy_token" @@ -57,9 +69,14 @@ get_status() { "data_path": "${data_path:-/srv/mitmproxy}", "mode": "${mode:-regular}", "nft_active": $([ "$nft_active" = "1" ] && echo "true" || echo "false"), + "nft_wan_active": $([ "$nft_wan_active" = "1" ] && echo "true" || echo "false"), "token": "${token:-}", "haproxy_router_enabled": $([ "$haproxy_router_enabled" = "1" ] && echo "true" || echo "false"), - "haproxy_listen_port": ${haproxy_listen_port:-8889} + "haproxy_listen_port": ${haproxy_listen_port:-8889}, + "wan_protection_enabled": $([ "$wan_protection_enabled" = "1" ] && echo "true" || echo "false"), + "wan_interface": "${wan_interface:-wan}", + "crowdsec_feed": $([ "$crowdsec_feed" = "1" ] && echo "true" || echo "false"), + "block_bots": $([ "$block_bots" = "1" ] && echo "true" || echo "false") } EOFJ } @@ -94,7 +111,24 @@ get_settings() { json_add_boolean "anticache" "${anticache:-0}" json_add_boolean "anticomp" "${anticomp:-0}" - # Transparent settings + # WAN Protection settings + local wan_protection_enabled=$(uci_get wan_protection.enabled) + local wan_interface=$(uci_get wan_protection.wan_interface) + local wan_http_port=$(uci_get wan_protection.wan_http_port) + local wan_https_port=$(uci_get wan_protection.wan_https_port) + local crowdsec_feed=$(uci_get wan_protection.crowdsec_feed) + local block_bots=$(uci_get wan_protection.block_bots) + local rate_limit=$(uci_get wan_protection.rate_limit) + + json_add_boolean "wan_protection_enabled" "${wan_protection_enabled:-0}" + json_add_string "wan_interface" "${wan_interface:-wan}" + json_add_int "wan_http_port" "${wan_http_port:-80}" + json_add_int "wan_https_port" "${wan_https_port:-443}" + json_add_boolean "crowdsec_feed" "${crowdsec_feed:-1}" + json_add_boolean "block_bots" "${block_bots:-0}" + json_add_int "rate_limit" "${rate_limit:-0}" + + # Transparent settings (LAN) local transparent_enabled=$(uci_get transparent.enabled) local transparent_iface=$(uci_get transparent.interface) local redirect_http=$(uci_get transparent.redirect_http) @@ -105,6 +139,17 @@ get_settings() { json_add_boolean "redirect_http" "${redirect_http:-1}" json_add_boolean "redirect_https" "${redirect_https:-1}" + # DPI Mirror settings + local dpi_mirror_enabled=$(uci_get dpi_mirror.enabled) + local dpi_interface=$(uci_get dpi_mirror.dpi_interface) + local mirror_wan=$(uci_get dpi_mirror.mirror_wan) + local mirror_lan=$(uci_get dpi_mirror.mirror_lan) + + json_add_boolean "dpi_mirror_enabled" "${dpi_mirror_enabled:-0}" + json_add_string "dpi_interface" "${dpi_interface:-br-lan}" + json_add_boolean "mirror_wan" "${mirror_wan:-0}" + json_add_boolean "mirror_lan" "${mirror_lan:-0}" + # Filtering settings local filtering_enabled=$(uci_get filtering.enabled) local log_requests=$(uci_get filtering.log_requests) @@ -128,7 +173,10 @@ save_settings() { # Get values from input local mode enabled proxy_port web_port web_host data_path memory_limit local upstream_proxy reverse_target ssl_insecure anticache anticomp + local wan_protection_enabled wan_interface wan_http_port wan_https_port + local crowdsec_feed block_bots rate_limit local transparent_enabled transparent_interface redirect_http redirect_https + local dpi_mirror_enabled dpi_interface mirror_wan mirror_lan local filtering_enabled log_requests filter_cdn filter_media block_ads local apply_now @@ -144,10 +192,21 @@ save_settings() { json_get_var ssl_insecure ssl_insecure json_get_var anticache anticache json_get_var anticomp anticomp + json_get_var wan_protection_enabled wan_protection_enabled + json_get_var wan_interface wan_interface + json_get_var wan_http_port wan_http_port + json_get_var wan_https_port wan_https_port + json_get_var crowdsec_feed crowdsec_feed + json_get_var block_bots block_bots + json_get_var rate_limit rate_limit json_get_var transparent_enabled transparent_enabled json_get_var transparent_interface transparent_interface json_get_var redirect_http redirect_http json_get_var redirect_https redirect_https + json_get_var dpi_mirror_enabled dpi_mirror_enabled + json_get_var dpi_interface dpi_interface + json_get_var mirror_wan mirror_wan + json_get_var mirror_lan mirror_lan json_get_var filtering_enabled filtering_enabled json_get_var log_requests log_requests json_get_var filter_cdn filter_cdn @@ -164,10 +223,18 @@ save_settings() { uci set mitmproxy.main.enabled='0' uci set mitmproxy.main.mode='regular' } + uci -q get mitmproxy.wan_protection >/dev/null 2>&1 || { + uci set mitmproxy.wan_protection=wan_protection + uci set mitmproxy.wan_protection.enabled='0' + } uci -q get mitmproxy.transparent >/dev/null 2>&1 || { uci set mitmproxy.transparent=transparent uci set mitmproxy.transparent.enabled='0' } + uci -q get mitmproxy.dpi_mirror >/dev/null 2>&1 || { + uci set mitmproxy.dpi_mirror=dpi_mirror + uci set mitmproxy.dpi_mirror.enabled='0' + } uci -q get mitmproxy.filtering >/dev/null 2>&1 || { uci set mitmproxy.filtering=filtering uci set mitmproxy.filtering.enabled='0' @@ -187,12 +254,27 @@ save_settings() { [ -n "$anticache" ] && uci_set main.anticache "$anticache" [ -n "$anticomp" ] && uci_set main.anticomp "$anticomp" - # Apply transparent settings + # Apply WAN protection settings + [ -n "$wan_protection_enabled" ] && uci_set wan_protection.enabled "$wan_protection_enabled" + [ -n "$wan_interface" ] && uci_set wan_protection.wan_interface "$wan_interface" + [ -n "$wan_http_port" ] && uci_set wan_protection.wan_http_port "$wan_http_port" + [ -n "$wan_https_port" ] && uci_set wan_protection.wan_https_port "$wan_https_port" + [ -n "$crowdsec_feed" ] && uci_set wan_protection.crowdsec_feed "$crowdsec_feed" + [ -n "$block_bots" ] && uci_set wan_protection.block_bots "$block_bots" + [ -n "$rate_limit" ] && uci_set wan_protection.rate_limit "$rate_limit" + + # Apply transparent settings (LAN) [ -n "$transparent_enabled" ] && uci_set transparent.enabled "$transparent_enabled" [ -n "$transparent_interface" ] && uci_set transparent.interface "$transparent_interface" [ -n "$redirect_http" ] && uci_set transparent.redirect_http "$redirect_http" [ -n "$redirect_https" ] && uci_set transparent.redirect_https "$redirect_https" + # Apply DPI mirror settings + [ -n "$dpi_mirror_enabled" ] && uci_set dpi_mirror.enabled "$dpi_mirror_enabled" + [ -n "$dpi_interface" ] && uci_set dpi_mirror.dpi_interface "$dpi_interface" + [ -n "$mirror_wan" ] && uci_set dpi_mirror.mirror_wan "$mirror_wan" + [ -n "$mirror_lan" ] && uci_set dpi_mirror.mirror_lan "$mirror_lan" + # Apply filtering settings [ -n "$filtering_enabled" ] && uci_set filtering.enabled "$filtering_enabled" [ -n "$log_requests" ] && uci_set filtering.log_requests "$log_requests" @@ -485,8 +567,58 @@ sync_routes() { json_dump } +wan_setup() { + json_init + + if ! command -v mitmproxyctl >/dev/null 2>&1; then + json_add_boolean "success" 0 + json_add_string "error" "mitmproxyctl not found" + json_dump + return + fi + + mitmproxyctl wan-setup >/tmp/wan-setup.log 2>&1 + local result=$? + + if [ $result -eq 0 ]; then + json_add_boolean "success" 1 + json_add_string "message" "WAN protection rules applied" + else + json_add_boolean "success" 0 + json_add_string "error" "Failed to setup WAN protection rules" + local log=$(cat /tmp/wan-setup.log 2>/dev/null) + [ -n "$log" ] && json_add_string "details" "$log" + fi + + json_dump +} + +wan_clear() { + json_init + + if ! command -v mitmproxyctl >/dev/null 2>&1; then + json_add_boolean "success" 0 + json_add_string "error" "mitmproxyctl not found" + json_dump + return + fi + + mitmproxyctl wan-clear >/tmp/wan-clear.log 2>&1 + local result=$? + + if [ $result -eq 0 ]; then + json_add_boolean "success" 1 + json_add_string "message" "WAN protection rules cleared" + else + json_add_boolean "success" 0 + json_add_string "error" "Failed to clear WAN protection rules" + fi + + json_dump +} + list_methods() { cat <<'EOFM' -{"status":{},"settings":{},"save_settings":{"mode":"str","enabled":"bool","proxy_port":"int","web_port":"int","apply_now":"bool"},"set_mode":{"mode":"str","apply_now":"bool"},"setup_firewall":{},"clear_firewall":{},"install":{},"start":{},"stop":{},"restart":{},"alerts":{},"threat_stats":{},"clear_alerts":{},"haproxy_enable":{},"haproxy_disable":{},"sync_routes":{}} +{"status":{},"settings":{},"save_settings":{"mode":"str","enabled":"bool","proxy_port":"int","web_port":"int","apply_now":"bool","wan_protection_enabled":"bool","wan_interface":"str"},"set_mode":{"mode":"str","apply_now":"bool"},"setup_firewall":{},"clear_firewall":{},"wan_setup":{},"wan_clear":{},"install":{},"start":{},"stop":{},"restart":{},"alerts":{},"threat_stats":{},"clear_alerts":{},"haproxy_enable":{},"haproxy_disable":{},"sync_routes":{}} EOFM } @@ -500,6 +632,8 @@ case "$1" in set_mode) set_mode ;; setup_firewall) setup_firewall ;; clear_firewall) clear_firewall ;; + wan_setup) wan_setup ;; + wan_clear) wan_clear ;; install) do_install ;; start) do_start ;; stop) do_stop ;; diff --git a/package/secubox/secubox-app-crowdsec-custom/files/parsers/s01-parse/secubox-mitmproxy.yaml b/package/secubox/secubox-app-crowdsec-custom/files/parsers/s01-parse/secubox-mitmproxy.yaml index 54fd117f..08b42c35 100644 --- a/package/secubox/secubox-app-crowdsec-custom/files/parsers/s01-parse/secubox-mitmproxy.yaml +++ b/package/secubox/secubox-app-crowdsec-custom/files/parsers/s01-parse/secubox-mitmproxy.yaml @@ -30,8 +30,18 @@ statics: expression: JsonExtract(evt.Line.Raw, "response_code") - parsed: is_bot expression: JsonExtract(evt.Line.Raw, "is_bot") + - parsed: bot_type + expression: JsonExtract(evt.Line.Raw, "bot_type") + - parsed: bot_behavior + expression: JsonExtract(evt.Line.Raw, "bot_behavior") + - parsed: suspicious_ua + expression: JsonExtract(evt.Line.Raw, "suspicious_ua") - parsed: country expression: JsonExtract(evt.Line.Raw, "country") + - parsed: fingerprint + expression: JsonExtract(evt.Line.Raw, "fingerprint") + - parsed: rate_limited + expression: JsonExtract(evt.Line.Raw, "rate_limited") - meta: log_type value: mitmproxy_threat - meta: service @@ -51,3 +61,14 @@ statics: expression: evt.Parsed.threat_type - meta: attack_pattern expression: evt.Parsed.pattern +--- +# Filter for bot scanner activity +onsuccess: next_stage +name: secubox/mitmproxy-bot-filter +description: "Filter bot scanner activity for analysis" +filter: "evt.Meta.log_type == 'mitmproxy_threat' && (evt.Parsed.is_bot == 'true' || evt.Parsed.bot_behavior != '')" +statics: + - meta: is_bot_activity + value: "true" + - meta: bot_category + expression: evt.Parsed.bot_type diff --git a/package/secubox/secubox-app-crowdsec-custom/files/scenarios/secubox-mitmproxy-threats.yaml b/package/secubox/secubox-app-crowdsec-custom/files/scenarios/secubox-mitmproxy-threats.yaml index 454697a8..234eccfa 100644 --- a/package/secubox/secubox-app-crowdsec-custom/files/scenarios/secubox-mitmproxy-threats.yaml +++ b/package/secubox/secubox-app-crowdsec-custom/files/scenarios/secubox-mitmproxy-threats.yaml @@ -63,3 +63,67 @@ labels: service: mitmproxy type: cve_exploit remediation: true +--- +# Detect automated bot/scanner reconnaissance +type: leaky +name: secubox/mitmproxy-botscan +description: "Detect automated bot/scanner reconnaissance via mitmproxy" +filter: | + evt.Meta.log_type == 'mitmproxy_threat' && + (evt.Parsed.is_bot == 'true' || + evt.Parsed.bot_type in ['vulnerability_scanner', 'directory_scanner', 'port_scanner', 'cms_scanner'] || + evt.Parsed.bot_behavior in ['config_hunting', 'admin_hunting', 'backup_hunting', 'shell_hunting', 'api_discovery']) +groupby: evt.Meta.source_ip +capacity: 5 +leakspeed: 30s +blackhole: 30m +labels: + service: mitmproxy + type: bot_scanner + remediation: true +--- +# Detect shell/backdoor hunting (critical - immediate action) +type: trigger +name: secubox/mitmproxy-shell-hunter +description: "Detect shell/backdoor hunting attempts via mitmproxy" +filter: | + evt.Meta.log_type == 'mitmproxy_threat' && + evt.Parsed.bot_behavior == 'shell_hunting' +blackhole: 60m +labels: + service: mitmproxy + type: shell_hunter + remediation: true +--- +# Detect credential/config file hunting +type: leaky +name: secubox/mitmproxy-config-hunter +description: "Detect credential/config file hunting via mitmproxy" +filter: | + evt.Meta.log_type == 'mitmproxy_threat' && + evt.Parsed.bot_behavior == 'config_hunting' +groupby: evt.Meta.source_ip +capacity: 3 +leakspeed: 60s +blackhole: 30m +labels: + service: mitmproxy + type: config_hunter + remediation: true +--- +# Detect suspicious user agents (empty, minimal, or clearly fake) +type: leaky +name: secubox/mitmproxy-suspicious-ua +description: "Detect requests with suspicious user agents via mitmproxy" +filter: | + evt.Meta.log_type == 'mitmproxy_threat' && + evt.Parsed.suspicious_ua == 'true' && + evt.Parsed.is_bot != 'true' +groupby: evt.Meta.source_ip +capacity: 15 +leakspeed: 60s +blackhole: 15m +labels: + service: mitmproxy + type: suspicious_ua + remediation: true diff --git a/package/secubox/secubox-app-mitmproxy/files/etc/config/mitmproxy b/package/secubox/secubox-app-mitmproxy/files/etc/config/mitmproxy index 27940747..ef01a7e4 100644 --- a/package/secubox/secubox-app-mitmproxy/files/etc/config/mitmproxy +++ b/package/secubox/secubox-app-mitmproxy/files/etc/config/mitmproxy @@ -15,7 +15,25 @@ config mitmproxy 'main' option anticomp '0' option flow_detail '1' -# Transparent mode settings +# WAN Protection Mode - protect services exposed to internet +# Acts as WAF/reverse proxy for incoming WAN traffic +config wan_protection 'wan_protection' + # Enable WAN protection mode (acts as WAF for incoming traffic) + option enabled '0' + # WAN interface name (incoming traffic interface) + option wan_interface 'wan' + # Ports to intercept on WAN (HTTP) + option wan_http_port '80' + # Ports to intercept on WAN (HTTPS) + option wan_https_port '443' + # Feed detected threats to CrowdSec for automatic blocking + option crowdsec_feed '1' + # Block requests from known bot scanners immediately + option block_bots '0' + # Rate limiting: max requests per IP per minute (0=disabled) + option rate_limit '0' + +# LAN Transparent mode settings (outbound traffic interception) config transparent 'transparent' option enabled '0' # Interface to intercept traffic from (e.g., br-lan) @@ -29,6 +47,16 @@ config transparent 'transparent' # Custom HTTPS port (default 443) option https_port '443' +# DPI Mirror Mode - feed traffic to network inspection engines +config dpi_mirror 'dpi_mirror' + option enabled '0' + # Interface for DPI mirroring (netifyd/ndpid listens on this) + option dpi_interface 'br-lan' + # Enable DPI for WAN traffic (incoming) + option mirror_wan '0' + # Enable DPI for LAN traffic (outgoing) + option mirror_lan '0' + # Whitelist/bypass - IPs and domains that bypass the proxy config whitelist 'whitelist' option enabled '1' diff --git a/package/secubox/secubox-app-mitmproxy/files/usr/sbin/mitmproxyctl b/package/secubox/secubox-app-mitmproxy/files/usr/sbin/mitmproxyctl index 6874fd4b..c44b66fe 100755 --- a/package/secubox/secubox-app-mitmproxy/files/usr/sbin/mitmproxyctl +++ b/package/secubox/secubox-app-mitmproxy/files/usr/sbin/mitmproxyctl @@ -6,6 +6,7 @@ CONFIG="mitmproxy" LXC_NAME="mitmproxy" OPKG_UPDATED=0 NFT_TABLE="mitmproxy" +NFT_TABLE_WAN="mitmproxy_wan" # Paths LXC_PATH="/srv/lxc" @@ -27,6 +28,8 @@ Commands: cert Show CA certificate info / export path firewall-setup Setup nftables rules for transparent mode firewall-clear Remove nftables transparent mode rules + wan-setup Setup WAN protection mode (incoming traffic) + wan-clear Remove WAN protection mode rules sync-routes Sync HAProxy backends to mitmproxy routes haproxy-enable Enable HAProxy backend inspection mode haproxy-disable Disable HAProxy backend inspection mode @@ -39,6 +42,11 @@ Modes (configure in /etc/config/mitmproxy): upstream - Forward to upstream proxy reverse - Reverse proxy mode +WAN Protection Mode: + Intercept incoming WAN traffic for threat detection (WAF mode). + Detects bot scanners, attacks, and feeds to CrowdSec. + Use 'mitmproxyctl wan-setup' to enable. + HAProxy Integration: When enabled, HAProxy backends route through mitmproxy for threat detection. Use 'mitmproxyctl haproxy-enable' to setup. @@ -75,7 +83,16 @@ load_config() { anticomp="$(uci_get main.anticomp || echo 0)" flow_detail="$(uci_get main.flow_detail || echo 1)" - # Transparent mode settings + # WAN protection mode settings + wan_protection_enabled="$(uci_get wan_protection.enabled || echo 0)" + wan_interface="$(uci_get wan_protection.wan_interface || echo wan)" + wan_http_port="$(uci_get wan_protection.wan_http_port || echo 80)" + wan_https_port="$(uci_get wan_protection.wan_https_port || echo 443)" + crowdsec_feed="$(uci_get wan_protection.crowdsec_feed || echo 1)" + block_bots="$(uci_get wan_protection.block_bots || echo 0)" + rate_limit="$(uci_get wan_protection.rate_limit || echo 0)" + + # Transparent mode settings (LAN) transparent_enabled="$(uci_get transparent.enabled || echo 0)" transparent_iface="$(uci_get transparent.interface || echo br-lan)" redirect_http="$(uci_get transparent.redirect_http || echo 1)" @@ -83,6 +100,12 @@ load_config() { http_port="$(uci_get transparent.http_port || echo 80)" https_port="$(uci_get transparent.https_port || echo 443)" + # DPI mirror settings + dpi_mirror_enabled="$(uci_get dpi_mirror.enabled || echo 0)" + dpi_interface="$(uci_get dpi_mirror.dpi_interface || echo br-lan)" + mirror_wan="$(uci_get dpi_mirror.mirror_wan || echo 0)" + mirror_lan="$(uci_get dpi_mirror.mirror_lan || echo 0)" + # Whitelist settings whitelist_enabled="$(uci_get whitelist.enabled || echo 1)" @@ -246,8 +269,135 @@ nft_status() { echo "Bypass IPv6 set:" nft list set inet $NFT_TABLE bypass_ipv6 2>/dev/null || echo " (empty or not created)" else - echo "No mitmproxy rules configured" + echo "No mitmproxy LAN transparent rules configured" fi + + echo "" + echo "=== WAN protection rules ===" + if nft list table inet $NFT_TABLE_WAN 2>/dev/null; then + echo "WAN protection mode active" + else + echo "No WAN protection rules configured" + fi +} + +# ============================================================================= +# WAN PROTECTION MODE FUNCTIONS +# ============================================================================= + +# Get the actual network interface name from UCI interface name +get_wan_device() { + local iface="$1" + local device="" + + # Try to get device from network config + device=$(uci -q get network.${iface}.device) + [ -n "$device" ] && { echo "$device"; return 0; } + + # Try ifname (older OpenWrt) + device=$(uci -q get network.${iface}.ifname) + [ -n "$device" ] && { echo "$device"; return 0; } + + # Fallback: check if interface exists directly + if ip link show "$iface" >/dev/null 2>&1; then + echo "$iface" + return 0 + fi + + # Common WAN interface names + for dev in eth1 eth0.2 wan pppoe-wan; do + if ip link show "$dev" >/dev/null 2>&1; then + echo "$dev" + return 0 + fi + done + + return 1 +} + +nft_wan_setup() { + load_config + require_root + + if ! has_nft; then + log_error "nftables not available" + return 1 + fi + + if [ "$wan_protection_enabled" != "1" ]; then + log_warn "WAN protection mode is disabled. Enable with: uci set mitmproxy.wan_protection.enabled='1'" + return 0 + fi + + log_info "Setting up nftables for WAN protection mode..." + + # Get actual WAN device name + local wan_dev=$(get_wan_device "$wan_interface") + if [ -z "$wan_dev" ]; then + log_error "Could not determine WAN device for interface '$wan_interface'" + return 1 + fi + log_info "WAN device: $wan_dev (from interface: $wan_interface)" + + # Create WAN protection table + nft add table inet $NFT_TABLE_WAN 2>/dev/null || true + + # Create prerouting chain for incoming WAN traffic + nft add chain inet $NFT_TABLE_WAN prerouting { type nat hook prerouting priority -100 \; } 2>/dev/null || true + + # Flush existing rules + nft flush chain inet $NFT_TABLE_WAN prerouting 2>/dev/null || true + + # Don't redirect traffic destined to router itself (management) + local router_ip=$(uci -q get network.lan.ipaddr || echo "192.168.1.1") + nft add rule inet $NFT_TABLE_WAN prerouting ip daddr "$router_ip" return 2>/dev/null || true + + # Skip private/local destinations (not exposed to WAN) + nft add rule inet $NFT_TABLE_WAN prerouting ip daddr 10.0.0.0/8 return 2>/dev/null || true + nft add rule inet $NFT_TABLE_WAN prerouting ip daddr 172.16.0.0/12 return 2>/dev/null || true + nft add rule inet $NFT_TABLE_WAN prerouting ip daddr 192.168.0.0/16 return 2>/dev/null || true + nft add rule inet $NFT_TABLE_WAN prerouting ip daddr 127.0.0.0/8 return 2>/dev/null || true + + # Redirect WAN incoming HTTP to mitmproxy + if [ "$wan_http_port" != "0" ]; then + nft add rule inet $NFT_TABLE_WAN prerouting iifname "$wan_dev" tcp dport "$wan_http_port" redirect to :$proxy_port + log_info "WAN HTTP redirect: $wan_dev:$wan_http_port -> mitmproxy:$proxy_port" + fi + + # Redirect WAN incoming HTTPS to mitmproxy + if [ "$wan_https_port" != "0" ]; then + nft add rule inet $NFT_TABLE_WAN prerouting iifname "$wan_dev" tcp dport "$wan_https_port" redirect to :$proxy_port + log_info "WAN HTTPS redirect: $wan_dev:$wan_https_port -> mitmproxy:$proxy_port" + fi + + # Optional: Add rate limiting if configured + if [ "$rate_limit" -gt 0 ] 2>/dev/null; then + log_info "Rate limiting: $rate_limit requests/minute per IP" + # Note: Rate limiting in nftables requires meter/limit + # This is a basic implementation - can be enhanced + nft add chain inet $NFT_TABLE_WAN ratelimit 2>/dev/null || true + fi + + log_info "WAN protection mode rules applied" + log_info "Table: inet $NFT_TABLE_WAN" + log_info "" + log_info "Incoming WAN traffic on ports $wan_http_port/$wan_https_port will be" + log_info "inspected by mitmproxy for threats before reaching backend services." +} + +nft_wan_teardown() { + require_root + + if ! has_nft; then + return 0 + fi + + log_info "Removing WAN protection mode rules..." + + # Delete the WAN protection table + nft delete table inet $NFT_TABLE_WAN 2>/dev/null || true + + log_info "WAN protection rules removed" } # ============================================================================= @@ -707,16 +857,22 @@ lxc_run() { ensure_dir "$data_path" ensure_dir "$ADDON_PATH" - # Setup firewall rules if in transparent mode + # Setup firewall rules if in transparent mode (LAN) if [ "$mode" = "transparent" ]; then nft_setup fi + # Setup WAN protection rules if enabled + if [ "$wan_protection_enabled" = "1" ]; then + nft_wan_setup + fi + log_info "Starting mitmproxy LXC container..." log_info "Mode: $mode" log_info "Web interface: http://0.0.0.0:$web_port" log_info "Proxy port: $proxy_port" [ "$filtering_enabled" = "1" ] && log_info "Filtering: enabled" + [ "$wan_protection_enabled" = "1" ] && log_info "WAN Protection: enabled (interface: $wan_interface)" exec lxc-start -n "$LXC_NAME" -F -f "$LXC_CONFIG" } @@ -739,7 +895,18 @@ lxc_status() { echo "Data path: $data_path" echo "Filtering: $([ "$filtering_enabled" = "1" ] && echo "enabled" || echo "disabled")" - if [ "$mode" = "transparent" ]; then + echo "" + echo "=== WAN Protection ===" + echo "Enabled: $([ "$wan_protection_enabled" = "1" ] && echo "yes" || echo "no")" + if [ "$wan_protection_enabled" = "1" ]; then + echo "WAN Interface: $wan_interface" + echo "HTTP Port: $wan_http_port" + echo "HTTPS Port: $wan_https_port" + echo "CrowdSec Feed: $([ "$crowdsec_feed" = "1" ] && echo "yes" || echo "no")" + echo "Block Bots: $([ "$block_bots" = "1" ] && echo "yes" || echo "no")" + fi + + if [ "$mode" = "transparent" ] || [ "$wan_protection_enabled" = "1" ]; then echo "" nft_status fi @@ -907,9 +1074,20 @@ cmd_service_stop() { nft_teardown fi + # Remove WAN protection rules + nft_wan_teardown 2>/dev/null || true + lxc_stop } +cmd_wan_setup() { + nft_wan_setup +} + +cmd_wan_clear() { + nft_wan_teardown +} + # ============================================================================= # HAPROXY BACKEND INSPECTION # ============================================================================= @@ -1120,6 +1298,8 @@ case "${1:-}" in cert) shift; cmd_cert "$@" ;; firewall-setup) shift; cmd_firewall_setup "$@" ;; firewall-clear) shift; cmd_firewall_clear "$@" ;; + wan-setup) shift; cmd_wan_setup "$@" ;; + wan-clear) shift; cmd_wan_clear "$@" ;; sync-routes) shift; cmd_sync_routes "$@" ;; haproxy-enable) shift; cmd_haproxy_enable "$@" ;; haproxy-disable) shift; cmd_haproxy_disable "$@" ;; diff --git a/package/secubox/secubox-app-mitmproxy/root/srv/mitmproxy/addons/secubox_analytics.py b/package/secubox/secubox-app-mitmproxy/root/srv/mitmproxy/addons/secubox_analytics.py index 00725455..faa3e727 100644 --- a/package/secubox/secubox-app-mitmproxy/root/srv/mitmproxy/addons/secubox_analytics.py +++ b/package/secubox/secubox-app-mitmproxy/root/srv/mitmproxy/addons/secubox_analytics.py @@ -172,23 +172,80 @@ AUTH_PATHS = [ BOT_SIGNATURES = [ # Generic bots 'bot', 'crawler', 'spider', 'scraper', 'scan', - # HTTP clients + # HTTP clients (often used by scanners) 'curl', 'wget', 'python-requests', 'python-urllib', 'httpx', 'go-http-client', 'java/', 'axios', 'node-fetch', 'got/', 'okhttp', 'apache-httpclient', 'guzzlehttp', 'libwww-perl', - # Security scanners + + # ==== VULNERABILITY SCANNERS ==== 'zgrab', 'masscan', 'nmap', 'nikto', 'nuclei', 'sqlmap', + 'censys', 'shodan', 'internetmeasurement', 'binaryedge', 'leakix', + 'onyphe', 'criminalip', 'netcraft', 'greynoise', + + # ==== WEB DIRECTORY SCANNERS ==== 'dirb', 'dirbuster', 'gobuster', 'ffuf', 'wfuzz', 'feroxbuster', + 'skipfish', 'whatweb', 'wpscan', 'joomscan', 'droopescan', + 'drupwn', 'cmsmap', 'vbscan', + + # ==== EXPLOITATION TOOLS ==== 'burpsuite', 'owasp', 'acunetix', 'nessus', 'qualys', 'openvas', - 'w3af', 'arachni', 'skipfish', 'vega', 'zap', 'appscan', - 'webinspect', 'metasploit', 'hydra', 'medusa', - # Known bad bots + 'w3af', 'arachni', 'vega', 'zap', 'appscan', + 'webinspect', 'metasploit', 'hydra', 'medusa', 'cobalt', + 'havij', 'commix', 'tplmap', 'xsstrike', 'dalfox', + + # ==== GENERIC SUSPICIOUS PATTERNS ==== + 'scanner', 'exploit', 'attack', 'hack', 'pwn', + 'fuzz', 'brute', 'inject', 'payload', 'pentest', + + # ==== KNOWN BAD BOTS ==== 'ahrefsbot', 'semrushbot', 'dotbot', 'mj12bot', 'blexbot', 'seznambot', 'yandexbot', 'baiduspider', 'sogou', - # Empty or suspicious UAs + 'bytespider', 'petalbot', 'dataforseo', 'serpstatbot', + + # ==== EMPTY/SUSPICIOUS USER AGENTS ==== '-', '', 'mozilla/4.0', 'mozilla/5.0', ] +# Behavioral patterns for bot detection (request path based) +BOT_BEHAVIOR_PATHS = [ + # Credential/config file hunting + r'/\.git/config', r'/\.git/HEAD', r'/\.gitignore', + r'/\.env', r'/\.env\.local', r'/\.env\.production', + r'/\.aws/credentials', r'/\.docker/config\.json', + r'/wp-config\.php\.bak', r'/config\.php\.old', r'/config\.php\.save', + r'/\.npmrc', r'/\.pypirc', r'/\.netrc', + + # Admin panel hunting + r'/administrator', r'/wp-login\.php', r'/wp-admin', + r'/phpmyadmin', r'/pma', r'/myadmin', r'/mysql', + r'/cpanel', r'/webmail', r'/admin', r'/manager', + r'/login', r'/signin', r'/dashboard', + + # Backup file hunting + r'\.sql\.gz$', r'\.sql\.bz2$', r'\.sql\.zip$', + r'\.tar\.gz$', r'\.tar\.bz2$', r'\.zip$', r'\.rar$', + r'\.bak$', r'\.old$', r'\.backup$', r'\.orig$', + r'/backup', r'/dump', r'/export', r'/db\.sql', + + # Shell/webshell hunting + r'/c99\.php', r'/r57\.php', r'/shell\.php', r'/cmd\.php', + r'/exec\.php', r'/webshell', r'/backdoor', r'/b374k', + r'\.php\?cmd=', r'\.php\?c=', r'\.asp\?cmd=', + + # API/endpoint discovery + r'/api/v\d+', r'/rest/', r'/graphql', r'/swagger', + r'/api-docs', r'/_cat/', r'/_cluster/', r'/actuator', + r'/__debug__', r'/debug/', r'/trace/', r'/metrics', +] + +# Rate limiting thresholds for different attack patterns +RATE_LIMITS = { + 'path_scan': {'window': 60, 'max': 20}, # 20 scans per minute + 'auth_attempt': {'window': 60, 'max': 10}, # 10 auth attempts per minute + 'bot_request': {'window': 60, 'max': 30}, # 30 bot requests per minute + 'normal': {'window': 60, 'max': 100}, # 100 normal requests per minute +} + # Suspicious headers indicating attack tools SUSPICIOUS_HEADERS = { 'x-forwarded-for': [r'\d+\.\d+\.\d+\.\d+.*,.*,.*,'], # Multiple proxies @@ -308,29 +365,108 @@ class SecuBoxAnalytics: fp_str = f"{ua}|{accept}|{accept_lang}|{accept_enc}" fp_hash = hashlib.md5(fp_str.encode()).hexdigest()[:12] - # Detect bot - is_bot = any(sig in ua.lower() for sig in BOT_SIGNATURES) + # Detect bot from user agent + ua_lower = ua.lower() + is_bot = any(sig in ua_lower for sig in BOT_SIGNATURES) + + # Additional bot detection heuristics + bot_type = None + if is_bot: + # Categorize the bot + if any(s in ua_lower for s in ['masscan', 'zgrab', 'censys', 'shodan', 'nmap']): + bot_type = 'port_scanner' + elif any(s in ua_lower for s in ['nikto', 'nuclei', 'acunetix', 'nessus', 'qualys']): + bot_type = 'vulnerability_scanner' + elif any(s in ua_lower for s in ['dirb', 'gobuster', 'ffuf', 'wfuzz', 'feroxbuster']): + bot_type = 'directory_scanner' + elif any(s in ua_lower for s in ['sqlmap', 'havij', 'commix']): + bot_type = 'injection_tool' + elif any(s in ua_lower for s in ['wpscan', 'joomscan', 'droopescan', 'cmsmap']): + bot_type = 'cms_scanner' + elif any(s in ua_lower for s in ['metasploit', 'cobalt', 'hydra', 'medusa']): + bot_type = 'exploitation_tool' + elif any(s in ua_lower for s in ['curl', 'wget', 'python', 'go-http', 'java/']): + bot_type = 'http_client' + else: + bot_type = 'generic_bot' + + # Suspicious UA patterns (empty, minimal, or clearly fake) + is_suspicious_ua = False + if not ua or ua == '-' or len(ua) < 10: + is_suspicious_ua = True + elif ua.lower() in ['mozilla/4.0', 'mozilla/5.0']: + is_suspicious_ua = True + elif not accept_lang and not accept_enc: + # Real browsers always send these + is_suspicious_ua = True # Parse UA for device info device = 'unknown' - if 'mobile' in ua.lower() or 'android' in ua.lower(): + if 'mobile' in ua_lower or 'android' in ua_lower: device = 'mobile' - elif 'iphone' in ua.lower() or 'ipad' in ua.lower(): + elif 'iphone' in ua_lower or 'ipad' in ua_lower: device = 'ios' - elif 'windows' in ua.lower(): + elif 'windows' in ua_lower: device = 'windows' - elif 'mac' in ua.lower(): + elif 'mac' in ua_lower: device = 'macos' - elif 'linux' in ua.lower(): + elif 'linux' in ua_lower: device = 'linux' return { 'fingerprint': fp_hash, 'user_agent': ua[:200], 'is_bot': is_bot, + 'bot_type': bot_type, + 'is_suspicious_ua': is_suspicious_ua, 'device': device } + def _detect_bot_behavior(self, request: http.Request) -> dict: + """Detect bot-like behavior based on request patterns""" + path = request.path.lower() + + for pattern in BOT_BEHAVIOR_PATHS: + if re.search(pattern, path, re.IGNORECASE): + # Categorize the behavior + if any(p in pattern for p in [r'\.git', r'\.env', r'\.aws', r'config', r'credential']): + return { + 'is_bot_behavior': True, + 'behavior_type': 'config_hunting', + 'pattern': pattern, + 'severity': 'high' + } + elif any(p in pattern for p in ['admin', 'login', 'cpanel', 'phpmyadmin']): + return { + 'is_bot_behavior': True, + 'behavior_type': 'admin_hunting', + 'pattern': pattern, + 'severity': 'medium' + } + elif any(p in pattern for p in ['backup', r'\.sql', r'\.tar', r'\.zip', 'dump']): + return { + 'is_bot_behavior': True, + 'behavior_type': 'backup_hunting', + 'pattern': pattern, + 'severity': 'high' + } + elif any(p in pattern for p in ['shell', 'cmd', 'exec', 'backdoor', 'c99', 'r57']): + return { + 'is_bot_behavior': True, + 'behavior_type': 'shell_hunting', + 'pattern': pattern, + 'severity': 'critical' + } + elif any(p in pattern for p in ['api', 'swagger', 'graphql', 'actuator']): + return { + 'is_bot_behavior': True, + 'behavior_type': 'api_discovery', + 'pattern': pattern, + 'severity': 'low' + } + + return {'is_bot_behavior': False, 'behavior_type': None, 'pattern': None, 'severity': None} + def _detect_scan(self, request: http.Request) -> dict: """Comprehensive threat detection with categorized patterns""" path = request.path.lower() @@ -553,25 +689,63 @@ class SecuBoxAnalytics: # CrowdSec compatible log (enhanced format) scan_data = entry.get('scan', {}) - if scan_data.get('is_scan') or entry.get('is_auth_attempt') or entry.get('suspicious_headers') or entry.get('rate_limit', {}).get('is_limited'): + bot_behavior_data = entry.get('bot_behavior', {}) + client_data = entry.get('client', {}) + + # Log to CrowdSec if any threat indicator is present + should_log = ( + scan_data.get('is_scan') or + bot_behavior_data.get('is_bot_behavior') or + client_data.get('is_bot') or + entry.get('is_auth_attempt') or + entry.get('suspicious_headers') or + entry.get('rate_limit', {}).get('is_limited') + ) + + if should_log: try: + # Determine the primary threat type for categorization + threat_type = 'suspicious' + if scan_data.get('is_scan'): + threat_type = scan_data.get('type', 'scan') + elif bot_behavior_data.get('is_bot_behavior'): + threat_type = bot_behavior_data.get('behavior_type', 'bot_behavior') + elif client_data.get('is_bot'): + threat_type = client_data.get('bot_type', 'bot') + elif entry.get('is_auth_attempt'): + threat_type = 'auth_attempt' + + # Determine severity + severity = 'low' + if scan_data.get('severity'): + severity = scan_data.get('severity') + elif bot_behavior_data.get('severity'): + severity = bot_behavior_data.get('severity') + elif client_data.get('bot_type') in ['exploitation_tool', 'injection_tool']: + severity = 'high' + elif client_data.get('bot_type') in ['vulnerability_scanner', 'directory_scanner']: + severity = 'medium' + cs_entry = { 'timestamp': entry['timestamp'], 'source_ip': entry['client_ip'], 'country': entry['country'], 'request': f"{entry['method']} {entry['path']}", 'host': entry.get('host', ''), - 'user_agent': entry['client'].get('user_agent', ''), - 'type': scan_data.get('type') or ('auth_attempt' if entry['is_auth_attempt'] else 'suspicious'), - 'pattern': scan_data.get('pattern', ''), - 'category': scan_data.get('category', ''), - 'severity': scan_data.get('severity', 'low'), + 'user_agent': client_data.get('user_agent', ''), + 'type': threat_type, + 'pattern': scan_data.get('pattern') or bot_behavior_data.get('pattern', ''), + 'category': scan_data.get('category') or bot_behavior_data.get('behavior_type', ''), + 'severity': severity, 'cve': scan_data.get('cve', ''), 'response_code': entry.get('response', {}).get('status', 0), - 'fingerprint': entry['client'].get('fingerprint', ''), - 'is_bot': entry['client'].get('is_bot', False), + 'fingerprint': client_data.get('fingerprint', ''), + 'is_bot': client_data.get('is_bot', False), + 'bot_type': client_data.get('bot_type', ''), + 'bot_behavior': bot_behavior_data.get('behavior_type', ''), 'rate_limited': entry.get('rate_limit', {}).get('is_limited', False), 'suspicious_headers': len(entry.get('suspicious_headers', [])) > 0, + 'suspicious_ua': client_data.get('is_suspicious_ua', False), } with open(CROWDSEC_LOG, 'a') as f: f.write(json.dumps(cs_entry) + '\n') @@ -644,6 +818,7 @@ class SecuBoxAnalytics: suspicious_headers = self._detect_suspicious_headers(request) rate_limit = self._check_rate_limit(source_ip) client_fp = self._get_client_fingerprint(request) + bot_behavior = self._detect_bot_behavior(request) # Build log entry entry = { @@ -658,6 +833,7 @@ class SecuBoxAnalytics: 'query': request.query.get('q', '')[:100] if request.query else '', 'client': client_fp, 'scan': scan_result, + 'bot_behavior': bot_behavior, 'is_auth_attempt': self._is_auth_attempt(request), 'content_length': len(request.content) if request.content else 0, 'routing': routing, @@ -722,6 +898,32 @@ class SecuBoxAnalytics: 'host': request.host }) + # Log bot behavior detection + if bot_behavior.get('is_bot_behavior'): + behavior_type = bot_behavior.get('behavior_type', 'unknown') + severity = bot_behavior.get('severity', 'medium') + + log_msg = f"BOT BEHAVIOR [{severity.upper()}]: {source_ip} ({entry['country']}) - {behavior_type}" + log_msg += f" - {request.method} {request.path}" + + if severity in ['critical', 'high']: + ctx.log.warn(log_msg) + else: + ctx.log.info(log_msg) + + self._add_alert({ + 'time': entry['timestamp'], + 'ip': source_ip, + 'country': entry['country'], + 'type': 'bot_behavior', + 'behavior_type': behavior_type, + 'severity': severity, + 'path': request.path, + 'method': request.method, + 'host': request.host, + 'bot_type': client_fp.get('bot_type') + }) + # Log suspicious headers if suspicious_headers: ctx.log.warn(f"SUSPICIOUS HEADERS: {source_ip} - {[h['header'] for h in suspicious_headers]}")