feat(mitmproxy): Add headless mode to reduce RAM from 3.4GB to 96MB
- Add headless UCI option to use mitmdump instead of mitmweb - Enable headless by default for WAF (mitmproxy-in) instance - Increase default memory limit from 256MB to 2GB - Fix LXC config generation to always recreate on service start - Fix rootfs check path (/usr/local/bin not /usr/bin) - Use exec in startup script for proper foreground execution Headless mode runs mitmdump (CLI) instead of mitmweb (web UI), saving ~3.3GB RAM while maintaining full WAF functionality. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
3fcad8e626
commit
ece237d194
@ -6,8 +6,10 @@ config mitmproxy 'main'
|
|||||||
option web_port '8081'
|
option web_port '8081'
|
||||||
option web_host '0.0.0.0'
|
option web_host '0.0.0.0'
|
||||||
option data_path '/srv/mitmproxy'
|
option data_path '/srv/mitmproxy'
|
||||||
option memory_limit '256M'
|
option memory_limit '2G'
|
||||||
option mode 'regular'
|
option mode 'regular'
|
||||||
|
# Headless mode: use mitmproxy instead of mitmweb (saves ~3GB RAM)
|
||||||
|
option headless '0'
|
||||||
option ssl_insecure '0'
|
option ssl_insecure '0'
|
||||||
option anticache '0'
|
option anticache '0'
|
||||||
option anticomp '0'
|
option anticomp '0'
|
||||||
@ -22,11 +24,13 @@ config instance 'out'
|
|||||||
option web_port '8089'
|
option web_port '8089'
|
||||||
option web_host '0.0.0.0'
|
option web_host '0.0.0.0'
|
||||||
option data_path '/srv/mitmproxy-out'
|
option data_path '/srv/mitmproxy-out'
|
||||||
option memory_limit '256M'
|
option memory_limit '2G'
|
||||||
option mode 'transparent'
|
option mode 'transparent'
|
||||||
option ssl_insecure '0'
|
option ssl_insecure '0'
|
||||||
option anticache '0'
|
option anticache '0'
|
||||||
option anticomp '0'
|
option anticomp '0'
|
||||||
|
# Headless mode: use mitmproxy instead of mitmweb (saves ~3GB RAM)
|
||||||
|
option headless '0'
|
||||||
|
|
||||||
# IN Instance - WAN to Services (WAF/reverse proxy)
|
# IN Instance - WAN to Services (WAF/reverse proxy)
|
||||||
config instance 'in'
|
config instance 'in'
|
||||||
@ -37,13 +41,16 @@ config instance 'in'
|
|||||||
option web_port '8090'
|
option web_port '8090'
|
||||||
option web_host '0.0.0.0'
|
option web_host '0.0.0.0'
|
||||||
option data_path '/srv/mitmproxy-in'
|
option data_path '/srv/mitmproxy-in'
|
||||||
option memory_limit '256M'
|
option memory_limit '2G'
|
||||||
option mode 'upstream'
|
option mode 'upstream'
|
||||||
option ssl_insecure '0'
|
option ssl_insecure '0'
|
||||||
option anticache '0'
|
option anticache '0'
|
||||||
option anticomp '0'
|
option anticomp '0'
|
||||||
# HAProxy sends traffic here
|
# HAProxy sends traffic here
|
||||||
option haproxy_backend '1'
|
option haproxy_backend '1'
|
||||||
|
# Headless mode: use mitmproxy instead of mitmweb (saves ~3GB RAM)
|
||||||
|
# Recommended for production WAF to reduce memory usage
|
||||||
|
option headless '1'
|
||||||
|
|
||||||
# WAN Protection Mode - protect services exposed to internet
|
# WAN Protection Mode - protect services exposed to internet
|
||||||
# Acts as WAF/reverse proxy for incoming WAN traffic
|
# Acts as WAF/reverse proxy for incoming WAN traffic
|
||||||
|
|||||||
@ -170,6 +170,7 @@ load_instance_config() {
|
|||||||
anticomp="$(uci_get ${inst}.anticomp || echo 0)"
|
anticomp="$(uci_get ${inst}.anticomp || echo 0)"
|
||||||
flow_detail="$(uci_get ${inst}.flow_detail || echo 1)"
|
flow_detail="$(uci_get ${inst}.flow_detail || echo 1)"
|
||||||
haproxy_backend="$(uci_get ${inst}.haproxy_backend || echo 0)"
|
haproxy_backend="$(uci_get ${inst}.haproxy_backend || echo 0)"
|
||||||
|
headless="$(uci_get ${inst}.headless || echo 0)"
|
||||||
else
|
else
|
||||||
# Legacy single-instance mode
|
# Legacy single-instance mode
|
||||||
INSTANCE=""
|
INSTANCE=""
|
||||||
@ -178,7 +179,7 @@ load_instance_config() {
|
|||||||
web_port="$(uci_get main.web_port || echo 8081)"
|
web_port="$(uci_get main.web_port || echo 8081)"
|
||||||
web_host="$(uci_get main.web_host || echo 0.0.0.0)"
|
web_host="$(uci_get main.web_host || echo 0.0.0.0)"
|
||||||
data_path="$(uci_get main.data_path || echo /srv/mitmproxy)"
|
data_path="$(uci_get main.data_path || echo /srv/mitmproxy)"
|
||||||
memory_limit="$(uci_get main.memory_limit || echo 256M)"
|
memory_limit="$(uci_get main.memory_limit || echo 2G)"
|
||||||
mode="$(uci_get main.mode || echo regular)"
|
mode="$(uci_get main.mode || echo regular)"
|
||||||
upstream_proxy="$(uci_get main.upstream_proxy || echo '')"
|
upstream_proxy="$(uci_get main.upstream_proxy || echo '')"
|
||||||
reverse_target="$(uci_get main.reverse_target || echo '')"
|
reverse_target="$(uci_get main.reverse_target || echo '')"
|
||||||
@ -187,6 +188,7 @@ load_instance_config() {
|
|||||||
anticomp="$(uci_get main.anticomp || echo 0)"
|
anticomp="$(uci_get main.anticomp || echo 0)"
|
||||||
flow_detail="$(uci_get main.flow_detail || echo 1)"
|
flow_detail="$(uci_get main.flow_detail || echo 1)"
|
||||||
haproxy_backend="0"
|
haproxy_backend="0"
|
||||||
|
headless="$(uci_get main.headless || echo 0)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Set derived paths
|
# Set derived paths
|
||||||
@ -555,7 +557,7 @@ lxc_check_prereqs() {
|
|||||||
lxc_create_rootfs() {
|
lxc_create_rootfs() {
|
||||||
load_config
|
load_config
|
||||||
|
|
||||||
if [ -d "$LXC_ROOTFS" ] && [ -x "$LXC_ROOTFS/usr/bin/mitmproxy" ]; then
|
if [ -d "$LXC_ROOTFS" ] && [ -x "$LXC_ROOTFS/usr/local/bin/mitmproxy" ]; then
|
||||||
log_info "LXC rootfs already exists with mitmproxy"
|
log_info "LXC rootfs already exists with mitmproxy"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
@ -672,6 +674,9 @@ HAPROXY_ROUTER_ENABLED="${MITMPROXY_HAPROXY_ROUTER_ENABLED:-0}"
|
|||||||
HAPROXY_LISTEN_PORT="${MITMPROXY_HAPROXY_LISTEN_PORT:-8890}"
|
HAPROXY_LISTEN_PORT="${MITMPROXY_HAPROXY_LISTEN_PORT:-8890}"
|
||||||
HAPROXY_ROUTES_FILE="${MITMPROXY_HAPROXY_ROUTES_FILE:-/data/haproxy-routes.json}"
|
HAPROXY_ROUTES_FILE="${MITMPROXY_HAPROXY_ROUTES_FILE:-/data/haproxy-routes.json}"
|
||||||
|
|
||||||
|
# Headless mode (use mitmproxy instead of mitmweb to save ~3GB RAM)
|
||||||
|
HEADLESS="${MITMPROXY_HEADLESS:-0}"
|
||||||
|
|
||||||
# Build args
|
# Build args
|
||||||
ARGS="--listen-host 0.0.0.0 --listen-port $PROXY_PORT --set confdir=/data"
|
ARGS="--listen-host 0.0.0.0 --listen-port $PROXY_PORT --set confdir=/data"
|
||||||
ARGS="$ARGS --web-host $WEB_HOST --web-port $WEB_PORT --no-web-open-browser"
|
ARGS="$ARGS --web-host $WEB_HOST --web-port $WEB_PORT --no-web-open-browser"
|
||||||
@ -705,44 +710,60 @@ if [ "$FILTERING_ENABLED" = "1" ] && [ -n "$ADDON_SCRIPT" ] && [ -f "$ADDON_SCRI
|
|||||||
echo "Loading addon: $ADDON_SCRIPT"
|
echo "Loading addon: $ADDON_SCRIPT"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
rm -f /data/.mitmproxy_token /tmp/mitmweb.log
|
rm -f /data/.mitmproxy_token /tmp/mitmweb.log /tmp/mitmproxy.log
|
||||||
|
|
||||||
echo "Starting mitmweb..."
|
# Choose between headless mitmproxy and mitmweb
|
||||||
echo "Command: mitmweb $ARGS"
|
if [ "$HEADLESS" = "1" ]; then
|
||||||
|
echo "Starting mitmproxy (headless mode - no web UI, ~3GB RAM saved)..."
|
||||||
|
echo "Command: mitmdump $ARGS"
|
||||||
|
|
||||||
# Start mitmweb in background, output to log file
|
# mitmdump is the headless CLI version
|
||||||
/usr/local/bin/mitmweb $ARGS 2>&1 | tee /tmp/mitmweb.log &
|
/usr/local/bin/mitmdump $ARGS 2>&1 | tee /tmp/mitmproxy.log &
|
||||||
MITMWEB_PID=$!
|
MITMPROXY_PID=$!
|
||||||
|
|
||||||
# Wait for token to appear in log (with timeout)
|
echo "mitmproxy running in headless mode (PID: $MITMPROXY_PID)"
|
||||||
echo "Waiting for authentication token..."
|
echo "No web UI available - use /tmp/mitmproxy.log for logs"
|
||||||
ATTEMPTS=0
|
|
||||||
MAX_ATTEMPTS=30
|
|
||||||
while [ $ATTEMPTS -lt $MAX_ATTEMPTS ]; do
|
|
||||||
sleep 1
|
|
||||||
ATTEMPTS=$((ATTEMPTS + 1))
|
|
||||||
|
|
||||||
if [ -f /tmp/mitmweb.log ]; then
|
# Wait for process to keep container running
|
||||||
# Extract token from log - mitmweb outputs: "Web server listening at http://x.x.x.x:8081/?token=XXXXX"
|
wait $MITMPROXY_PID
|
||||||
# Token can be alphanumeric, not just hex
|
else
|
||||||
TOKEN=$(grep -o 'token=[a-zA-Z0-9_-]*' /tmp/mitmweb.log 2>/dev/null | head -1 | cut -d= -f2)
|
echo "Starting mitmweb (web UI mode)..."
|
||||||
if [ -n "$TOKEN" ]; then
|
echo "Command: mitmweb $ARGS"
|
||||||
echo "$TOKEN" > /data/.mitmproxy_token
|
|
||||||
chmod 644 /data/.mitmproxy_token
|
# Start mitmweb in background, output to log file
|
||||||
echo "Token captured: $(echo "$TOKEN" | cut -c1-8)..."
|
/usr/local/bin/mitmweb $ARGS 2>&1 | tee /tmp/mitmweb.log &
|
||||||
echo "Web UI: http://$WEB_HOST:$WEB_PORT/?token=$TOKEN"
|
MITMWEB_PID=$!
|
||||||
break
|
|
||||||
|
# Wait for token to appear in log (with timeout)
|
||||||
|
echo "Waiting for authentication token..."
|
||||||
|
ATTEMPTS=0
|
||||||
|
MAX_ATTEMPTS=30
|
||||||
|
while [ $ATTEMPTS -lt $MAX_ATTEMPTS ]; do
|
||||||
|
sleep 1
|
||||||
|
ATTEMPTS=$((ATTEMPTS + 1))
|
||||||
|
|
||||||
|
if [ -f /tmp/mitmweb.log ]; then
|
||||||
|
# Extract token from log - mitmweb outputs: "Web server listening at http://x.x.x.x:8081/?token=XXXXX"
|
||||||
|
# Token can be alphanumeric, not just hex
|
||||||
|
TOKEN=$(grep -o 'token=[a-zA-Z0-9_-]*' /tmp/mitmweb.log 2>/dev/null | head -1 | cut -d= -f2)
|
||||||
|
if [ -n "$TOKEN" ]; then
|
||||||
|
echo "$TOKEN" > /data/.mitmproxy_token
|
||||||
|
chmod 644 /data/.mitmproxy_token
|
||||||
|
echo "Token captured: $(echo "$TOKEN" | cut -c1-8)..."
|
||||||
|
echo "Web UI: http://$WEB_HOST:$WEB_PORT/?token=$TOKEN"
|
||||||
|
break
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ ! -f /data/.mitmproxy_token ]; then
|
||||||
|
echo "Warning: Could not capture authentication token after ${MAX_ATTEMPTS}s"
|
||||||
|
echo "Check /tmp/mitmweb.log for details"
|
||||||
fi
|
fi
|
||||||
done
|
|
||||||
|
|
||||||
if [ ! -f /data/.mitmproxy_token ]; then
|
# Wait for mitmweb process to keep container running
|
||||||
echo "Warning: Could not capture authentication token after ${MAX_ATTEMPTS}s"
|
wait $MITMWEB_PID
|
||||||
echo "Check /tmp/mitmweb.log for details"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Wait for mitmweb process to keep container running
|
|
||||||
wait $MITMWEB_PID
|
|
||||||
START
|
START
|
||||||
chmod +x "$rootfs/opt/start-mitmproxy.sh"
|
chmod +x "$rootfs/opt/start-mitmproxy.sh"
|
||||||
|
|
||||||
@ -954,6 +975,9 @@ lxc.environment = MITMPROXY_HAPROXY_ROUTER_ENABLED=$haproxy_router_enabled
|
|||||||
lxc.environment = MITMPROXY_HAPROXY_LISTEN_PORT=$haproxy_listen_port
|
lxc.environment = MITMPROXY_HAPROXY_LISTEN_PORT=$haproxy_listen_port
|
||||||
lxc.environment = MITMPROXY_HAPROXY_ROUTES_FILE=/data/haproxy-routes.json
|
lxc.environment = MITMPROXY_HAPROXY_ROUTES_FILE=/data/haproxy-routes.json
|
||||||
|
|
||||||
|
# Headless mode (use mitmproxy instead of mitmweb to save ~3GB RAM)
|
||||||
|
lxc.environment = MITMPROXY_HEADLESS=$headless
|
||||||
|
|
||||||
# Capabilities - drop dangerous ones (sys_admin needed for some ops)
|
# Capabilities - drop dangerous ones (sys_admin needed for some ops)
|
||||||
lxc.cap.drop = sys_module mac_admin mac_override sys_time
|
lxc.cap.drop = sys_module mac_admin mac_override sys_time
|
||||||
|
|
||||||
@ -984,12 +1008,13 @@ lxc_run() {
|
|||||||
load_config
|
load_config
|
||||||
lxc_stop
|
lxc_stop
|
||||||
|
|
||||||
if [ ! -f "$LXC_CONFIG" ]; then
|
# Check rootfs exists (mitmproxy is at /usr/local/bin in the container)
|
||||||
log_error "LXC not configured. Run 'mitmproxyctl install' first."
|
if [ ! -d "$LXC_ROOTFS" ] || [ ! -x "$LXC_ROOTFS/usr/local/bin/mitmproxy" ]; then
|
||||||
|
log_error "LXC rootfs not installed. Run 'mitmproxyctl install' first."
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Regenerate config to pick up any UCI changes
|
# Create/regenerate config (always do this to pick up any UCI changes)
|
||||||
lxc_create_config
|
lxc_create_config
|
||||||
|
|
||||||
# Ensure mount points exist
|
# Ensure mount points exist
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user