From 8d08ccd4a491003e815a213ce5fdf39c617c9c24 Mon Sep 17 00:00:00 2001 From: CyberMind-FR Date: Wed, 28 Jan 2026 05:32:57 +0100 Subject: [PATCH] fix(service-registry): Fix RPC data handling and landing page permissions - Remove expect clause from RPC declarations to get raw response - Add proper error handling with catch blocks for all RPC calls - Fix landing page generator to chmod 644 after generation - Fixes "No Services Found" issue in dashboard - Fixes "Forbidden" error when accessing landing page Co-Authored-By: Claude Opus 4.5 --- .claude/settings.local.json | 4 +- .../resources/view/secubox-portal/index.js | 18 +- .../resources/service-registry/api.js | 12 +- .../usr/libexec/rpcd/luci.service-registry | 94 ++++++- .../root/usr/sbin/secubox-landing-gen | 3 + .../secubox/secubox-app-streamlit/README.md | 255 ++++++++++++++++++ .../files/usr/sbin/streamlitctl | 19 +- 7 files changed, 378 insertions(+), 27 deletions(-) create mode 100644 package/secubox/secubox-app-streamlit/README.md diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 995439b9..c3aa94e1 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -164,7 +164,9 @@ "Bash(git diff:*)", "Bash(git log:*)", "Bash(nc:*)", - "Bash(pkill:*)" + "Bash(pkill:*)", + "Bash(python3 -m json.tool:*)", + "Bash(git restore:*)" ] } } diff --git a/package/secubox/luci-app-secubox-portal/htdocs/luci-static/resources/view/secubox-portal/index.js b/package/secubox/luci-app-secubox-portal/htdocs/luci-static/resources/view/secubox-portal/index.js index 19a10ea0..9a110465 100644 --- a/package/secubox/luci-app-secubox-portal/htdocs/luci-static/resources/view/secubox-portal/index.js +++ b/package/secubox/luci-app-secubox-portal/htdocs/luci-static/resources/view/secubox-portal/index.js @@ -34,6 +34,12 @@ var callGetServices = rpc.declare({ expect: { services: [] } }); +var callDashboardData = rpc.declare({ + object: 'luci.secubox', + method: 'get_dashboard_data', + expect: { counts: {} } +}); + return view.extend({ currentSection: 'dashboard', appStatuses: {}, @@ -48,10 +54,13 @@ return view.extend({ callCrowdSecStats().catch(function() { return null; }), portal.checkInstalledApps(), callGetServices().catch(function() { return []; }), - callSecurityStats().catch(function() { return null; }) + callSecurityStats().catch(function() { return null; }), + callDashboardData().catch(function() { return { counts: {} }; }) ]).then(function(results) { // Store installed apps info from the last promise self.installedApps = results[4] || {}; + // Store dashboard counts from RPCD (reliable source) + self.dashboardCounts = results[7] || {}; // RPC expect unwraps the services array directly var svcResult = results[5] || []; self.detectedServices = Array.isArray(svcResult) ? svcResult : (svcResult.services || []); @@ -244,9 +253,10 @@ return view.extend({ var networkApps = portal.getAppsBySection('network'); var monitoringApps = portal.getAppsBySection('monitoring'); - // Count running services - var runningCount = Object.values(this.appStatuses).filter(function(s) { return s === 'running'; }).length; - var totalServices = Object.keys(this.appStatuses).length; + // Count running services - prefer RPCD counts (reliable), fallback to local check + var counts = this.dashboardCounts || {}; + var runningCount = counts.running || Object.values(this.appStatuses).filter(function(s) { return s === 'running'; }).length; + var totalServices = counts.total || Object.keys(this.appStatuses).length; // CrowdSec blocked IPs count var blockedIPv4 = (crowdSecStats.ipv4_total_count || 0); diff --git a/package/secubox/luci-app-service-registry/htdocs/luci-static/resources/service-registry/api.js b/package/secubox/luci-app-service-registry/htdocs/luci-static/resources/service-registry/api.js index abeac027..4e13a396 100644 --- a/package/secubox/luci-app-service-registry/htdocs/luci-static/resources/service-registry/api.js +++ b/package/secubox/luci-app-service-registry/htdocs/luci-static/resources/service-registry/api.js @@ -5,8 +5,7 @@ // RPC method declarations var callListServices = rpc.declare({ object: 'luci.service-registry', - method: 'list_services', - expect: { services: [], providers: {} } + method: 'list_services' }); var callGetService = rpc.declare({ @@ -65,8 +64,7 @@ var callGetQrData = rpc.declare({ var callListCategories = rpc.declare({ object: 'luci.service-registry', - method: 'list_categories', - expect: { categories: [] } + method: 'list_categories' }); var callGetCertificateStatus = rpc.declare({ @@ -179,9 +177,9 @@ return baseclass.extend({ // Get dashboard data (services + provider status) getDashboardData: function() { return Promise.all([ - callListServices(), - callListCategories(), - callGetLandingConfig(), + callListServices().catch(function(e) { console.error('list_services failed:', e); return { services: [], providers: {} }; }), + callListCategories().catch(function(e) { console.error('list_categories failed:', e); return { categories: [] }; }), + callGetLandingConfig().catch(function(e) { console.error('get_landing_config failed:', e); return {}; }), callHAProxyStatus().catch(function() { return { enabled: false }; }), callTorStatus().catch(function() { return { enabled: false }; }) ]).then(function(results) { diff --git a/package/secubox/luci-app-service-registry/root/usr/libexec/rpcd/luci.service-registry b/package/secubox/luci-app-service-registry/root/usr/libexec/rpcd/luci.service-registry index 58d9e8f3..b6bf6ebd 100644 --- a/package/secubox/luci-app-service-registry/root/usr/libexec/rpcd/luci.service-registry +++ b/package/secubox/luci-app-service-registry/root/usr/libexec/rpcd/luci.service-registry @@ -208,34 +208,78 @@ _aggregate_haproxy_services() { local lan_ip="$1" local tmp_file="$2" - # Call HAProxy RPCD to get vhosts - local vhosts_json + # Call HAProxy RPCD to get vhosts and backends + local vhosts_json backends_json certs_json vhosts_json=$(ubus call luci.haproxy list_vhosts 2>/dev/null) [ -z "$vhosts_json" ] && return - echo "$vhosts_json" | jsonfilter -e '@.vhosts[*]' 2>/dev/null | while read -r line; do - local domain backend ssl - domain=$(echo "$vhosts_json" | jsonfilter -e "@.vhosts[$line].domain" 2>/dev/null) - backend=$(echo "$vhosts_json" | jsonfilter -e "@.vhosts[$line].backend" 2>/dev/null) - ssl=$(echo "$vhosts_json" | jsonfilter -e "@.vhosts[$line].ssl" 2>/dev/null) + backends_json=$(ubus call luci.haproxy list_backends 2>/dev/null) + certs_json=$(ubus call luci.haproxy list_certificates 2>/dev/null) - # Skip if domain empty + # Get array length + local count + count=$(echo "$vhosts_json" | jsonfilter -e '@.vhosts[*].domain' 2>/dev/null | wc -l) + [ "$count" -eq 0 ] && return + + local i=0 + while [ $i -lt "$count" ]; do + local id domain backend ssl ssl_redirect acme enabled + id=$(echo "$vhosts_json" | jsonfilter -e "@.vhosts[$i].id" 2>/dev/null) + domain=$(echo "$vhosts_json" | jsonfilter -e "@.vhosts[$i].domain" 2>/dev/null) + backend=$(echo "$vhosts_json" | jsonfilter -e "@.vhosts[$i].backend" 2>/dev/null) + ssl=$(echo "$vhosts_json" | jsonfilter -e "@.vhosts[$i].ssl" 2>/dev/null) + ssl_redirect=$(echo "$vhosts_json" | jsonfilter -e "@.vhosts[$i].ssl_redirect" 2>/dev/null) + acme=$(echo "$vhosts_json" | jsonfilter -e "@.vhosts[$i].acme" 2>/dev/null) + enabled=$(echo "$vhosts_json" | jsonfilter -e "@.vhosts[$i].enabled" 2>/dev/null) + + i=$((i + 1)) + + # Skip if domain empty or disabled [ -z "$domain" ] && continue - # Check if already processed by local_port - # For HAProxy, we identify by domain + # Check if already processed grep -q "^haproxy_${domain}$" "$tmp_file" 2>/dev/null && continue + # Get backend port from servers + local backend_port="" + if [ -n "$backends_json" ] && [ -n "$backend" ]; then + backend_port=$(echo "$backends_json" | jsonfilter -e "@.backends[@.name='$backend'].servers[0].port" 2>/dev/null) + [ -z "$backend_port" ] && backend_port=$(echo "$backends_json" | jsonfilter -e "@.backends[@.id='$backend'].servers[0].port" 2>/dev/null) + fi + + # Check certificate status + local cert_status="none" + if [ "$ssl" = "true" ] || [ "$ssl" = "1" ]; then + if [ "$acme" = "true" ] || [ "$acme" = "1" ]; then + cert_status="acme" + else + cert_status="manual" + fi + fi + + # Determine status based on enabled and HAProxy running + local status="stopped" + if [ "$enabled" = "true" ] || [ "$enabled" = "1" ]; then + # Check if HAProxy container is running + if lxc-info -n haproxy -s 2>/dev/null | grep -q "RUNNING"; then + status="running" + fi + else + status="disabled" + fi + json_add_object - json_add_string "id" "haproxy_$(echo "$domain" | sed 's/[^a-zA-Z0-9]/_/g')" + json_add_string "id" "$id" json_add_string "name" "$domain" json_add_string "category" "proxy" json_add_string "icon" "arrow" - json_add_string "status" "running" + json_add_string "status" "$status" json_add_boolean "published" 1 json_add_string "source" "haproxy" + # URLs json_add_object "urls" + json_add_string "local" "http://${lan_ip}${backend_port:+:$backend_port}" if [ "$ssl" = "true" ] || [ "$ssl" = "1" ]; then json_add_string "clearnet" "https://${domain}" else @@ -243,11 +287,33 @@ _aggregate_haproxy_services() { fi json_close_object + # HAProxy details json_add_object "haproxy" - json_add_boolean "enabled" 1 + json_add_string "id" "$id" json_add_string "domain" "$domain" json_add_string "backend" "$backend" - json_add_boolean "ssl" "$ssl" + [ -n "$backend_port" ] && json_add_int "backend_port" "$backend_port" + if [ "$ssl" = "true" ] || [ "$ssl" = "1" ]; then + json_add_boolean "ssl" 1 + else + json_add_boolean "ssl" 0 + fi + if [ "$ssl_redirect" = "true" ] || [ "$ssl_redirect" = "1" ]; then + json_add_boolean "ssl_redirect" 1 + else + json_add_boolean "ssl_redirect" 0 + fi + if [ "$acme" = "true" ] || [ "$acme" = "1" ]; then + json_add_boolean "acme" 1 + else + json_add_boolean "acme" 0 + fi + if [ "$enabled" = "true" ] || [ "$enabled" = "1" ]; then + json_add_boolean "enabled" 1 + else + json_add_boolean "enabled" 0 + fi + json_add_string "cert_status" "$cert_status" json_close_object json_close_object diff --git a/package/secubox/luci-app-service-registry/root/usr/sbin/secubox-landing-gen b/package/secubox/luci-app-service-registry/root/usr/sbin/secubox-landing-gen index 1786875e..f12d5f2b 100644 --- a/package/secubox/luci-app-service-registry/root/usr/sbin/secubox-landing-gen +++ b/package/secubox/luci-app-service-registry/root/usr/sbin/secubox-landing-gen @@ -483,4 +483,7 @@ TMP_OUTPUT="${OUTPUT_PATH}.tmp" awk -v json="$SERVICES_JSON" '{gsub(/SERVICES_JSON_PLACEHOLDER/, json); print}' "$OUTPUT_PATH" > "$TMP_OUTPUT" mv "$TMP_OUTPUT" "$OUTPUT_PATH" +# Ensure web server can read the file +chmod 644 "$OUTPUT_PATH" + echo "Landing page generated: $OUTPUT_PATH" diff --git a/package/secubox/secubox-app-streamlit/README.md b/package/secubox/secubox-app-streamlit/README.md new file mode 100644 index 00000000..fee3bb80 --- /dev/null +++ b/package/secubox/secubox-app-streamlit/README.md @@ -0,0 +1,255 @@ +# SecuBox Streamlit Platform + +Multi-instance Streamlit hosting platform for OpenWrt with LXC containers and Gitea integration. + +## Features + +- **Multi-instance support**: Run multiple Streamlit apps on different ports +- **Folder-based apps**: Each app in its own directory with dependencies +- **Gitea integration**: Clone and sync apps directly from Gitea repositories +- **LXC isolation**: Apps run in isolated Alpine Linux container +- **Auto-dependency install**: `requirements.txt` processed automatically + +## Quick Start + +### 1. Install & Enable + +```bash +opkg install secubox-app-streamlit +/etc/init.d/streamlit enable +streamlitctl install +``` + +### 2. Create Your First App + +```bash +streamlitctl app create myapp +streamlitctl instance add myapp 8502 +/etc/init.d/streamlit restart +``` + +Access at: `http://:8502` + +## Deploy from Gitea + +The platform integrates with Gitea for source-controlled app deployment. + +### Setup Gitea Credentials + +```bash +# Configure Gitea connection +uci set streamlit.gitea.enabled=1 +uci set streamlit.gitea.url='http://192.168.255.1:3000' +uci set streamlit.gitea.user='admin' +uci set streamlit.gitea.token='your-access-token' +uci commit streamlit + +# Store git credentials in container +streamlitctl gitea setup +``` + +### Clone App from Gitea Repository + +**Method 1: Using streamlitctl (recommended)** + +```bash +# Clone using repo shorthand (user/repo) +streamlitctl gitea clone yijing CyberMood/yijing-oracle + +# Add instance on port +streamlitctl instance add yijing 8505 + +# Restart to apply +/etc/init.d/streamlit restart +``` + +**Method 2: Manual Clone + UCI Config** + +```bash +# Clone directly to apps directory +git clone http://192.168.255.1:3000/CyberMood/yijing-oracle.git /srv/streamlit/apps/yijing + +# Register in UCI +uci set streamlit.yijing=app +uci set streamlit.yijing.name='Yijing Oracle' +uci set streamlit.yijing.path='yijing/app.py' +uci set streamlit.yijing.enabled='1' +uci set streamlit.yijing.port='8505' +uci commit streamlit + +# Add instance and restart +streamlitctl instance add yijing 8505 +/etc/init.d/streamlit restart +``` + +### Update App from Gitea + +```bash +# Pull latest changes +streamlitctl gitea pull yijing + +# Restart to apply changes +/etc/init.d/streamlit restart +``` + +## App Folder Structure + +Each app lives in `/srv/streamlit/apps//`: + +``` +/srv/streamlit/apps/myapp/ +├── app.py # Main entry point (or main.py, .py) +├── requirements.txt # Python dependencies (auto-installed) +├── .streamlit/ # Optional Streamlit config +│ └── config.toml +└── ... # Other files (pages/, data/, etc.) +``` + +**Main file detection order**: `app.py` > `main.py` > `.py` > first `.py` file + +## CLI Reference + +### Container Management + +```bash +streamlitctl install # Setup LXC container +streamlitctl uninstall # Remove container (keeps apps) +streamlitctl update # Update Streamlit version +streamlitctl status # Show platform status +streamlitctl logs [app] # View logs +streamlitctl shell # Open container shell +``` + +### App Management + +```bash +streamlitctl app list # List all apps +streamlitctl app create # Create new app folder +streamlitctl app delete # Delete app +streamlitctl app deploy # Deploy from path/archive +``` + +### Instance Management + +```bash +streamlitctl instance list # List instances +streamlitctl instance add # Add instance +streamlitctl instance remove # Remove instance +streamlitctl instance start # Start single instance +streamlitctl instance stop # Stop single instance +``` + +### Gitea Integration + +```bash +streamlitctl gitea setup # Configure git credentials +streamlitctl gitea clone # Clone from Gitea +streamlitctl gitea pull # Pull latest changes +``` + +## UCI Configuration + +Main config: `/etc/config/streamlit` + +``` +config streamlit 'main' + option enabled '1' + option http_port '8501' + option data_path '/srv/streamlit' + option memory_limit '512M' + +config streamlit 'gitea' + option enabled '1' + option url 'http://192.168.255.1:3000' + option user 'admin' + option token 'your-token' + +config app 'myapp' + option name 'My App' + option enabled '1' + option repo 'user/myapp' + +config instance 'myapp' + option app 'myapp' + option port '8502' + option enabled '1' +``` + +## Example: Complete Gitea Workflow + +```bash +# 1. Create repo in Gitea with your Streamlit app +# - app.py (main file) +# - requirements.txt (dependencies) + +# 2. Configure streamlit platform +uci set streamlit.gitea.enabled=1 +uci set streamlit.gitea.url='http://192.168.255.1:3000' +uci set streamlit.gitea.user='admin' +uci set streamlit.gitea.token='abc123' +uci commit streamlit + +# 3. Clone and deploy +streamlitctl gitea setup +streamlitctl gitea clone myapp admin/my-streamlit-app +streamlitctl instance add myapp 8502 +/etc/init.d/streamlit restart + +# 4. Access app +curl http://192.168.255.1:8502 + +# 5. Update from Gitea when code changes +streamlitctl gitea pull myapp +/etc/init.d/streamlit restart +``` + +## HAProxy Integration + +To expose Streamlit apps via HAProxy vhost: + +```bash +# Add backend for app +uci add haproxy backend +uci set haproxy.@backend[-1].name='streamlit_myapp' +uci set haproxy.@backend[-1].mode='http' +uci add_list haproxy.@backend[-1].server='myapp 127.0.0.1:8502' +uci commit haproxy + +# Add vhost +uci add haproxy vhost +uci set haproxy.@vhost[-1].name='myapp_vhost' +uci set haproxy.@vhost[-1].domain='myapp.example.com' +uci set haproxy.@vhost[-1].backend='streamlit_myapp' +uci set haproxy.@vhost[-1].ssl='1' +uci commit haproxy + +/etc/init.d/haproxy restart +``` + +## Troubleshooting + +**Container won't start:** +```bash +streamlitctl status +lxc-info -n streamlit +``` + +**App not loading:** +```bash +streamlitctl logs myapp +streamlitctl shell +# Inside container: +cd /srv/apps/myapp && streamlit run app.py +``` + +**Git clone fails:** +```bash +# Check credentials +streamlitctl gitea setup +# Test manually +git clone http://admin:token@192.168.255.1:3000/user/repo.git /tmp/test +``` + +## License + +Copyright (C) 2025 CyberMind.fr diff --git a/package/secubox/secubox-app-streamlit/files/usr/sbin/streamlitctl b/package/secubox/secubox-app-streamlit/files/usr/sbin/streamlitctl index 80d48b66..5e19d979 100644 --- a/package/secubox/secubox-app-streamlit/files/usr/sbin/streamlitctl +++ b/package/secubox/secubox-app-streamlit/files/usr/sbin/streamlitctl @@ -113,9 +113,26 @@ App Folder Structure: ... Other files Examples: + # Create local app streamlitctl app create myapp streamlitctl instance add myapp 8502 - streamlitctl gitea clone myapp myuser/myapp-repo + /etc/init.d/streamlit restart + + # Deploy from Gitea (complete workflow) + uci set streamlit.gitea.enabled=1 + uci set streamlit.gitea.url='http://192.168.255.1:3000' + uci set streamlit.gitea.user='admin' + uci set streamlit.gitea.token='your-token' + uci commit streamlit + streamlitctl gitea setup + streamlitctl gitea clone yijing CyberMood/yijing-oracle + streamlitctl instance add yijing 8505 + /etc/init.d/streamlit restart + # Access at: http://:8505 + + # Update app from Gitea + streamlitctl gitea pull yijing + /etc/init.d/streamlit restart EOF }