#!/bin/sh # slforge - Streamlit Forge CLI # Manage Streamlit apps: create, deploy, publish . /lib/functions.sh # Load config config_load streamlit-forge config_get GITEA_URL main gitea_url 'http://127.0.0.1:3001' config_get GITEA_ORG main gitea_org 'streamlit-apps' config_get APPS_DIR main apps_dir '/srv/streamlit/apps' config_get PREVIEWS_DIR main previews_dir '/srv/streamlit/previews' config_get TEMPLATES_DIR main templates_dir '/usr/share/streamlit-forge/templates' config_get BASE_DOMAIN main base_domain 'apps.secubox.in' config_get DEFAULT_PORT main default_port_start '8501' config_get DEFAULT_MEMORY main default_memory '512M' # Load Gitea token from gitea config if available config_load gitea config_get GITEA_TOKEN main api_token '' config_get GITEA_URL main url "$GITEA_URL" # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_ok() { echo -e "${GREEN}[OK]${NC} $1"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } log_err() { echo -e "${RED}[ERROR]${NC} $1"; } # NFO Parser library NFO_PARSER="/usr/share/streamlit-forge/lib/nfo-parser.sh" [ -f "$NFO_PARSER" ] && . "$NFO_PARSER" # Launcher integration LAUNCHER_TRACKING_DIR="/tmp/streamlit-access" # Track app access (for launcher idle detection) track_access() { local name="$1" mkdir -p "$LAUNCHER_TRACKING_DIR" touch "$LAUNCHER_TRACKING_DIR/$name" } # Check if launcher is managing this app is_launcher_managed() { local enabled config_load streamlit-launcher 2>/dev/null || return 1 config_get enabled global enabled '0' [ "$enabled" = "1" ] } usage() { cat < [options] App Management: create [opts] Create new Streamlit app --from-upload Create from uploaded ZIP file --from-git Create from Git repository --from-template Create from template (basic, dashboard, data-viewer) list List all apps info Show app details delete Remove app and all data Instance Control: start [--port N] Start app instance stop Stop app instance restart Restart app instance status [app] Show instance status logs [-f] View app logs shell Open shell in app container Configuration: config list List app config config get Get config value config set Set config value sync-config Sync UCI <-> app config Source Management: edit Open in Gitea web editor pull Pull latest from Gitea push Push changes to Gitea Preview & Publishing: preview Generate preview screenshot expose [--domain] Create vhost + SSL hide Remove public access Mesh AppStore: publish Publish to mesh catalog unpublish Remove from mesh catalog catalog Browse mesh catalog install Install from mesh Launcher Integration: launcher status Show launcher status launcher priority Set app priority (higher=keep longer) launcher always-on Mark app as always-on (never auto-stop) Module Manifest (NFO): nfo init Generate README.nfo for app nfo info Show NFO summary nfo edit Edit README.nfo nfo validate Validate NFO file nfo json Export NFO as JSON nfo install Install app from directory with NFO Templates: templates List available templates Examples: slforge create myapp --from-template dashboard slforge start myapp --port 8510 slforge expose myapp --domain myapp.gk2.secubox.in slforge publish myapp EOF } # Get next available port get_next_port() { local port=$DEFAULT_PORT while netstat -tln 2>/dev/null | grep -q ":$port " || \ grep -q "option port '$port'" /etc/config/streamlit-forge 2>/dev/null; do port=$((port + 1)) done echo "$port" } # Check if app exists app_exists() { local name="$1" [ -d "$APPS_DIR/$name" ] || uci -q get streamlit-forge."$name" >/dev/null 2>&1 } # Get app UCI section get_app_config() { local name="$1" local key="$2" uci -q get streamlit-forge."$name"."$key" } # Set app UCI config set_app_config() { local name="$1" local key="$2" local value="$3" uci set streamlit-forge."$name"."$key"="$value" uci commit streamlit-forge } # ============================================ # Gitea Integration Functions (Phase 2) # ============================================ # Call Gitea API gitea_api() { local method="$1" local endpoint="$2" local data="$3" if [ -z "$GITEA_TOKEN" ]; then log_err "Gitea API token not configured" return 1 fi local url="${GITEA_URL}/api/v1${endpoint}" if [ "$method" = "GET" ]; then curl -s -H "Authorization: token $GITEA_TOKEN" "$url" elif [ "$method" = "POST" ]; then curl -s -X POST -H "Authorization: token $GITEA_TOKEN" \ -H "Content-Type: application/json" \ -d "$data" "$url" elif [ "$method" = "DELETE" ]; then curl -s -X DELETE -H "Authorization: token $GITEA_TOKEN" "$url" fi } # Check if Gitea org exists, create if not gitea_ensure_org() { local org="$GITEA_ORG" # Check if org exists local result=$(gitea_api GET "/orgs/$org" 2>/dev/null) if echo "$result" | grep -q '"id"'; then return 0 fi # Create org log_info "Creating Gitea organization: $org" gitea_api POST "/orgs" "{\"username\":\"$org\",\"description\":\"Streamlit Forge Apps\"}" >/dev/null } # Create Gitea repo for app gitea_create_repo() { local name="$1" local app_dir="$APPS_DIR/$name" gitea_ensure_org # Check if repo exists local result=$(gitea_api GET "/repos/$GITEA_ORG/$name" 2>/dev/null) if echo "$result" | grep -q '"id"'; then log_info "Repository already exists: $GITEA_ORG/$name" return 0 fi # Create repo log_info "Creating repository: $GITEA_ORG/$name" gitea_api POST "/orgs/$GITEA_ORG/repos" \ "{\"name\":\"$name\",\"description\":\"Streamlit app: $name\",\"private\":false,\"auto_init\":false}" >/dev/null # Initialize git in app directory cd "$app_dir/src" if [ ! -d ".git" ]; then git init -q git add -A git commit -q -m "Initial commit from Streamlit Forge" fi # Add remote local remote_url="${GITEA_URL}/${GITEA_ORG}/${name}.git" git remote remove origin 2>/dev/null git remote add origin "$remote_url" # Push git push -u origin main 2>/dev/null || git push -u origin master 2>/dev/null # Save repo info to UCI set_app_config "$name" gitea_repo "$name" set_app_config "$name" gitea_branch "$(git rev-parse --abbrev-ref HEAD)" log_ok "Created and pushed to: $remote_url" } # Open Gitea editor cmd_edit() { local name="$1" [ -z "$name" ] && { log_err "App name required"; return 1; } if ! app_exists "$name"; then log_err "App not found: $name" return 1 fi local gitea_repo=$(get_app_config "$name" gitea_repo) # Create repo if doesn't exist if [ -z "$gitea_repo" ]; then log_info "Setting up Gitea repository..." gitea_create_repo "$name" gitea_repo="$name" fi local branch=$(get_app_config "$name" gitea_branch) [ -z "$branch" ] && branch="main" local edit_url="${GITEA_URL}/${GITEA_ORG}/${gitea_repo}/_edit/${branch}/app.py" echo "" log_ok "Edit your app at:" echo " $edit_url" echo "" echo "After editing, run: slforge pull $name" echo "" } # Pull from Gitea cmd_pull() { local name="$1" [ -z "$name" ] && { log_err "App name required"; return 1; } if ! app_exists "$name"; then log_err "App not found: $name" return 1 fi local app_dir="$APPS_DIR/$name" local gitea_repo=$(get_app_config "$name" gitea_repo) if [ -z "$gitea_repo" ]; then log_err "No Gitea repository configured for $name" log_info "Run: slforge edit $name to set up the repository" return 1 fi cd "$app_dir/src" if [ ! -d ".git" ]; then log_err "Not a git repository. Setting up..." git init -q git remote add origin "${GITEA_URL}/${GITEA_ORG}/${gitea_repo}.git" fi log_info "Pulling latest changes..." git fetch origin 2>/dev/null git pull origin "$(git rev-parse --abbrev-ref HEAD)" 2>/dev/null || { # Try with reset for forced sync git reset --hard origin/"$(git rev-parse --abbrev-ref HEAD)" 2>/dev/null } # Update timestamp set_app_config "$name" last_pull "$(date -Iseconds)" # Check if app is running and restart local port=$(get_app_config "$name" port) if [ -n "$port" ] && netstat -tln 2>/dev/null | grep -q ":$port "; then log_info "App is running, restarting to apply changes..." cmd_restart "$name" fi log_ok "Pulled latest for $name" } # Push to Gitea cmd_push() { local name="$1" shift [ -z "$name" ] && { log_err "App name required"; return 1; } if ! app_exists "$name"; then log_err "App not found: $name" return 1 fi local message="Update from Streamlit Forge" while [ $# -gt 0 ]; do case "$1" in -m|--message) message="$2"; shift 2 ;; *) shift ;; esac done local app_dir="$APPS_DIR/$name" local gitea_repo=$(get_app_config "$name" gitea_repo) if [ -z "$gitea_repo" ]; then log_info "Setting up Gitea repository..." gitea_create_repo "$name" return 0 fi cd "$app_dir/src" if [ ! -d ".git" ]; then log_err "Not a git repository" return 1 fi # Check for changes if git diff --quiet && git diff --cached --quiet; then log_info "No changes to push" return 0 fi log_info "Pushing changes..." git add -A git commit -m "$message" 2>/dev/null git push origin "$(git rev-parse --abbrev-ref HEAD)" 2>/dev/null || { log_err "Failed to push. Check remote access." return 1 } # Update timestamp set_app_config "$name" last_push "$(date -Iseconds)" log_ok "Pushed changes for $name" } # Generate preview screenshot cmd_preview() { local name="$1" [ -z "$name" ] && { log_err "App name required"; return 1; } if ! app_exists "$name"; then log_err "App not found: $name" return 1 fi local port=$(get_app_config "$name" port) # Check if running if ! netstat -tln 2>/dev/null | grep -q ":$port "; then log_warn "App not running. Starting temporarily..." cmd_start "$name" --quiet sleep 5 fi mkdir -p "$PREVIEWS_DIR" local preview_file="$PREVIEWS_DIR/${name}.png" local preview_html="$PREVIEWS_DIR/${name}.html" # Method 1: Try wkhtmltoimage if available (best quality) if command -v wkhtmltoimage >/dev/null 2>&1; then log_info "Generating preview with wkhtmltoimage..." wkhtmltoimage --width 1280 --height 800 --quality 85 \ "http://127.0.0.1:$port" "$preview_file" 2>/dev/null if [ -f "$preview_file" ]; then log_ok "Preview saved: $preview_file" set_app_config "$name" preview "$preview_file" return 0 fi fi # Method 2: Capture HTML content for later rendering log_info "Capturing page content..." curl -s -m 10 "http://127.0.0.1:$port" > "$preview_html" 2>/dev/null if [ -s "$preview_html" ]; then log_ok "HTML preview saved: $preview_html" set_app_config "$name" preview "$preview_html" # Generate a simple placeholder PNG using SVG cat > "${preview_file%.*}.svg" <<-SVGEOF 📊 $name Streamlit App Port: $port SVGEOF log_info "SVG placeholder: ${preview_file%.*}.svg" return 0 fi log_err "Failed to generate preview" return 1 } # ============================================ # End Gitea Integration Functions # ============================================ # Create app from template cmd_create() { local name="$1" shift [ -z "$name" ] && { log_err "App name required"; return 1; } # Check if exists app_exists "$name" && { log_err "App '$name' already exists"; return 1; } local from_upload="" from_git="" from_template="basic" while [ $# -gt 0 ]; do case "$1" in --from-upload) from_upload="$2"; shift 2 ;; --from-git) from_git="$2"; shift 2 ;; --from-template) from_template="$2"; shift 2 ;; *) shift ;; esac done local app_dir="$APPS_DIR/$name" mkdir -p "$app_dir/src" "$app_dir/data" if [ -n "$from_upload" ]; then # Extract ZIP log_info "Extracting from $from_upload..." if [ -f "$from_upload" ]; then unzip -q "$from_upload" -d "$app_dir/src" 2>/dev/null || { log_err "Failed to extract ZIP" rm -rf "$app_dir" return 1 } # Move files if extracted to subdirectory local subdir=$(ls -1 "$app_dir/src" 2>/dev/null | head -1) if [ -d "$app_dir/src/$subdir" ] && [ "$subdir" != "." ]; then mv "$app_dir/src/$subdir"/* "$app_dir/src/" 2>/dev/null rmdir "$app_dir/src/$subdir" 2>/dev/null fi else log_err "File not found: $from_upload" rm -rf "$app_dir" return 1 fi elif [ -n "$from_git" ]; then # Clone from Git log_info "Cloning from $from_git..." git clone "$from_git" "$app_dir/src" 2>/dev/null || { log_err "Failed to clone repository" rm -rf "$app_dir" return 1 } else # Use template local tpl_dir="$TEMPLATES_DIR/$from_template" if [ -d "$tpl_dir" ]; then log_info "Creating from template: $from_template" cp -r "$tpl_dir"/* "$app_dir/src/" # Replace placeholders find "$app_dir/src" -type f -name "*.py" -exec sed -i "s/APPNAME/$name/g" {} \; find "$app_dir/src" -type f -name "*.md" -exec sed -i "s/APPNAME/$name/g" {} \; else # Create minimal app log_info "Creating minimal Streamlit app" cat > "$app_dir/src/app.py" <<-PYEOF import streamlit as st st.set_page_config(page_title="$name", page_icon="🚀") st.title("$name") st.write("Welcome to your new Streamlit app!") st.sidebar.header("Navigation") page = st.sidebar.radio("Go to", ["Home", "About"]) if page == "Home": st.header("Home") st.write("Edit src/app.py to customize this app.") else: st.header("About") st.write("Created with Streamlit Forge") PYEOF cat > "$app_dir/src/requirements.txt" <<-REQEOF streamlit>=1.30.0 REQEOF fi fi # Create UCI config local port=$(get_next_port) uci set streamlit-forge."$name"=app uci set streamlit-forge."$name".name="$name" uci set streamlit-forge."$name".enabled='0' uci set streamlit-forge."$name".port="$port" uci set streamlit-forge."$name".entrypoint='app.py' uci set streamlit-forge."$name".memory="$DEFAULT_MEMORY" uci set streamlit-forge."$name".created="$(date +%Y-%m-%d)" uci commit streamlit-forge # Create Gitea repo if token available if [ -n "$GITEA_TOKEN" ]; then gitea_create_repo "$name" 2>/dev/null || log_warn "Gitea repo creation skipped" fi # Generate README.nfo manifest cmd_nfo init "$name" 2>/dev/null || log_warn "NFO generation skipped" log_ok "Created app: $name" log_info " Directory: $app_dir" log_info " Port: $port" log_info " Manifest: $app_dir/README.nfo" log_info " Start with: slforge start $name" } # List all apps cmd_list() { echo "" echo "Streamlit Apps" echo "==============" echo "" local found=0 for app_dir in "$APPS_DIR"/*/; do [ -d "$app_dir" ] || continue local name=$(basename "$app_dir") local enabled=$(get_app_config "$name" enabled) local port=$(get_app_config "$name" port) local domain=$(get_app_config "$name" domain) local status="stopped" # Check if running by port if [ -n "$port" ] && netstat -tln 2>/dev/null | grep -q ":$port "; then status="running" fi printf " %-20s %-8s port:%-5s %s\n" "$name" "[$status]" "${port:-?}" "${domain:-}" found=1 done [ "$found" = "0" ] && echo " No apps found. Create one with: slforge create " echo "" } # Show app info cmd_info() { local name="$1" [ -z "$name" ] && { log_err "App name required"; return 1; } if ! app_exists "$name"; then log_err "App not found: $name" return 1 fi local app_dir="$APPS_DIR/$name" echo "" echo "App: $name" echo "=========================================" echo "" echo "Directory: $app_dir" echo "Enabled: $(get_app_config "$name" enabled)" echo "Port: $(get_app_config "$name" port)" echo "Memory: $(get_app_config "$name" memory)" echo "Domain: $(get_app_config "$name" domain)" echo "Entrypoint: $(get_app_config "$name" entrypoint)" echo "Created: $(get_app_config "$name" created)" echo "" # Check status by port local port=$(get_app_config "$name" port) if [ -n "$port" ] && netstat -tln 2>/dev/null | grep -q ":$port "; then echo "Status: RUNNING on port $port" else echo "Status: STOPPED" fi # Show NFO info if available local nfo_file="$app_dir/README.nfo" if [ -f "$nfo_file" ] && [ -f "$NFO_PARSER" ]; then nfo_parse "$nfo_file" echo "" echo "Module Info (from NFO):" echo " Category: $(nfo_get tags category)" echo " Keywords: $(nfo_get tags keywords)" echo " Author: $(nfo_get identity author)" local prompt=$(nfo_get dynamics prompt_context | head -1) [ -n "$prompt" ] && echo " AI Context: ${prompt:0:60}..." elif [ -f "$nfo_file" ]; then echo "" echo "Manifest: $nfo_file (run: slforge nfo info $name)" fi # Show files if [ -d "$app_dir/src" ]; then echo "" echo "Source Files:" ls -la "$app_dir/src" 2>/dev/null | head -10 fi echo "" } # Start app instance cmd_start() { local name="$1" shift [ -z "$name" ] && { log_err "App name required"; return 1; } if ! app_exists "$name"; then log_err "App not found: $name" return 1 fi local port=$(get_app_config "$name" port) local quiet=0 while [ $# -gt 0 ]; do case "$1" in --port) port="$2"; shift 2 ;; --quiet) quiet=1; shift ;; *) shift ;; esac done local app_dir="$APPS_DIR/$name" local entrypoint=$(get_app_config "$name" entrypoint) [ -z "$entrypoint" ] && entrypoint="app.py" # Check if already running if [ -f "/var/run/streamlit-$name.pid" ]; then local pid=$(cat "/var/run/streamlit-$name.pid") if kill -0 "$pid" 2>/dev/null; then [ "$quiet" = "0" ] && log_warn "App $name already running (PID: $pid)" return 0 fi fi [ "$quiet" = "0" ] && log_info "Starting $name on port $port..." # Check for Streamlit in LXC container if lxc-info -n streamlit 2>/dev/null | grep -q "RUNNING"; then # Run in Streamlit LXC container # Note: /srv/streamlit/apps is mounted at /srv/apps inside container # Create wrapper script to avoid shell escaping issues cat > "$APPS_DIR/.slforge-start.sh" <<-'STARTSCRIPT' #!/bin/sh cd "$1" [ -f requirements.txt ] && pip3 install -q -r requirements.txt 2>/dev/null nohup streamlit run "$2" --server.port "$3" --server.address 0.0.0.0 --server.headless true > /tmp/streamlit-$4.log 2>&1 & echo $! STARTSCRIPT chmod +x "$APPS_DIR/.slforge-start.sh" lxc-attach -n streamlit -- /srv/apps/.slforge-start.sh \ "/srv/apps/$name/src" "$entrypoint" "$port" "$name" \ > "/var/run/streamlit-$name.pid" else # Run directly on host cd "$app_dir/src" if [ -f requirements.txt ]; then pip3 install -q -r requirements.txt 2>/dev/null fi nohup streamlit run "$entrypoint" \ --server.port "$port" \ --server.address 0.0.0.0 \ --server.headless true \ > "/var/log/streamlit-$name.log" 2>&1 & echo $! > "/var/run/streamlit-$name.pid" fi # Update UCI set_app_config "$name" enabled '1' set_app_config "$name" port "$port" # Wait for port to become available sleep 3 if netstat -tln 2>/dev/null | grep -q ":$port "; then [ "$quiet" = "0" ] && log_ok "Started $name" [ "$quiet" = "0" ] && log_info "URL: http://192.168.255.1:$port" # Track access for launcher idle detection track_access "$name" return 0 fi [ "$quiet" = "0" ] && log_err "Failed to start $name" return 1 } # Stop app instance cmd_stop() { local name="$1" [ -z "$name" ] && { log_err "App name required"; return 1; } if [ -f "/var/run/streamlit-$name.pid" ]; then local pid=$(cat "/var/run/streamlit-$name.pid") if kill -0 "$pid" 2>/dev/null; then log_info "Stopping $name (PID: $pid)..." kill "$pid" 2>/dev/null sleep 1 kill -9 "$pid" 2>/dev/null fi rm -f "/var/run/streamlit-$name.pid" fi set_app_config "$name" enabled '0' log_ok "Stopped $name" } # Restart app cmd_restart() { local name="$1" cmd_stop "$name" sleep 1 cmd_start "$name" } # Show status cmd_status() { local name="$1" if [ -n "$name" ]; then cmd_info "$name" else cmd_list fi } # View logs cmd_logs() { local name="$1" local follow=0 [ -z "$name" ] && { log_err "App name required"; return 1; } shift [ "$1" = "-f" ] && follow=1 local logfile="/var/log/streamlit-$name.log" if [ ! -f "$logfile" ]; then log_err "No logs found for $name" return 1 fi if [ "$follow" = "1" ]; then tail -f "$logfile" else tail -100 "$logfile" fi } # Delete app cmd_delete() { local name="$1" [ -z "$name" ] && { log_err "App name required"; return 1; } if ! app_exists "$name"; then log_err "App not found: $name" return 1 fi # Stop if running cmd_stop "$name" 2>/dev/null # Remove files rm -rf "$APPS_DIR/$name" rm -rf "$PREVIEWS_DIR/$name" rm -f "/var/log/streamlit-$name.log" # Remove UCI config uci delete streamlit-forge."$name" 2>/dev/null uci commit streamlit-forge log_ok "Deleted app: $name" } # Expose app with vhost + SSL cmd_expose() { local name="$1" shift [ -z "$name" ] && { log_err "App name required"; return 1; } if ! app_exists "$name"; then log_err "App not found: $name" return 1 fi local domain="" while [ $# -gt 0 ]; do case "$1" in --domain) domain="$2"; shift 2 ;; *) shift ;; esac done # Default domain [ -z "$domain" ] && domain="$name.$BASE_DOMAIN" local port=$(get_app_config "$name" port) log_info "Exposing $name at $domain..." # Create HAProxy vhost if command -v haproxyctl >/dev/null 2>&1; then haproxyctl vhost add "$domain" 2>/dev/null # Add mitmproxy route local routes_file="/srv/mitmproxy/haproxy-routes.json" if [ -f "$routes_file" ]; then # Add route using jsonfilter + sed local tmp=$(mktemp) cat "$routes_file" | sed "s/}$/,\"$domain\":[\"192.168.255.1\",$port]}/" > "$tmp" mv "$tmp" "$routes_file" fi log_info "Requesting SSL certificate..." haproxyctl ssl request "$domain" 2>/dev/null else log_warn "haproxyctl not available - manual vhost setup required" fi # Update UCI set_app_config "$name" domain "$domain" log_ok "Exposed at: https://$domain" } # Hide app (remove public access) cmd_hide() { local name="$1" [ -z "$name" ] && { log_err "App name required"; return 1; } local domain=$(get_app_config "$name" domain) if [ -n "$domain" ] && command -v haproxyctl >/dev/null 2>&1; then log_info "Removing vhost: $domain" haproxyctl vhost remove "$domain" 2>/dev/null fi set_app_config "$name" domain "" log_ok "Hidden app: $name" } # List templates cmd_templates() { echo "" echo "Available Templates" echo "===================" echo "" for tpl in "$TEMPLATES_DIR"/*/; do [ -d "$tpl" ] || continue local name=$(basename "$tpl") local desc="" [ -f "$tpl/README.md" ] && desc=$(head -1 "$tpl/README.md" | sed 's/^#\s*//') printf " %-15s %s\n" "$name" "$desc" done echo "" echo "Usage: slforge create myapp --from-template