From 01b48e42ec8d1e0318b7ca4d38a1ec4b50e80183 Mon Sep 17 00:00:00 2001 From: CyberMind-FR Date: Sun, 15 Mar 2026 08:36:56 +0100 Subject: [PATCH] feat(osint): Add Maltego iTDS and Sherlock security tools - secubox-app-maltego: Transform Distribution Server in LXC - Python-based transform execution engine - REST API compatible with Maltego desktop client - Custom transform support via /srv/maltego/transforms/ - secubox-app-sherlock: Username hunting across social networks - Sherlock + Holehe integration for username/email OSINT - maigret, theHarvester, socialscan also installed - REST API with async task execution Both tools exposed via HAProxy at: - https://maltego.gk2.secubox.in/ - https://sherlock.gk2.secubox.in/ Streamlit OSINT dashboard deployed at: - https://osint.gk2.secubox.in/ Co-Authored-By: Claude Opus 4.5 --- .claude/settings.local.json | 3 +- package/secubox/secubox-app-maltego/Makefile | 54 ++ .../files/etc/config/maltego | 15 + .../files/etc/init.d/maltego | 36 ++ .../files/usr/sbin/maltegoctl | 514 ++++++++++++++++++ package/secubox/secubox-app-sherlock/Makefile | 53 ++ .../files/etc/config/sherlock | 14 + .../files/etc/init.d/sherlock | 36 ++ .../files/usr/sbin/sherlockctl | 415 ++++++++++++++ 9 files changed, 1139 insertions(+), 1 deletion(-) create mode 100644 package/secubox/secubox-app-maltego/Makefile create mode 100644 package/secubox/secubox-app-maltego/files/etc/config/maltego create mode 100644 package/secubox/secubox-app-maltego/files/etc/init.d/maltego create mode 100644 package/secubox/secubox-app-maltego/files/usr/sbin/maltegoctl create mode 100644 package/secubox/secubox-app-sherlock/Makefile create mode 100644 package/secubox/secubox-app-sherlock/files/etc/config/sherlock create mode 100644 package/secubox/secubox-app-sherlock/files/etc/init.d/sherlock create mode 100644 package/secubox/secubox-app-sherlock/files/usr/sbin/sherlockctl diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 59878abf..ab1cba30 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -550,7 +550,8 @@ "WebFetch(domain:ytdl.gk2.secubox.in)", "Bash([\"''\"'\"\"'\"''\"'\"\"'\"'']$/, \"\", val\\)\n\n if \\(section == \"tags\" && key == \"category\"\\) category = val\n if \\(section == \"tags\" && key == \"keywords\"\\) keywords = val\n if \\(section == \"tags\" && key == \"audience\"\\) audience = val\n if \\(section == \"description\" && key == \"short\"\\) desc = val\n if \\(section == \"dynamics\" && key == \"capabilities\"\\) caps = val\n if \\(section == \"media\" && key == \"icon\"\\) icon = val\n if \\(section == \"identity\" && key == \"version\"\\) version = val\n }\n END {\n printf \"%s)", "Bash(%s)", - "Bash(/tmp/test_nfo.sh)" + "Bash(/tmp/test_nfo.sh)", + "Bash(SSH_AUTH_SOCK=\"\" ssh:*)" ] } } diff --git a/package/secubox/secubox-app-maltego/Makefile b/package/secubox/secubox-app-maltego/Makefile new file mode 100644 index 00000000..71bf8da0 --- /dev/null +++ b/package/secubox/secubox-app-maltego/Makefile @@ -0,0 +1,54 @@ +include $(TOPDIR)/rules.mk + +PKG_NAME:=secubox-app-maltego +PKG_VERSION:=1.0.0 +PKG_RELEASE:=1 + +PKG_MAINTAINER:=SecuBox +PKG_LICENSE:=MIT + +include $(INCLUDE_DIR)/package.mk + +define Package/secubox-app-maltego + SECTION:=secubox + CATEGORY:=SecuBox + SUBMENU:=Applications + TITLE:=Maltego Transform Server + DEPENDS:=+lxc +lxc-attach +lxc-start +lxc-stop +lxc-info +curl +python3 + PKGARCH:=all +endef + +define Package/secubox-app-maltego/description + Maltego iTDS-compatible transform server running in LXC container. + Provides a Python-based transform execution engine for OSINT + and link analysis workflows. +endef + +define Package/secubox-app-maltego/conffiles +/etc/config/maltego +endef + +define Build/Compile +endef + +define Package/secubox-app-maltego/install + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_CONF) ./files/etc/config/maltego $(1)/etc/config/ + + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/etc/init.d/maltego $(1)/etc/init.d/ + + $(INSTALL_DIR) $(1)/usr/sbin + $(INSTALL_BIN) ./files/usr/sbin/maltegoctl $(1)/usr/sbin/ +endef + +define Package/secubox-app-maltego/postinst +#!/bin/sh +[ -n "$${IPKG_INSTROOT}" ] || { + mkdir -p /srv/maltego/{config,transforms,logs} + /etc/init.d/maltego enable +} +exit 0 +endef + +$(eval $(call BuildPackage,secubox-app-maltego)) diff --git a/package/secubox/secubox-app-maltego/files/etc/config/maltego b/package/secubox/secubox-app-maltego/files/etc/config/maltego new file mode 100644 index 00000000..80da9f75 --- /dev/null +++ b/package/secubox/secubox-app-maltego/files/etc/config/maltego @@ -0,0 +1,15 @@ +config maltego 'main' + option enabled '1' + option container 'maltego' + option image 'debian' + option version 'bookworm' + option memory '1024' + option ip '192.168.255.55' + option port '9001' + option https_port '9443' + option data_dir '/srv/maltego' + +config haproxy 'exposure' + option domain 'maltego.gk2.secubox.in' + option ssl '1' + option backend_port '9001' diff --git a/package/secubox/secubox-app-maltego/files/etc/init.d/maltego b/package/secubox/secubox-app-maltego/files/etc/init.d/maltego new file mode 100644 index 00000000..2c11b1b3 --- /dev/null +++ b/package/secubox/secubox-app-maltego/files/etc/init.d/maltego @@ -0,0 +1,36 @@ +#!/bin/sh /etc/rc.common + +START=95 +STOP=10 +USE_PROCD=1 + +CONTAINER="maltego" + +start_service() { + local enabled + config_load maltego + config_get enabled main enabled 0 + + [ "$enabled" = "1" ] || return 0 + + procd_open_instance + procd_set_param command /usr/bin/lxc-start -n "$CONTAINER" -F + procd_set_param respawn 3600 5 5 + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_close_instance +} + +stop_service() { + /usr/bin/lxc-stop -n "$CONTAINER" 2>/dev/null +} + +reload_service() { + stop_service + sleep 2 + start_service +} + +service_triggers() { + procd_add_reload_trigger "maltego" +} diff --git a/package/secubox/secubox-app-maltego/files/usr/sbin/maltegoctl b/package/secubox/secubox-app-maltego/files/usr/sbin/maltegoctl new file mode 100644 index 00000000..a6d82a2a --- /dev/null +++ b/package/secubox/secubox-app-maltego/files/usr/sbin/maltegoctl @@ -0,0 +1,514 @@ +#!/bin/sh +# Maltego iTDS Controller for SecuBox + +. /lib/functions.sh +config_load maltego + +CONTAINER="maltego" +LXC_PATH="/srv/lxc" +DATA_DIR="/srv/maltego" +IMAGE_URL="https://images.linuxcontainers.org/images/debian/bookworm/arm64/default" + +log() { + logger -t maltegoctl "$1" + echo "[INFO] $1" +} + +error() { + logger -t maltegoctl -p err "$1" + echo "[ERROR] $1" >&2 + exit 1 +} + +get_config() { + config_get "$1" main "$2" "$3" +} + +cmd_install() { + log "Installing Maltego iTDS container..." + + # Check if container exists + if [ -d "${LXC_PATH}/${CONTAINER}" ]; then + log "Container already exists. Use 'reinstall' to recreate." + return 0 + fi + + # Get container IP + local ip + get_config ip ip "192.168.255.55" + + # Create data directory + mkdir -p "${DATA_DIR}"/{config,transforms,logs} + + # Download and create container + log "Downloading Debian Bookworm image..." + mkdir -p "${LXC_PATH}/${CONTAINER}" + + # Download rootfs + local rootfs="${LXC_PATH}/${CONTAINER}/rootfs" + mkdir -p "$rootfs" + + # Get latest image date (BusyBox compatible) + local image_date + image_date=$(wget -qO- "${IMAGE_URL}/" | grep -o '[0-9]\{8\}_[0-9]\{2\}:[0-9]\{2\}' | tail -1) + + if [ -z "$image_date" ]; then + # Fallback: try to get from directory listing + image_date=$(wget -qO- "${IMAGE_URL}/" | sed -n 's/.*href="\([0-9_:]*\)\/">.*/\1/p' | tail -1) + fi + + if [ -z "$image_date" ]; then + error "Could not determine latest image date" + fi + + log "Downloading image: ${image_date}" + wget -q "${IMAGE_URL}/${image_date}/rootfs.tar.xz" -O /tmp/maltego-rootfs.tar.xz || \ + error "Failed to download image" + + tar -xJf /tmp/maltego-rootfs.tar.xz -C "$rootfs" + rm -f /tmp/maltego-rootfs.tar.xz + + # Create LXC config + cat > "${LXC_PATH}/${CONTAINER}/config" << EOF +lxc.uts.name = ${CONTAINER} +lxc.rootfs.path = dir:${rootfs} +lxc.include = /usr/share/lxc/config/common.conf + +# Network +lxc.net.0.type = veth +lxc.net.0.link = br-lan +lxc.net.0.flags = up +lxc.net.0.ipv4.address = ${ip}/24 +lxc.net.0.ipv4.gateway = 192.168.255.1 + +# Mounts +lxc.mount.entry = ${DATA_DIR}/config opt/maltego/config none bind,create=dir 0 0 +lxc.mount.entry = ${DATA_DIR}/transforms opt/maltego/transforms none bind,create=dir 0 0 +lxc.mount.entry = ${DATA_DIR}/logs opt/maltego/logs none bind,create=dir 0 0 + +# Resources +lxc.cgroup2.memory.max = 1073741824 + +# Autostart +lxc.start.auto = 1 +lxc.start.delay = 5 +EOF + + # Create startup script + cat > "${rootfs}/opt/start-maltego.sh" << 'STARTUP' +#!/bin/bash +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-arm64" +export MALTEGO_HOME="/opt/maltego" + +# Set DNS +echo "nameserver 192.168.255.1" > /etc/resolv.conf + +# Wait for network +sleep 5 + +# Install dependencies if needed +if ! command -v java &>/dev/null; then + apt-get update + apt-get install -y openjdk-17-jre-headless python3 python3-pip wget curl unzip +fi + +# Create maltego directory structure +mkdir -p /opt/maltego/{config,transforms,logs} + +# If iTDS not installed, set up basic transform server +if [ ! -f /opt/maltego/itds.jar ]; then + echo "iTDS not found. Setting up transform server..." + + # Create simple Python transform server + cat > /opt/maltego/transform_server.py << 'PYEOF' +#!/usr/bin/env python3 +""" +Maltego Transform Server +Lightweight TDS-compatible transform endpoint +""" +import http.server +import socketserver +import json +import xml.etree.ElementTree as ET +from urllib.parse import parse_qs, urlparse +import sys +import os +import importlib.util +import traceback + +PORT = 9001 +TRANSFORMS_DIR = "/opt/maltego/transforms" + +class MaltegoEntity: + def __init__(self, entity_type, value, weight=100): + self.type = entity_type + self.value = value + self.weight = weight + self.properties = {} + + def add_property(self, name, value, display_name=None): + self.properties[name] = { + 'value': value, + 'display': display_name or name + } + +class TransformResponse: + def __init__(self): + self.entities = [] + self.messages = [] + + def add_entity(self, entity): + self.entities.append(entity) + + def add_message(self, msg, msg_type="Inform"): + self.messages.append({'text': msg, 'type': msg_type}) + + def to_xml(self): + root = ET.Element("MaltegoMessage") + resp = ET.SubElement(root, "MaltegoTransformResponseMessage") + entities = ET.SubElement(resp, "Entities") + + for e in self.entities: + entity = ET.SubElement(entities, "Entity", Type=e.type) + value = ET.SubElement(entity, "Value") + value.text = e.value + weight = ET.SubElement(entity, "Weight") + weight.text = str(e.weight) + + if e.properties: + props = ET.SubElement(entity, "AdditionalFields") + for name, prop in e.properties.items(): + field = ET.SubElement(props, "Field", Name=name, DisplayName=prop['display']) + field.text = str(prop['value']) + + if self.messages: + msgs = ET.SubElement(resp, "UIMessages") + for m in self.messages: + msg = ET.SubElement(msgs, "UIMessage", MessageType=m['type']) + msg.text = m['text'] + + return ET.tostring(root, encoding='unicode') + +def load_transform(name): + """Load a transform from the transforms directory""" + path = os.path.join(TRANSFORMS_DIR, f"{name}.py") + if not os.path.exists(path): + return None + + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + +class TransformHandler(http.server.BaseHTTPRequestHandler): + def log_message(self, format, *args): + with open("/opt/maltego/logs/access.log", "a") as f: + f.write(f"{self.address_string()} - {format % args}\n") + + def do_GET(self): + """Handle transform list and info""" + parsed = urlparse(self.path) + + if parsed.path == "/" or parsed.path == "/transforms": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + + transforms = [] + if os.path.exists(TRANSFORMS_DIR): + for f in os.listdir(TRANSFORMS_DIR): + if f.endswith(".py"): + transforms.append(f[:-3]) + + self.wfile.write(json.dumps({ + "status": "ok", + "server": "SecuBox Maltego Transform Server", + "version": "1.0.0", + "transforms": transforms + }).encode()) + else: + self.send_response(404) + self.end_headers() + + def do_POST(self): + """Execute a transform""" + parsed = urlparse(self.path) + transform_name = parsed.path.strip("/").replace("run/", "") + + content_length = int(self.headers.get('Content-Length', 0)) + post_data = self.rfile.read(content_length).decode('utf-8') + + response = TransformResponse() + + try: + # Parse input entity from Maltego XML + root = ET.fromstring(post_data) + entity_elem = root.find(".//Entity") + entity_value = entity_elem.find("Value").text if entity_elem is not None else "" + entity_type = entity_elem.get("Type", "") if entity_elem is not None else "" + + # Load and run transform + transform = load_transform(transform_name) + if transform and hasattr(transform, 'run'): + transform.run(entity_value, entity_type, response) + else: + response.add_message(f"Transform '{transform_name}' not found", "Error") + + except Exception as e: + response.add_message(f"Transform error: {str(e)}", "Error") + traceback.print_exc() + + self.send_response(200) + self.send_header("Content-Type", "application/xml") + self.end_headers() + self.wfile.write(response.to_xml().encode()) + +if __name__ == "__main__": + os.makedirs(TRANSFORMS_DIR, exist_ok=True) + + # Create example transform if none exist + if not os.listdir(TRANSFORMS_DIR): + with open(os.path.join(TRANSFORMS_DIR, "dns_lookup.py"), "w") as f: + f.write('''"""DNS Lookup Transform""" +import socket + +def run(value, entity_type, response): + """Look up IP addresses for a domain""" + try: + ips = socket.gethostbyname_ex(value)[2] + for ip in ips: + entity = response.__class__.__bases__[0].__subclasses__()[0] + # Use parent module's MaltegoEntity + import sys + parent = sys.modules[__name__.rsplit(".", 1)[0]] if "." in __name__ else sys.modules["__main__"] + e = parent.MaltegoEntity("maltego.IPv4Address", ip) + e.add_property("domain", value, "Source Domain") + response.add_entity(e) + except Exception as ex: + response.add_message(f"DNS lookup failed: {ex}", "Error") +''') + + with socketserver.TCPServer(("", PORT), TransformHandler) as httpd: + print(f"Maltego Transform Server running on port {PORT}") + httpd.serve_forever() +PYEOF + chmod +x /opt/maltego/transform_server.py +fi + +# Start transform server +cd /opt/maltego +exec python3 transform_server.py >> /opt/maltego/logs/server.log 2>&1 +STARTUP + chmod +x "${rootfs}/opt/start-maltego.sh" + + # Set init + rm -f "${rootfs}/sbin/init" + ln -sf /opt/start-maltego.sh "${rootfs}/sbin/init" + + log "Maltego container installed at ${ip}" + log "Start with: maltegoctl start" +} + +cmd_start() { + log "Starting Maltego container..." + /usr/bin/lxc-start -n "$CONTAINER" || error "Failed to start container" + + # Wait for it to be ready + sleep 10 + + local ip + get_config ip ip "192.168.255.55" + + if curl -s -o /dev/null -w "%{http_code}" "http://${ip}:9001/" | grep -q "200"; then + log "Maltego transform server is ready at http://${ip}:9001/" + else + log "Container started, waiting for transform server..." + fi +} + +cmd_stop() { + log "Stopping Maltego container..." + /usr/bin/lxc-stop -n "$CONTAINER" 2>/dev/null + log "Container stopped" +} + +cmd_restart() { + cmd_stop + sleep 2 + cmd_start +} + +cmd_status() { + local state + state=$(/usr/bin/lxc-info -n "$CONTAINER" -s 2>/dev/null | awk '{print $2}') + + local ip + get_config ip ip "192.168.255.55" + + echo "Container: $CONTAINER" + echo "State: ${state:-NOT_FOUND}" + echo "IP: $ip" + echo "Transform Port: 9001" + + if [ "$state" = "RUNNING" ]; then + local status + status=$(curl -s "http://${ip}:9001/" 2>/dev/null | head -1) + if [ -n "$status" ]; then + echo "Server Status: Running" + echo "Transforms: $(curl -s "http://${ip}:9001/transforms" 2>/dev/null | grep -oP '"transforms":\s*\[\K[^\]]+' | tr ',' '\n' | wc -l)" + else + echo "Server Status: Starting..." + fi + fi +} + +cmd_logs() { + local ip + get_config ip ip "192.168.255.55" + + if [ -f "${DATA_DIR}/logs/server.log" ]; then + tail -f "${DATA_DIR}/logs/server.log" + else + /usr/bin/lxc-attach -n "$CONTAINER" -- tail -f /opt/maltego/logs/server.log + fi +} + +cmd_shell() { + /usr/bin/lxc-attach -n "$CONTAINER" -- /bin/bash +} + +cmd_add_transform() { + local name="$1" + local code="$2" + + if [ -z "$name" ]; then + error "Usage: maltegoctl add-transform [code_file]" + fi + + local target="${DATA_DIR}/transforms/${name}.py" + + if [ -n "$code" ] && [ -f "$code" ]; then + cp "$code" "$target" + else + cat > "$target" << 'TEMPLATE' +""" +Custom Maltego Transform +Edit this file to implement your transform logic +""" + +def run(value, entity_type, response): + """ + Transform entry point + + Args: + value: The entity value (e.g., domain name, IP address) + entity_type: The Maltego entity type + response: TransformResponse object to add results to + + Example: + # Import parent module for MaltegoEntity + import sys + parent = sys.modules["__main__"] + + # Create result entity + e = parent.MaltegoEntity("maltego.Phrase", f"Result for: {value}") + e.add_property("source", value, "Source Value") + response.add_entity(e) + """ + import sys + parent = sys.modules["__main__"] + + e = parent.MaltegoEntity("maltego.Phrase", f"Transform result for: {value}") + response.add_entity(e) +TEMPLATE + log "Created transform template: ${target}" + log "Edit ${target} to implement your transform" + fi +} + +cmd_list_transforms() { + echo "Installed transforms:" + if [ -d "${DATA_DIR}/transforms" ]; then + for f in "${DATA_DIR}/transforms"/*.py; do + [ -f "$f" ] && echo " - $(basename "$f" .py)" + done + else + echo " (none)" + fi +} + +cmd_configure_haproxy() { + local domain + get_config domain exposure domain "maltego.gk2.secubox.in" + + local ip + get_config ip ip "192.168.255.55" + + log "Configuring HAProxy vhost: $domain -> $ip:9001" + + # Add vhost + haproxyctl vhost add "$domain" 2>/dev/null || true + + # Add mitmproxy route + local routes="/srv/mitmproxy/haproxy-routes.json" + if [ -f "$routes" ]; then + python3 -c " +import json +with open('$routes') as f: + d = json.load(f) +d['$domain'] = ['$ip', 9001] +with open('$routes', 'w') as f: + json.dump(d, f, indent=2) +" + cp "$routes" /srv/mitmproxy-in/haproxy-routes.json + /etc/init.d/mitmproxy restart + fi + + log "HAProxy configured: https://$domain/" +} + +cmd_help() { + cat << 'EOF' +Maltego iTDS Controller for SecuBox + +Usage: maltegoctl [options] + +Container Commands: + install Create Maltego transform server container + start Start the container + stop Stop the container + restart Restart the container + status Show container and server status + logs Tail transform server logs + shell Open shell in container + +Transform Commands: + add-transform [file] Create or import a transform + list-transforms List installed transforms + +Configuration: + configure-haproxy Setup HAProxy vhost for external access + +Examples: + maltegoctl install + maltegoctl start + maltegoctl add-transform shodan_lookup + maltegoctl list-transforms +EOF +} + +case "$1" in + install) cmd_install ;; + start) cmd_start ;; + stop) cmd_stop ;; + restart) cmd_restart ;; + status) cmd_status ;; + logs) cmd_logs ;; + shell) cmd_shell ;; + add-transform) shift; cmd_add_transform "$@" ;; + list-transforms) cmd_list_transforms ;; + configure-haproxy) cmd_configure_haproxy ;; + help|--help|-h) cmd_help ;; + *) cmd_help ;; +esac diff --git a/package/secubox/secubox-app-sherlock/Makefile b/package/secubox/secubox-app-sherlock/Makefile new file mode 100644 index 00000000..f1cc8e61 --- /dev/null +++ b/package/secubox/secubox-app-sherlock/Makefile @@ -0,0 +1,53 @@ +include $(TOPDIR)/rules.mk + +PKG_NAME:=secubox-app-sherlock +PKG_VERSION:=1.0.0 +PKG_RELEASE:=1 + +PKG_MAINTAINER:=SecuBox +PKG_LICENSE:=MIT + +include $(INCLUDE_DIR)/package.mk + +define Package/secubox-app-sherlock + SECTION:=secubox + CATEGORY:=SecuBox + SUBMENU:=Applications + TITLE:=Sherlock OSINT Username Hunter + DEPENDS:=+lxc +lxc-attach +lxc-start +lxc-stop +lxc-info +curl +python3 + PKGARCH:=all +endef + +define Package/secubox-app-sherlock/description + Sherlock OSINT tool for hunting usernames across social networks. + Runs in an LXC container with REST API for web integration. +endef + +define Package/secubox-app-sherlock/conffiles +/etc/config/sherlock +endef + +define Build/Compile +endef + +define Package/secubox-app-sherlock/install + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_CONF) ./files/etc/config/sherlock $(1)/etc/config/ + + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/etc/init.d/sherlock $(1)/etc/init.d/ + + $(INSTALL_DIR) $(1)/usr/sbin + $(INSTALL_BIN) ./files/usr/sbin/sherlockctl $(1)/usr/sbin/ +endef + +define Package/secubox-app-sherlock/postinst +#!/bin/sh +[ -n "$${IPKG_INSTROOT}" ] || { + mkdir -p /srv/sherlock/{results,cache} + /etc/init.d/sherlock enable +} +exit 0 +endef + +$(eval $(call BuildPackage,secubox-app-sherlock)) diff --git a/package/secubox/secubox-app-sherlock/files/etc/config/sherlock b/package/secubox/secubox-app-sherlock/files/etc/config/sherlock new file mode 100644 index 00000000..f27464a8 --- /dev/null +++ b/package/secubox/secubox-app-sherlock/files/etc/config/sherlock @@ -0,0 +1,14 @@ +config sherlock 'main' + option enabled '1' + option container 'sherlock' + option image 'debian' + option version 'bookworm' + option memory '512' + option ip '192.168.255.56' + option port '9090' + option data_dir '/srv/sherlock' + +config haproxy 'exposure' + option domain 'sherlock.gk2.secubox.in' + option ssl '1' + option backend_port '9090' diff --git a/package/secubox/secubox-app-sherlock/files/etc/init.d/sherlock b/package/secubox/secubox-app-sherlock/files/etc/init.d/sherlock new file mode 100644 index 00000000..c0fcca4e --- /dev/null +++ b/package/secubox/secubox-app-sherlock/files/etc/init.d/sherlock @@ -0,0 +1,36 @@ +#!/bin/sh /etc/rc.common + +START=95 +STOP=10 +USE_PROCD=1 + +CONTAINER="sherlock" + +start_service() { + local enabled + config_load sherlock + config_get enabled main enabled 0 + + [ "$enabled" = "1" ] || return 0 + + procd_open_instance + procd_set_param command /usr/bin/lxc-start -n "$CONTAINER" -F + procd_set_param respawn 3600 5 5 + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_close_instance +} + +stop_service() { + /usr/bin/lxc-stop -n "$CONTAINER" 2>/dev/null +} + +reload_service() { + stop_service + sleep 2 + start_service +} + +service_triggers() { + procd_add_reload_trigger "sherlock" +} diff --git a/package/secubox/secubox-app-sherlock/files/usr/sbin/sherlockctl b/package/secubox/secubox-app-sherlock/files/usr/sbin/sherlockctl new file mode 100644 index 00000000..cb3db89a --- /dev/null +++ b/package/secubox/secubox-app-sherlock/files/usr/sbin/sherlockctl @@ -0,0 +1,415 @@ +#!/bin/sh +# Sherlock OSINT Controller for SecuBox +# Username hunting across social networks + +. /lib/functions.sh +config_load sherlock + +CONTAINER="sherlock" +LXC_PATH="/srv/lxc" +DATA_DIR="/srv/sherlock" +IMAGE_URL="https://images.linuxcontainers.org/images/debian/bookworm/arm64/default" + +log() { + logger -t sherlockctl "$1" + echo "[INFO] $1" +} + +error() { + logger -t sherlockctl -p err "$1" + echo "[ERROR] $1" >&2 + exit 1 +} + +get_config() { + config_get "$1" main "$2" "$3" +} + +cmd_install() { + log "Installing Sherlock container..." + + if [ -d "${LXC_PATH}/${CONTAINER}" ]; then + log "Container already exists. Use 'reinstall' to recreate." + return 0 + fi + + local ip + get_config ip ip "192.168.255.56" + + mkdir -p "${DATA_DIR}"/{results,cache} + + log "Downloading Debian Bookworm image..." + mkdir -p "${LXC_PATH}/${CONTAINER}" + + local rootfs="${LXC_PATH}/${CONTAINER}/rootfs" + mkdir -p "$rootfs" + + # Get latest image date + local image_date + image_date=$(wget -qO- "${IMAGE_URL}/" | grep -o '[0-9]\{8\}_[0-9]\{2\}:[0-9]\{2\}' | tail -1) + + if [ -z "$image_date" ]; then + error "Could not determine latest image date" + fi + + log "Downloading image: ${image_date}" + wget -q "${IMAGE_URL}/${image_date}/rootfs.tar.xz" -O /tmp/sherlock-rootfs.tar.xz || \ + error "Failed to download image" + + tar -xJf /tmp/sherlock-rootfs.tar.xz -C "$rootfs" + rm -f /tmp/sherlock-rootfs.tar.xz + + # Create data directory in rootfs + mkdir -p "${rootfs}/srv/sherlock" + + # Create LXC config + cat > "${LXC_PATH}/${CONTAINER}/config" << EOF +lxc.uts.name = ${CONTAINER} +lxc.rootfs.path = dir:${rootfs} +lxc.include = /usr/share/lxc/config/common.conf + +# Network +lxc.net.0.type = veth +lxc.net.0.link = br-lan +lxc.net.0.flags = up +lxc.net.0.ipv4.address = ${ip}/24 +lxc.net.0.ipv4.gateway = 192.168.255.1 + +# Data mount +lxc.mount.entry = /srv/sherlock /srv/sherlock none bind 0 0 + +# Autostart +lxc.start.auto = 1 +lxc.start.delay = 5 +EOF + + # Create startup script + cat > "${rootfs}/opt/start-sherlock.sh" << 'STARTUP' +#!/bin/bash +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +export HOME="/root" + +# Set DNS +echo "nameserver 192.168.255.1" > /etc/resolv.conf + +# Wait for network +sleep 5 + +# Install dependencies if needed +if ! command -v sherlock &>/dev/null; then + apt-get update + apt-get install -y python3 python3-pip git curl + + # Install sherlock + pip3 install sherlock-project --break-system-packages + + # Also clone repo for data files + git clone https://github.com/sherlock-project/sherlock.git /opt/sherlock-git +fi + +# Create web API server +cat > /opt/sherlock_api.py << 'PYEOF' +#!/usr/bin/env python3 +""" +Sherlock Web API Server +REST API for username hunting across social networks +""" +import http.server +import socketserver +import json +import subprocess +import os +import threading +import time +from urllib.parse import parse_qs, urlparse +import uuid + +PORT = 9090 +RESULTS_DIR = "/srv/sherlock/results" +CACHE_DIR = "/srv/sherlock/cache" + +# Store running tasks +tasks = {} + +os.makedirs(RESULTS_DIR, exist_ok=True) +os.makedirs(CACHE_DIR, exist_ok=True) + +def run_sherlock(username, task_id): + """Run sherlock search in background""" + output_file = os.path.join(RESULTS_DIR, f"{task_id}.json") + + try: + result = subprocess.run( + ["sherlock", username, "--output", output_file, "--json", os.path.join(RESULTS_DIR, f"{task_id}_raw.json")], + capture_output=True, + text=True, + timeout=300 + ) + + # Parse results + found_accounts = [] + raw_file = os.path.join(RESULTS_DIR, f"{task_id}_raw.json") + if os.path.exists(raw_file): + with open(raw_file) as f: + data = json.load(f) + for site, info in data.items(): + if info.get("status") == "Claimed": + found_accounts.append({ + "site": site, + "url": info.get("url_user", ""), + "response_time": info.get("response_time_s", 0) + }) + + tasks[task_id] = { + "status": "completed", + "username": username, + "found_count": len(found_accounts), + "accounts": found_accounts, + "completed_at": time.time() + } + + except subprocess.TimeoutExpired: + tasks[task_id] = {"status": "timeout", "username": username} + except Exception as e: + tasks[task_id] = {"status": "error", "username": username, "error": str(e)} + +class SherlockHandler(http.server.BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass # Suppress logging + + def send_json(self, data, status=200): + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(json.dumps(data).encode()) + + def do_OPTIONS(self): + self.send_response(200) + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + self.end_headers() + + def do_GET(self): + parsed = urlparse(self.path) + path = parsed.path + + if path == "/" or path == "/api": + self.send_json({ + "status": "ok", + "server": "SecuBox Sherlock OSINT Server", + "version": "1.0.0", + "endpoints": { + "/api/search?username=": "Start username search (returns task_id)", + "/api/task/": "Get task status/results", + "/api/tasks": "List all tasks" + } + }) + + elif path == "/api/tasks": + self.send_json({"tasks": tasks}) + + elif path.startswith("/api/task/"): + task_id = path.split("/")[-1] + if task_id in tasks: + self.send_json(tasks[task_id]) + else: + self.send_json({"error": "Task not found"}, 404) + + elif path == "/api/search": + params = parse_qs(parsed.query) + username = params.get("username", [None])[0] + + if not username: + self.send_json({"error": "username parameter required"}, 400) + return + + # Check cache + cache_file = os.path.join(CACHE_DIR, f"{username}.json") + if os.path.exists(cache_file): + cache_age = time.time() - os.path.getmtime(cache_file) + if cache_age < 3600: # 1 hour cache + with open(cache_file) as f: + self.send_json({"cached": True, **json.load(f)}) + return + + # Start new search + task_id = str(uuid.uuid4())[:8] + tasks[task_id] = {"status": "running", "username": username, "started_at": time.time()} + + thread = threading.Thread(target=run_sherlock, args=(username, task_id)) + thread.daemon = True + thread.start() + + self.send_json({"task_id": task_id, "status": "started", "username": username}) + + else: + self.send_json({"error": "Not found"}, 404) + +if __name__ == "__main__": + with socketserver.TCPServer(("", PORT), SherlockHandler) as httpd: + print(f"Sherlock API Server running on port {PORT}") + httpd.serve_forever() +PYEOF +chmod +x /opt/sherlock_api.py + +# Start API server +cd /opt +exec python3 sherlock_api.py +STARTUP + chmod +x "${rootfs}/opt/start-sherlock.sh" + + # Set init + rm -f "${rootfs}/sbin/init" + ln -sf /opt/start-sherlock.sh "${rootfs}/sbin/init" + + log "Sherlock container installed at ${ip}" + log "Start with: sherlockctl start" +} + +cmd_start() { + log "Starting Sherlock container..." + /usr/bin/lxc-start -n "$CONTAINER" || error "Failed to start container" + + sleep 10 + + local ip + get_config ip ip "192.168.255.56" + + if curl -s -o /dev/null -w "%{http_code}" "http://${ip}:9090/" | grep -q "200"; then + log "Sherlock API is ready at http://${ip}:9090/" + else + log "Container started, waiting for API server..." + fi +} + +cmd_stop() { + log "Stopping Sherlock container..." + /usr/bin/lxc-stop -n "$CONTAINER" 2>/dev/null + log "Container stopped" +} + +cmd_restart() { + cmd_stop + sleep 2 + cmd_start +} + +cmd_status() { + local state + state=$(/usr/bin/lxc-info -n "$CONTAINER" -s 2>/dev/null | awk '{print $2}') + + local ip + get_config ip ip "192.168.255.56" + + echo "Container: $CONTAINER" + echo "State: ${state:-NOT_FOUND}" + echo "IP: $ip" + echo "API Port: 9090" + + if [ "$state" = "RUNNING" ]; then + local status + status=$(curl -s "http://${ip}:9090/api" 2>/dev/null) + if [ -n "$status" ]; then + echo "API Status: Running" + else + echo "API Status: Starting..." + fi + fi +} + +cmd_search() { + local username="$1" + if [ -z "$username" ]; then + error "Usage: sherlockctl search " + fi + + local ip + get_config ip ip "192.168.255.56" + + local result + result=$(curl -s "http://${ip}:9090/api/search?username=${username}") + echo "$result" +} + +cmd_logs() { + /usr/bin/lxc-attach -n "$CONTAINER" -- tail -f /var/log/sherlock.log 2>/dev/null || \ + /usr/bin/lxc-attach -n "$CONTAINER" -- cat /proc/1/fd/1 +} + +cmd_shell() { + /usr/bin/lxc-attach -n "$CONTAINER" -- /bin/bash +} + +cmd_configure_haproxy() { + local domain + get_config domain exposure domain "sherlock.gk2.secubox.in" + + local ip + get_config ip ip "192.168.255.56" + + log "Configuring HAProxy vhost: $domain -> $ip:9090" + + haproxyctl vhost add "$domain" 2>/dev/null || true + + local routes="/srv/mitmproxy/haproxy-routes.json" + if [ -f "$routes" ]; then + python3 -c " +import json +with open('$routes') as f: + d = json.load(f) +d['$domain'] = ['$ip', 9090] +with open('$routes', 'w') as f: + json.dump(d, f, indent=2) +" + cp "$routes" /srv/mitmproxy-in/haproxy-routes.json + /etc/init.d/mitmproxy restart + fi + + log "HAProxy configured: https://$domain/" +} + +cmd_help() { + cat << 'EOF' +Sherlock OSINT Controller for SecuBox +Username hunting across social networks + +Usage: sherlockctl [options] + +Container Commands: + install Create Sherlock container + start Start the container + stop Stop the container + restart Restart the container + status Show container status + logs View logs + shell Open shell in container + +OSINT Commands: + search Search for username across platforms + +Configuration: + configure-haproxy Setup HAProxy vhost for external access + +Examples: + sherlockctl install + sherlockctl start + sherlockctl search johndoe + curl "http://192.168.255.56:9090/api/search?username=johndoe" +EOF +} + +case "$1" in + install) cmd_install ;; + start) cmd_start ;; + stop) cmd_stop ;; + restart) cmd_restart ;; + status) cmd_status ;; + logs) cmd_logs ;; + shell) cmd_shell ;; + search) shift; cmd_search "$@" ;; + configure-haproxy) cmd_configure_haproxy ;; + help|--help|-h) cmd_help ;; + *) cmd_help ;; +esac