From e39c2a3c7a4d576d9f4dfb9b207eea891102410d Mon Sep 17 00:00:00 2001 From: ashisgreat22 Date: Sat, 21 Mar 2026 21:51:06 +0000 Subject: [PATCH] docs: add gosearch implementation plan Add comprehensive step-by-step implementation plan for integrating gosearch (kafka) metasearch engine into the NixOS VPS configuration. Co-Authored-By: Claude Opus 4.6 --- .../plans/2026-03-21-gosearch-integration.md | 399 ++++++++++++++++++ 1 file changed, 399 insertions(+) create mode 100644 docs/superpowers/plans/2026-03-21-gosearch-integration.md diff --git a/docs/superpowers/plans/2026-03-21-gosearch-integration.md b/docs/superpowers/plans/2026-03-21-gosearch-integration.md new file mode 100644 index 0000000..6cda1e7 --- /dev/null +++ b/docs/superpowers/plans/2026-03-21-gosearch-integration.md @@ -0,0 +1,399 @@ +# gosearch Integration Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Integrate gosearch (kafka) metasearch engine into the NixOS VPS configuration as a hardened systemd service with Nginx reverse proxy and Valkey caching. + +**Architecture:** Add gosearch as a flake input, import its native NixOS module, configure service to run on localhost:8889, connect to shared Valkey cache (DB index 1), and expose via Nginx at search2.ashisgreat.xyz. + +**Tech Stack:** NixOS flakes, systemd, Podman (for Valkey), Nginx, SOPS for secrets + +--- + +## File Structure + +**Files to create:** +- None (all modifications to existing files) + +**Files to modify:** +| File | Purpose | +|------|---------| +| `flake.nix` | Add gosearch flake input | +| `configuration.nix` | Import kafka module, configure service, add SOPS template, add systemd hardening, add nginx domain | +| `modules/searxng.nix` | Add Valkey host port forward for gosearch access | + +--- + +## Task 1: Add gosearch as Flake Input + +**Files:** +- Modify: `flake.nix` + +- [ ] **Step 1: Add gosearch input to flake.nix** + +The gosearch input goes in the `inputs` attribute set. Place it alphabetically after existing inputs. + +```nix +inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + sops-nix.url = "github:Mic92/sops-nix"; + gosearch.url = "git+ssh://forgejo@git.ashisgreat.xyz:2222/penal-colony/gosearch.git"; + openclaw-superpowers = { + url = "github:ArchieIndian/openclaw-superpowers"; + flake = false; + }; +}; +``` + +- [ ] **Step 2: Verify flake update works** + +Run: `nix flake update --commit-lockfile` +Expected: Lockfile updates successfully, no errors + +- [ ] **Step 3: Commit** + +```bash +git add flake.nix flake.lock +git commit -m "feat(gosearch): add gosearch flake input" +``` + +--- + +## Task 2: Import kafka Module and Configure Basic Service + +**Files:** +- Modify: `configuration.nix` + +- [ ] **Step 1: Import the kafka module** + +Add to the `imports` array in `configuration.nix` (around line 3, after `./hardware-configuration.nix`): + +```nix +imports = [ + ./hardware-configuration.nix + inputs.gosearch.nixosModules.default # Add this line +]; +``` + +- [ ] **Step 2: Add basic service configuration** + +Add after the OpenWeb UI configuration block (around line 240, before `# === Backups ===`): + +```nix +# === gosearch (kafka) Metasearch Engine === +services.kafka = { + enable = true; + port = 8889; + baseUrl = "https://search2.ashisgreat.xyz"; + config = config.sops.templates."kafka-config.toml".path; +}; +``` + +- [ ] **Step 3: Test configuration build** + +Run: `nixos-rebuild build --flake .#nixos` +Expected: Build succeeds (may have warnings about missing config file, that's expected at this stage) + +- [ ] **Step 4: Commit** + +```bash +git add configuration.nix +git commit -m "feat(gosearch): import kafka module and configure basic service" +``` + +--- + +## Task 3: Create SOPS Config Template + +**Files:** +- Modify: `configuration.nix` + +- [ ] **Step 1: Add kafka-config.toml SOPS template** + +Add the SOPS template after the OpenClaw SOPS templates block (around line 175): + +```nix +# gosearch/kafka config template +sops.templates."kafka-config.toml" = { + content = '' + [server] + port = 8889 + http_timeout = "10s" + base_url = "https://search2.ashisgreat.xyz" + + [engines] + local_ported = ["wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"] + + [engines.brave] + api_key = "${config.sops.placeholder.openclaw_brave_api_key}" + + [cache] + address = "127.0.0.1:6379" + db = 1 + default_ttl = "5m" + + [cors] + allowed_origins = ["*"] + + [rate_limit] + requests = 30 + window = "1m" + cleanup_interval = "5m" + + [global_rate_limit] + requests = 0 + window = "1m" + + [burst_rate_limit] + burst = 0 + burst_window = "5s" + sustained = 0 + sustained_window = "1m" + ''; +}; +``` + +- [ ] **Step 2: Test configuration build** + +Run: `nixos-rebuild build --flake .#nixos` +Expected: Build succeeds + +- [ ] **Step 3: Commit** + +```bash +git add configuration.nix +git commit -m "feat(gosearch): add SOPS config template for kafka" +``` + +--- + +## Task 4: Add Systemd Hardening + +**Files:** +- Modify: `configuration.nix` + +- [ ] **Step 1: Add systemd service hardening** + +Add the hardening configuration after the kafka service block (added in Task 2): + +```nix +# Hardening for kafka service +systemd.services.kafka.serviceConfig = { + # Capability bounds + CapabilityBoundingSet = [ "" ]; + AmbientCapabilities = [ "" ]; + + # Filesystem + ProtectSystem = "strict"; + ProtectHome = true; + ReadWritePaths = [ "/var/lib/kafka" ]; + PrivateTmp = true; + + # Network + PrivateDevices = true; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; + + # Process isolation + ProtectProc = "invisible"; + ProcSubset = "pid"; + NoNewPrivileges = true; + ProtectClock = true; + ProtectHostname = true; + + # System call filtering + SystemCallFilter = [ "@system-service" "~@privileged" ]; + SystemCallArchitectures = "native"; + + # Memory + MemoryDenyWriteExecute = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + + # Resource limits + RestrictNamespaces = true; + LockPersonality = true; + RemoveIPC = true; +}; +``` + +- [ ] **Step 2: Test configuration build** + +Run: `nixos-rebuild build --flake .#nixos` +Expected: Build succeeds + +- [ ] **Step 3: Commit** + +```bash +git add configuration.nix +git commit -m "feat(gosearch): add systemd hardening for kafka service" +``` + +--- + +## Task 5: Add Nginx Domain Configuration + +**Files:** +- Modify: `configuration.nix` + +- [ ] **Step 1: Add search2 domain to nginx configuration** + +Find the `myModules.nginx.domains` section (around line 127) and add the new domain after the search.ashisgreat.xyz entry: + +```nix +myModules.nginx = { + enable = true; + email = "info@ashisgreat.xyz"; + rateLimit = { + enable = true; + requests = 10; + burst = 20; + }; + domains = { + "search.ashisgreat.xyz" = { + port = 8888; + contentSecurityPolicy = null; + rateLimit.requests = 20; + rateLimit.burst = 40; + }; + "search2.ashisgreat.xyz" = { + port = 8889; + }; + }; +}; +``` + +- [ ] **Step 2: Test configuration build** + +Run: `nixos-rebuild build --flake .#nixos` +Expected: Build succeeds + +- [ ] **Step 3: Commit** + +```bash +git add configuration.nix +git commit -m "feat(gosearch): add search2 domain to nginx" +``` + +--- + +## Task 6: Add Valkey Host Port Forward + +**Files:** +- Modify: `modules/searxng.nix` + +- [ ] **Step 1: Add port mapping to searxng-valkey container** + +Find the `searxng-valkey` container definition in `modules/searxng.nix` (around line 94) and add the `ports` option: + +```nix +virtualisation.oci-containers.containers."searxng-valkey" = { + image = "docker.io/valkey/valkey:alpine"; + cmd = [ "valkey-server" "--save" "" "--appendonly" "no" ]; + extraOptions = [ + "--network=searxng-net" + "--network-alias=valkey" + ]; + ports = [ "127.0.0.1:6379:6379" ]; # Add this line for gosearch access +}; +``` + +- [ ] **Step 2: Test configuration build** + +Run: `nixos-rebuild build --flake .#nixos` +Expected: Build succeeds + +- [ ] **Step 3: Commit** + +```bash +git add modules/searxng.nix +git commit -m "feat(gosearch): add Valkey host port forward for local access" +``` + +--- + +## Task 7: Deploy and Verify + +**Files:** +- None (deployment and testing) + +- [ ] **Step 1: Dry-run rebuild to verify configuration** + +Run: `nixos-rebuild build --flake .#nixos` +Expected: Build completes successfully with no errors + +- [ ] **Step 2: Apply configuration** + +Run: `sudo nixos-rebuild switch --flake .#nixos` +Expected: Configuration applies successfully, services start + +- [ ] **Step 3: Verify kafka service is running** + +Run: `systemctl status kafka` +Expected: Service shows as `active (running)` + +- [ ] **Step 4: Verify systemd hardening is applied** + +Run: `systemctl show kafka | grep -E 'ProtectSystem|ProtectHome|PrivateTmp|NoNewPrivileges'` +Expected: All hardening options show `yes` + +- [ ] **Step 5: Verify web interface responds locally** + +Run: `curl -I http://localhost:8889` +Expected: HTTP 200 OK response + +- [ ] **Step 6: Verify health endpoint** + +Run: `curl http://localhost:8889/healthz` +Expected: `OK` response + +- [ ] **Step 7: Verify nginx proxy works** + +Run: `curl -I https://search2.ashisgreat.xyz` +Expected: HTTP 200 OK response via nginx + +- [ ] **Step 8: Test search functionality** + +Run: `curl "https://search2.ashisgreat.xyz/search?q=test&format=json"` +Expected: JSON response with search results + +- [ ] **Step 9: Check kafka logs for Valkey connection** + +Run: `journalctl -u kafka -n 50` +Expected: No errors about Valkey connection, cache operations visible + +- [ ] **Step 10: Commit final deployment** + +```bash +git add -A +git commit -m "feat(gosearch): complete gosearch integration - service running and verified" +``` + +--- + +## Rollback Procedure (if issues occur) + +If deployment fails: + +```bash +# Rollback to previous generation +sudo nixos-rebuild switch --rollback + +# Or list generations and select a specific one +sudo nixos-rebuild list-generations +sudo nixos-rebuild switch --profile /nix/var/nix/profiles/system --rollback +``` + +--- + +## Verification Checklist + +After completing all tasks: + +- [ ] `nixos-rebuild build` succeeds without errors +- [ ] `systemctl status kafka` shows active (running) +- [ ] `systemctl show kafka | grep ProtectSystem` returns `ProtectSystem=yes` +- [ ] `curl -I http://localhost:8889` returns HTTP 200 +- [ ] `curl -I https://search2.ashisgreat.xyz` returns HTTP 200 +- [ ] `curl "https://search2.ashisgreat.xyz/search?q=nixos&format=json"` returns results +- [ ] `journalctl -u kafka` shows no Valkey connection errors +- [ ] `curl http://localhost:8889/healthz` returns `OK`