From a2f8077669aaee9294c06a9f233000ed6def7736 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 00:20:43 +0000 Subject: [PATCH 01/27] feat: add autocomplete dropdown UI with keyboard nav - Inline JS in base.html: debounced fetch from /autocompleter on keyup - Keyboard nav: arrows to navigate, Enter to select, Esc to close - Highlight matching prefix in suggestions - Click to select and submit - Dropdown positioned absolutely below search input - Dark mode compatible via existing CSS variables --- internal/views/static/css/kafka.css | 57 ++++++++++++++ internal/views/templates/base.html | 117 ++++++++++++++++++++++++++++ internal/views/templates/index.html | 3 +- 3 files changed, 176 insertions(+), 1 deletion(-) diff --git a/internal/views/static/css/kafka.css b/internal/views/static/css/kafka.css index ad794c4..376b2d8 100644 --- a/internal/views/static/css/kafka.css +++ b/internal/views/static/css/kafka.css @@ -421,6 +421,63 @@ footer a:hover { display: block; } +/* Autocomplete dropdown */ +#search { + position: relative; +} + +#autocomplete-dropdown { + position: absolute; + top: 100%; + left: 0; + right: 0; + background: var(--color-base-background); + border: 1px solid var(--color-search-border); + border-top: none; + border-radius: 0 0 var(--radius) var(--radius); + box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); + z-index: 100; + max-height: 320px; + overflow-y: auto; + display: none; +} + +#autocomplete-dropdown.open { + display: block; +} + +.autocomplete-suggestion { + padding: 0.6rem 1rem; + cursor: pointer; + font-size: 0.95rem; + color: var(--color-base-font); + border-bottom: 1px solid var(--color-result-border); + transition: background 0.15s; +} + +.autocomplete-suggestion:last-child { + border-bottom: none; +} + +.autocomplete-suggestion:hover, +.autocomplete-suggestion.active { + background: var(--color-header-background); +} + +.autocomplete-suggestion mark { + background: none; + color: var(--color-link); + font-weight: 600; +} + +.autocomplete-footer { + padding: 0.4rem 1rem; + font-size: 0.75rem; + color: var(--color-suggestion); + border-top: 1px solid var(--color-result-border); + background: var(--color-header-background); +} + /* Responsive */ @media (max-width: 768px) { #results { diff --git a/internal/views/templates/base.html b/internal/views/templates/base.html index 10de540..6572b19 100644 --- a/internal/views/templates/base.html +++ b/internal/views/templates/base.html @@ -20,6 +20,123 @@ + {{end}} diff --git a/internal/views/templates/index.html b/internal/views/templates/index.html index e2ca279..c9df700 100644 --- a/internal/views/templates/index.html +++ b/internal/views/templates/index.html @@ -3,11 +3,12 @@

kafka

From 4482cb4dde5e39a8cfb517182fcfd2a79d7a6c33 Mon Sep 17 00:00:00 2001 From: ashisgreat22 Date: Sun, 22 Mar 2026 01:26:46 +0100 Subject: [PATCH 02/27] docs: update CLAUDE.md with autocomplete package and endpoint Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index bba67e1..1ba6bdc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,7 +37,8 @@ There is no Makefile. There is no linter configured. - `internal/config` — TOML-based configuration with env var fallbacks. `Load(path)` reads `config.toml`; env vars override zero-value fields. See `config.example.toml` for all settings. - `internal/engines` — `Engine` interface and all 9 Go-native implementations. `factory.go` registers engines via `NewDefaultPortedEngines()`. `planner.go` routes engines to local or upstream based on `LOCAL_PORTED_ENGINES` env var. - `internal/search` — `Service` orchestrates the pipeline: cache check, planning, parallel engine execution via goroutines/WaitGroup, upstream proxying, response merging. Individual engine failures are reported as `unresponsive_engines` rather than aborting the search. Qwant has fallback logic to upstream on empty results. -- `internal/httpapi` — HTTP handlers for `/`, `/search`, `/healthz`, `/opensearch.xml`. Detects HTMX requests via `HX-Request` header to return fragments instead of full pages. +- `internal/autocomplete` — Fetches search suggestions. Proxies to upstream SearXNG `/autocompleter` if configured, falls back to Wikipedia OpenSearch API otherwise. +- `internal/httpapi` — HTTP handlers for `/`, `/search`, `/autocompleter`, `/healthz`, `/opensearch.xml`. Detects HTMX requests via `HX-Request` header to return fragments instead of full pages. - `internal/upstream` — Client that proxies requests to an upstream SearXNG instance via POST. - `internal/cache` — Valkey/Redis-backed cache with SHA-256 cache keys. No-op if unconfigured. - `internal/middleware` — Three rate limiters (per-IP sliding window, burst+sustained, global) and CORS. All disabled by default. From fcd9be16df3c3916207ca12dc2bb591e6d0750cc Mon Sep 17 00:00:00 2001 From: ashisgreat22 Date: Sun, 22 Mar 2026 01:47:03 +0100 Subject: [PATCH 03/27] refactor: remove SearXNG references and rename binary to kafka - Rename cmd/searxng-go to cmd/kafka - Remove all SearXNG references from source comments while keeping "SearXNG-compatible API" in user-facing docs - Update binary paths in README, CLAUDE.md, and Dockerfile - Update log message to "kafka starting" Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 10 +++++----- Dockerfile | 2 +- README.md | 12 ++++++------ cmd/{searxng-go => kafka}/main.go | 2 +- config.example.toml | 4 ++-- internal/autocomplete/service.go | 6 +++--- internal/contracts/main_result.go | 10 +++++----- internal/contracts/types.go | 6 +++--- internal/engines/braveapi.go | 8 ++++---- internal/engines/engine.go | 2 +- internal/engines/planner.go | 4 ++-- internal/engines/qwant.go | 10 +++++----- internal/search/merge.go | 2 +- internal/search/request_params.go | 4 ++-- internal/search/response.go | 8 ++++---- internal/search/service.go | 2 +- internal/upstream/client.go | 2 +- internal/views/static/css/kafka.css | 1 - 18 files changed, 47 insertions(+), 48 deletions(-) rename cmd/{searxng-go => kafka}/main.go (98%) diff --git a/CLAUDE.md b/CLAUDE.md index 1ba6bdc..b7f254e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -kafka is a privacy-respecting metasearch engine written in Go. It provides a SearXNG-compatible `/search` API and an HTML frontend (HTMX + Go templates). 9 engines are implemented natively in Go; unlisted engines can be proxied to an upstream SearXNG instance. Responses from multiple engines are merged into a single JSON/CSV/RSS/HTML response. +kafka is a privacy-respecting metasearch engine written in Go. It provides a SearXNG-compatible `/search` API and an HTML frontend (HTMX + Go templates). 9 engines are implemented natively in Go; unlisted engines can be proxied to an upstream metasearch instance. Responses from multiple engines are merged into a single JSON/CSV/RSS/HTML response. ## Build & Run Commands @@ -22,7 +22,7 @@ go test -run TestWikipedia ./internal/engines/ go test -v ./internal/engines/ # Run the server (requires config.toml) -go run ./cmd/searxng-go -config config.toml +go run ./cmd/kafka -config config.toml ``` There is no Makefile. There is no linter configured. @@ -37,13 +37,13 @@ There is no Makefile. There is no linter configured. - `internal/config` — TOML-based configuration with env var fallbacks. `Load(path)` reads `config.toml`; env vars override zero-value fields. See `config.example.toml` for all settings. - `internal/engines` — `Engine` interface and all 9 Go-native implementations. `factory.go` registers engines via `NewDefaultPortedEngines()`. `planner.go` routes engines to local or upstream based on `LOCAL_PORTED_ENGINES` env var. - `internal/search` — `Service` orchestrates the pipeline: cache check, planning, parallel engine execution via goroutines/WaitGroup, upstream proxying, response merging. Individual engine failures are reported as `unresponsive_engines` rather than aborting the search. Qwant has fallback logic to upstream on empty results. -- `internal/autocomplete` — Fetches search suggestions. Proxies to upstream SearXNG `/autocompleter` if configured, falls back to Wikipedia OpenSearch API otherwise. +- `internal/autocomplete` — Fetches search suggestions. Proxies to upstream `/autocompleter` if configured, falls back to Wikipedia OpenSearch API otherwise. - `internal/httpapi` — HTTP handlers for `/`, `/search`, `/autocompleter`, `/healthz`, `/opensearch.xml`. Detects HTMX requests via `HX-Request` header to return fragments instead of full pages. -- `internal/upstream` — Client that proxies requests to an upstream SearXNG instance via POST. +- `internal/upstream` — Client that proxies requests to an upstream metasearch instance via POST. - `internal/cache` — Valkey/Redis-backed cache with SHA-256 cache keys. No-op if unconfigured. - `internal/middleware` — Three rate limiters (per-IP sliding window, burst+sustained, global) and CORS. All disabled by default. - `internal/views` — HTML templates and static files embedded via `//go:embed`. Renders full pages or HTMX fragments. Templates: `base.html`, `index.html`, `results.html`, `results_inner.html`, `result_item.html`. -- `cmd/searxng-go` — Entry point. Loads TOML config, seeds env vars for engine code, wires up middleware chain, starts HTTP server. +- `cmd/kafka` — Entry point. Loads TOML config, seeds env vars for engine code, wires up middleware chain, starts HTTP server. **Engine interface** (`internal/engines/engine.go`): ```go diff --git a/Dockerfile b/Dockerfile index c41b5a1..e21960f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN go mod download # Copy source and build COPY . . -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /kafka ./cmd/searxng-go +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /kafka ./cmd/kafka # Runtime stage FROM alpine:3.21 diff --git a/README.md b/README.md index 2f0868f..c03019e 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ A privacy-respecting, open metasearch engine written in Go. SearXNG-compatible A ```bash git clone https://git.ashisgreat.xyz/penal-colony/gosearch.git cd kafka -go build ./cmd/searxng-go -./searxng-go -config config.toml +go build ./cmd/kafka +./kafka -config config.toml ``` ### Docker Compose @@ -76,7 +76,7 @@ sudo nixos-rebuild switch --flake .# ```bash nix develop go test ./... -go run ./cmd/searxng-go -config config.toml +go run ./cmd/kafka -config config.toml ``` ## Endpoints @@ -138,7 +138,7 @@ Copy `config.example.toml` to `config.toml` and edit. All settings can also be o ### Key Sections - **`[server]`** — port, timeout, public base URL for OpenSearch -- **`[upstream]`** — optional upstream SearXNG proxy for unported engines +- **`[upstream]`** — optional upstream metasearch proxy for unported engines - **`[engines]`** — which engines run locally, engine-specific settings - **`[cache]`** — Valkey/Redis address, password, TTL - **`[cors]`** — allowed origins and methods @@ -152,7 +152,7 @@ Copy `config.example.toml` to `config.toml` and edit. All settings can also be o |---|---| | `PORT` | Listen port (default: 8080) | | `BASE_URL` | Public URL for OpenSearch XML | -| `UPSTREAM_SEARXNG_URL` | Upstream SearXNG instance URL | +| `UPSTREAM_SEARXNG_URL` | Upstream instance URL | | `LOCAL_PORTED_ENGINES` | Comma-separated local engine list | | `HTTP_TIMEOUT` | Upstream request timeout | | `BRAVE_API_KEY` | Brave Search API key | @@ -177,7 +177,7 @@ See `config.example.toml` for the full list including rate limiting and CORS var | Reddit | Reddit JSON API | Discussions | | Bing | Bing RSS | General web | -Engines not listed in `engines.local_ported` are proxied to an upstream SearXNG instance if `upstream.url` is configured. +Engines not listed in `engines.local_ported` are proxied to an upstream metasearch instance if `upstream.url` is configured. ## Architecture diff --git a/cmd/searxng-go/main.go b/cmd/kafka/main.go similarity index 98% rename from cmd/searxng-go/main.go rename to cmd/kafka/main.go index dac6258..ab29852 100644 --- a/cmd/searxng-go/main.go +++ b/cmd/kafka/main.go @@ -103,7 +103,7 @@ func main() { }, logger)(handler) addr := fmt.Sprintf(":%d", cfg.Server.Port) - logger.Info("searxng-go starting", + logger.Info("kafka starting", "addr", addr, "cache", searchCache.Enabled(), "rate_limit", cfg.RateLimit.Requests > 0, diff --git a/config.example.toml b/config.example.toml index df77184..1e3b75c 100644 --- a/config.example.toml +++ b/config.example.toml @@ -15,13 +15,13 @@ http_timeout = "10s" base_url = "" [upstream] -# URL of an upstream SearXNG instance for unported engines (env: UPSTREAM_SEARXNG_URL) +# URL of an upstream metasearch instance for unported engines (env: UPSTREAM_SEARXNG_URL) # Leave empty to run without an upstream proxy. url = "" [engines] # Comma-separated list of engines to execute locally in Go (env: LOCAL_PORTED_ENGINES) -# Engines not listed here will be proxied to upstream SearXNG. +# Engines not listed here will be proxied to the upstream instance. local_ported = ["wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"] [engines.brave] diff --git a/internal/autocomplete/service.go b/internal/autocomplete/service.go index 3892d63..99d963a 100644 --- a/internal/autocomplete/service.go +++ b/internal/autocomplete/service.go @@ -11,7 +11,7 @@ import ( "time" ) -// Service fetches search suggestions from an upstream SearXNG instance +// Service fetches search suggestions from an upstream metasearch instance // or falls back to Wikipedia's OpenSearch API. type Service struct { upstreamURL string @@ -40,7 +40,7 @@ func (s *Service) Suggestions(ctx context.Context, query string) ([]string, erro return s.wikipediaSuggestions(ctx, query) } -// upstreamSuggestions proxies to an upstream SearXNG /autocompleter endpoint. +// upstreamSuggestions proxies to an upstream /autocompleter endpoint. func (s *Service) upstreamSuggestions(ctx context.Context, query string) ([]string, error) { u := s.upstreamURL + "/autocompleter?" + url.Values{"q": {query}}.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) @@ -64,7 +64,7 @@ func (s *Service) upstreamSuggestions(ctx context.Context, query string) ([]stri return nil, err } - // SearXNG /autocompleter returns a plain JSON array of strings. + // The /autocompleter endpoint returns a plain JSON array of strings. var out []string if err := json.Unmarshal(body, &out); err != nil { return nil, err diff --git a/internal/contracts/main_result.go b/internal/contracts/main_result.go index 48005f8..20c9231 100644 --- a/internal/contracts/main_result.go +++ b/internal/contracts/main_result.go @@ -5,15 +5,15 @@ import ( "encoding/json" ) -// MainResult represents one element of SearXNG's `results` array. +// MainResult represents one element of the `results` array. // -// SearXNG returns many additional keys beyond what templates use. To keep the +// The API returns many additional keys beyond what templates use. To keep the // contract stable for proxying/merging, we preserve all unknown keys in // `raw` and re-emit them via MarshalJSON. type MainResult struct { raw map[string]any - // Common fields used by SearXNG templates (RSS uses: title, url, content, pubdate). + // Common fields used by templates (RSS uses: title, url, content, pubdate). Template string `json:"template"` Title string `json:"title"` Content string `json:"content"` @@ -28,12 +28,12 @@ type MainResult struct { Positions []int `json:"positions"` Engines []string `json:"engines"` - // These fields exist in SearXNG's MainResult base; keep them so downstream + // These fields exist in the MainResult base; keep them so downstream // callers can generate richer output later. OpenGroup bool `json:"open_group"` CloseGroup bool `json:"close_group"` - // parsed_url in SearXNG is emitted as a tuple; we preserve it as-is. + // parsed_url is emitted as a tuple; we preserve it as-is. ParsedURL any `json:"parsed_url"` } diff --git a/internal/contracts/types.go b/internal/contracts/types.go index a68f77a..81103ce 100644 --- a/internal/contracts/types.go +++ b/internal/contracts/types.go @@ -1,6 +1,6 @@ package contracts -// OutputFormat matches SearXNG's `/search?format=...` values. +// OutputFormat matches the `/search?format=...` values. type OutputFormat string const ( @@ -28,7 +28,7 @@ type SearchRequest struct { Engines []string Categories []string - // EngineData matches SearXNG's `engine_data--=` parameters. + // EngineData matches the `engine_data--=` parameters. EngineData map[string]map[string]string // AccessToken is an optional request token used to gate paid/limited engines. @@ -36,7 +36,7 @@ type SearchRequest struct { AccessToken string } -// SearchResponse matches the JSON schema returned by SearXNG's `webutils.get_json_response()`. +// SearchResponse matches the JSON schema used by `webutils.get_json_response()`. type SearchResponse struct { Query string `json:"query"` NumberOfResults int `json:"number_of_results"` diff --git a/internal/engines/braveapi.go b/internal/engines/braveapi.go index 2cb20ff..77c7abe 100644 --- a/internal/engines/braveapi.go +++ b/internal/engines/braveapi.go @@ -14,7 +14,7 @@ import ( "github.com/metamorphosis-dev/kafka/internal/contracts" ) -// BraveEngine implements the SearXNG `braveapi` engine (Brave Web Search API). +// BraveEngine implements the `braveapi` engine (Brave Web Search API). // // Config / gating: // - BRAVE_API_KEY: required to call Brave @@ -35,8 +35,8 @@ func (e *BraveEngine) Search(ctx context.Context, req contracts.SearchRequest) ( return contracts.SearchResponse{}, errors.New("brave engine not initialized") } - // Gate / config checks should not be treated as fatal errors; SearXNG - // treats misconfigured engines as unresponsive. + // Gate / config checks should not be treated as fatal errors; the reference + // implementation treats misconfigured engines as unresponsive. if strings.TrimSpace(e.apiKey) == "" { return contracts.SearchResponse{ Query: req.Query, @@ -93,7 +93,7 @@ func (e *BraveEngine) Search(ctx context.Context, req contracts.SearchRequest) ( } } - // SearXNG's python checks `if params["safesearch"]:` which treats any + // The reference implementation checks `if params["safesearch"]:` which treats any // non-zero (moderate/strict) as strict. if req.Safesearch > 0 { args.Set("safesearch", "strict") diff --git a/internal/engines/engine.go b/internal/engines/engine.go index d07aec9..ee87cfd 100644 --- a/internal/engines/engine.go +++ b/internal/engines/engine.go @@ -6,7 +6,7 @@ import ( "github.com/metamorphosis-dev/kafka/internal/contracts" ) -// Engine is a Go-native implementation of a SearXNG engine. +// Engine is a Go-native implementation of a search engine. // // Implementations should return a SearchResponse containing only the results // for that engine subset; the caller will merge multiple engine responses. diff --git a/internal/engines/planner.go b/internal/engines/planner.go index 543f253..56df656 100644 --- a/internal/engines/planner.go +++ b/internal/engines/planner.go @@ -48,7 +48,7 @@ func NewPlanner(portedEngines []string) *Planner { // Plan returns: // - localEngines: engines that are configured as ported for this service -// - upstreamEngines: engines that should be executed by upstream SearXNG +// - upstreamEngines: engines that should be executed by the upstream instance // - requestedEngines: the (possibly inferred) requested engines list // // If the request provides an explicit `engines` parameter, we use it. @@ -80,7 +80,7 @@ func (p *Planner) Plan(req contracts.SearchRequest) (localEngines, upstreamEngin func inferFromCategories(categories []string) []string { // Minimal mapping for the initial porting subset. - // This mirrors the idea of selecting from SearXNG categories without + // This mirrors the idea of selecting from engine categories without // embedding the whole engine registry. set := map[string]bool{} for _, c := range categories { diff --git a/internal/engines/qwant.go b/internal/engines/qwant.go index bb2a03c..8221781 100644 --- a/internal/engines/qwant.go +++ b/internal/engines/qwant.go @@ -14,11 +14,11 @@ import ( "github.com/PuerkitoBio/goquery" ) -// QwantEngine implements a SearXNG-like `qwant` (web) adapter using +// QwantEngine implements a `qwant` (web) adapter using // Qwant v3 endpoint: https://api.qwant.com/v3/search/web. // -// Qwant's API is not fully documented; this mirrors SearXNG's parsing logic -// for the `web` category from `.agent/searxng/searx/engines/qwant.py`. +// Qwant's API is not fully documented; this implements parsing logic +// for the `web` category. type QwantEngine struct { client *http.Client category string // "web" (JSON API) or "web-lite" (HTML fallback) @@ -37,7 +37,7 @@ func (e *QwantEngine) Search(ctx context.Context, req contracts.SearchRequest) ( return contracts.SearchResponse{Query: req.Query}, nil } - // For API parity we use SearXNG web defaults: count=10, offset=(pageno-1)*count. + // For API parity we use web defaults: count=10, offset=(pageno-1)*count. // The engine's config field exists so we can expand to news/images/videos later. count := e.resultsPerPage if count <= 0 { @@ -262,7 +262,7 @@ func (e *QwantEngine) searchWebLite(ctx context.Context, req contracts.SearchReq return } - // In SearXNG: "./span[contains(@class, 'url partner')]" + // Selector: "./span[contains(@class, 'url partner')]" urlText := strings.TrimSpace(item.Find("span.url.partner").First().Text()) if urlText == "" { // fallback: any span with class containing both 'url' and 'partner' diff --git a/internal/search/merge.go b/internal/search/merge.go index 54ff9bb..64ebd6e 100644 --- a/internal/search/merge.go +++ b/internal/search/merge.go @@ -8,7 +8,7 @@ import ( "github.com/metamorphosis-dev/kafka/internal/contracts" ) -// MergeResponses merges multiple SearXNG-compatible JSON responses. +// MergeResponses merges multiple compatible JSON responses. // // MVP merge semantics: // - results are concatenated with a simple de-dup key (engine|title|url) diff --git a/internal/search/request_params.go b/internal/search/request_params.go index 1d48a04..9fdd799 100644 --- a/internal/search/request_params.go +++ b/internal/search/request_params.go @@ -11,7 +11,7 @@ import ( var languageCodeRe = regexp.MustCompile(`^[a-z]{2,3}(-[a-zA-Z]{2})?$`) func ParseSearchRequest(r *http.Request) (SearchRequest, error) { - // SearXNG supports both GET and POST and relies on form values for routing. + // Supports both GET and POST and relies on form values for routing. if err := r.ParseForm(); err != nil { return SearchRequest{}, errors.New("invalid request: cannot parse form") } @@ -90,7 +90,7 @@ func ParseSearchRequest(r *http.Request) (SearchRequest, error) { // engines is an explicit list of engine names. engines := splitCSV(strings.TrimSpace(r.FormValue("engines"))) - // categories and category_ params mirror SearXNG's webadapter parsing. + // categories and category_ params mirror the webadapter parsing. // We don't validate against a registry here; we just preserve the requested values. catSet := map[string]bool{} if catsParam := strings.TrimSpace(r.FormValue("categories")); catsParam != "" { diff --git a/internal/search/response.go b/internal/search/response.go index 3b07096..1a9ce26 100644 --- a/internal/search/response.go +++ b/internal/search/response.go @@ -38,7 +38,7 @@ func WriteSearchResponse(w http.ResponseWriter, format OutputFormat, resp Search } } -// csvRowHeader matches the SearXNG CSV writer key order. +// csvRowHeader matches the CSV writer key order. var csvRowHeader = []string{"title", "url", "content", "host", "engine", "score", "type"} func writeCSV(w http.ResponseWriter, resp SearchResponse) error { @@ -111,14 +111,14 @@ func writeCSV(w http.ResponseWriter, resp SearchResponse) error { func writeRSS(w http.ResponseWriter, resp SearchResponse) error { q := resp.Query - escapedTitle := xmlEscape("SearXNG search: " + q) - escapedDesc := xmlEscape("Search results for \"" + q + "\" - SearXNG") + escapedTitle := xmlEscape("kafka search: " + q) + escapedDesc := xmlEscape("Search results for \"" + q + "\" - kafka") escapedQueryTerms := xmlEscape(q) link := "/search?q=" + url.QueryEscape(q) opensearchQuery := fmt.Sprintf(``, escapedQueryTerms) - // SearXNG template uses the number of results for both totalResults and itemsPerPage. + // The template uses the number of results for both totalResults and itemsPerPage. nr := resp.NumberOfResults var items bytes.Buffer diff --git a/internal/search/service.go b/internal/search/service.go index 91fef2b..62a9308 100644 --- a/internal/search/service.go +++ b/internal/search/service.go @@ -50,7 +50,7 @@ func NewService(cfg ServiceConfig) *Service { } // Search executes the request against local engines (in parallel) and -// optionally upstream SearXNG for unported engines. +// optionally the upstream instance for unported engines. // // Individual engine failures are reported as unresponsive_engines rather // than aborting the entire search. diff --git a/internal/upstream/client.go b/internal/upstream/client.go index 3a11843..64ddec4 100644 --- a/internal/upstream/client.go +++ b/internal/upstream/client.go @@ -68,7 +68,7 @@ func (c *Client) SearchJSON(ctx context.Context, req contracts.SearchRequest, en for engineName, kv := range req.EngineData { for key, value := range kv { - // Mirror SearXNG's naming: `engine_data--=` + // Mirror the naming convention: `engine_data--=` form.Set(fmt.Sprintf("engine_data-%s-%s", engineName, key), value) } } diff --git a/internal/views/static/css/kafka.css b/internal/views/static/css/kafka.css index 376b2d8..824f489 100644 --- a/internal/views/static/css/kafka.css +++ b/internal/views/static/css/kafka.css @@ -1,5 +1,4 @@ /* kafka — clean, minimal search engine CSS */ -/* Inspired by SearXNG's simple theme class conventions */ :root { --color-base: #f5f5f5; From 4be9cf2725ce5245076a128bdb4a873f263c82d9 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 01:25:04 +0000 Subject: [PATCH 04/27] feat: add Google engine using GSA User-Agent scraping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SearXNG approach: use Google Search Appliance (GSA) User-Agent pool — these are whitelisted enterprise identifiers Google trusts. Key techniques: - GSA User-Agent (iPhone OS + GSA/ version) instead of Chrome desktop - CONSENT=YES+ cookie to bypass EU consent wall - Parse /url?q= redirector URLs (unquote + strip &sa= params) - div.MjjYud class for result containers (SearXNG selector) - data-sncf divs for snippets - detect sorry.google.com blocks - Suggestions from ouy7Mc class cards --- internal/engines/factory.go | 3 +- internal/engines/google.go | 271 ++++++++++++++++++++++++++++++++++++ internal/engines/planner.go | 3 +- 3 files changed, 275 insertions(+), 2 deletions(-) create mode 100644 internal/engines/google.go diff --git a/internal/engines/factory.go b/internal/engines/factory.go index 310a20e..937225f 100644 --- a/internal/engines/factory.go +++ b/internal/engines/factory.go @@ -31,6 +31,7 @@ func NewDefaultPortedEngines(client *http.Client) map[string]Engine { "duckduckgo": &DuckDuckGoEngine{client: client}, "github": &GitHubEngine{client: client}, "reddit": &RedditEngine{client: client}, - "bing": &BingEngine{client: client}, + "bing": &BingEngine{client: client}, + "google": &GoogleEngine{client: client}, } } diff --git a/internal/engines/google.go b/internal/engines/google.go new file mode 100644 index 0000000..0371283 --- /dev/null +++ b/internal/engines/google.go @@ -0,0 +1,271 @@ +package engines + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/metamorphosis-dev/kafka/internal/contracts" +) + +// GSA User-Agent pool — these are Google Search Appliance identifiers +// that Google trusts for enterprise search appliance traffic. +var gsaUserAgents = []string{ + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/399.2.845414227 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/406.0.862495628 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/406.0.862495628 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/406.0.862495628 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/399.2.845414227 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_5_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/406.0.862495628 Mobile/15E148 Safari/604.1", +} + +func gsaUA() string { + return gsaUserAgents[0] // deterministic for now; could rotate +} + +type GoogleEngine struct { + client *http.Client +} + +func (e *GoogleEngine) Name() string { return "google" } + +func (e *GoogleEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) { + if strings.TrimSpace(req.Query) == "" { + return contracts.SearchResponse{Query: req.Query}, nil + } + + start := (req.Pageno - 1) * 10 + query := url.QueryEscape(req.Query) + + // Build URL like SearXNG does. + u := fmt.Sprintf( + "https://www.google.com/search?q=%s&filter=0&start=%d&hl=%s&lr=%s&safe=%s", + query, + start, + googleHL(req.Language), + googleUILanguage(req.Language), + googleSafeSearchLevel(req.Safesearch), + ) + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return contracts.SearchResponse{}, err + } + httpReq.Header.Set("User-Agent", gsaUA()) + httpReq.Header.Set("Accept", "*/*") + httpReq.AddCookie(&http.Cookie{Name: "CONSENT", Value: "YES+"}) + + resp, err := e.client.Do(httpReq) + if err != nil { + return contracts.SearchResponse{}, err + } + defer resp.Body.Close() + + // Check for Google block / CAPTCHA page. + if detectGoogleSorry(resp) { + return contracts.SearchResponse{ + Query: req.Query, + NumberOfResults: 0, + Results: nil, + Answers: []map[string]any{}, + Corrections: []string{}, + Infoboxes: []map[string]any{}, + Suggestions: []string{}, + UnresponsiveEngines: [][2]string{{"google", "blocked by Google (CAPTCHA/sorry page)"}}, + }, nil + } + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return contracts.SearchResponse{}, fmt.Errorf("google error: status=%d body=%q", resp.StatusCode, string(body)) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, 128*1024)) + if err != nil { + return contracts.SearchResponse{}, err + } + + results := parseGoogleResults(string(body), req.Query) + return contracts.SearchResponse{ + Query: req.Query, + NumberOfResults: len(results), + Results: results, + Answers: []map[string]any{}, + Corrections: []string{}, + Infoboxes: []map[string]any{}, + Suggestions: extractGoogleSuggestions(string(body)), + UnresponsiveEngines: [][2]string{}, + }, nil +} + +// detectGoogleSorry returns true if the response is a Google block/CAPTCHA page. +func detectGoogleSorry(resp *http.Response) bool { + if resp.Request != nil { + if resp.Request.URL.Host == "sorry.google.com" || strings.HasPrefix(resp.Request.URL.Path, "/sorry") { + return true + } + } + return false +} + +// parseGoogleResults extracts search results from Google's HTML. +// Uses the same selectors as SearXNG: div.MjjYud for result containers. +func parseGoogleResults(body, query string) []contracts.MainResult { + var results []contracts.MainResult + + // SearXNG selector: .//div[contains(@class, "MjjYud")] + // Each result block contains a title link and snippet. + // We simulate the XPath matching with regex-based extraction. + + // Find all MjjYud div blocks. + mjjPattern := regexp.MustCompile(`]*class="[^"]*MjjYud[^"]*"[^>]*>(.*?)\s*(?=]*class="[^"]*MjjYud|$)`) + matches := mjjPattern.FindAllStringSubmatch(body, -1) + + for i, match := range matches { + if len(match) < 2 { + continue + } + block := match[1] + + // Extract title and URL from the result link. + // Pattern: TITLE + urlPattern := regexp.MustCompile(`]+href="(/url\?q=[^"&]+)`) + urlMatch := urlPattern.FindStringSubmatch(block) + if len(urlMatch) < 2 { + continue + } + rawURL := urlMatch[1] + // Remove /url?q= prefix and decode. + actualURL := strings.TrimPrefix(rawURL, "/url?q=") + if amp := strings.Index(actualURL, "&"); amp != -1 { + actualURL = actualURL[:amp] + } + if decoded, err := url.QueryUnescape(actualURL); err == nil { + actualURL = decoded + } + + if actualURL == "" || !strings.HasPrefix(actualURL, "http") { + continue + } + + // Extract title from the title tag. + titlePattern := regexp.MustCompile(`]*class="[^"]*qrStP[^"]*"[^>]*>([^<]+)`) + titleMatch := titlePattern.FindStringSubmatch(block) + title := query + if len(titleMatch) >= 2 { + title = stripTags(titleMatch[1]) + } else { + // Fallback: extract visible text from an with data-title or role="link" + linkTitlePattern := regexp.MustCompile(`]+role="link"[^>]*>([^<]+)<`) + ltMatch := linkTitlePattern.FindStringSubmatch(block) + if len(ltMatch) >= 2 { + title = stripTags(ltMatch[1]) + } + } + + // Extract snippet from data-sncf divs (SearXNG's approach). + snippet := extractGoogleSnippet(block) + + urlPtr := actualURL + results = append(results, contracts.MainResult{ + Title: title, + URL: &urlPtr, + Content: snippet, + Engine: "google", + Score: float64(len(matches) - i), + Category: "general", + Engines: []string{"google"}, + Template: "default.html", + }) + } + + return results +} + +// extractGoogleSnippet extracts the snippet text from a Google result block. +func extractGoogleSnippet(block string) string { + // Google's snippets live in divs with data-sncf attribute. + // SearXNG looks for: .//div[contains(@data-sncf, "1")] + snippetPattern := regexp.MustCompile(`]+data-sncf="1"[^>]*>(.*?)`) + matches := snippetPattern.FindAllStringSubmatch(block, -1) + var parts []string + for _, m := range matches { + if len(m) < 2 { + continue + } + text := stripTags(m[1]) + if text != "" { + parts = append(parts, text) + } + } + return strings.Join(parts, " ") +} + +// extractGoogleSuggestions extracts search suggestions from Google result cards. +func extractGoogleSuggestions(body string) []string { + var suggestions []string + // SearXNG xpath: //div[contains(@class, "ouy7Mc")]//a + suggestionPattern := regexp.MustCompile(`]*class="[^"]*ouy7Mc[^"]*"[^>]*>.*?]*>([^<]+)`, regexp.DotAll) + matches := suggestionPattern.FindAllStringSubmatch(body, -1) + seen := map[string]bool{} + for _, m := range matches { + if len(m) < 2 { + continue + } + s := strings.TrimSpace(stripTags(m[1])) + if s != "" && !seen[s] { + seen[s] = true + suggestions = append(suggestions, s) + } + } + return suggestions +} + +// googleHL maps SearXNG locale to Google hl (host language) parameter. +// e.g. "en-US" -> "en-US" +func googleHL(lang string) string { + lang = strings.ToLower(strings.TrimSpace(lang)) + if lang == "" || lang == "auto" { + return "en" + } + return lang +} + +// googleUILanguage maps SearXNG language to Google lr (language restrict) parameter. +// e.g. "en" -> "lang_en", "de" -> "lang_de" +func googleUILanguage(lang string) string { + lang = strings.ToLower(strings.Split(lang, "-")[0]) + if lang == "" || lang == "auto" { + return "" + } + return "lang_" + lang +} + +// googleSafeSearchLevel maps safesearch (0-2) to Google's safe parameter. +func googleSafeSearchLevel(safesearch int) string { + switch safesearch { + case 0: + return "off" + case 1: + return "medium" + case 2: + return "high" + default: + return "medium" + } +} + +// stripTags removes HTML tags from a string. +func stripTags(s string) string { + stripper := regexp.MustCompile(`<[^>]*>`) + s = stripper.ReplaceAllString(s, "") + s = strings.ReplaceAll(s, "&", "&") + s = strings.ReplaceAll(s, """, `"`) + s = strings.ReplaceAll(s, "'", "'") + s = strings.ReplaceAll(s, " ", " ") + return strings.TrimSpace(s) +} diff --git a/internal/engines/planner.go b/internal/engines/planner.go index 543f253..08b0a27 100644 --- a/internal/engines/planner.go +++ b/internal/engines/planner.go @@ -91,6 +91,7 @@ func inferFromCategories(categories []string) []string { set["qwant"] = true set["duckduckgo"] = true set["bing"] = true + set["google"] = true case "science", "scientific publications": set["arxiv"] = true set["crossref"] = true @@ -106,7 +107,7 @@ func inferFromCategories(categories []string) []string { out = append(out, e) } // stable order - order := map[string]int{"wikipedia": 0, "braveapi": 1, "qwant": 2, "duckduckgo": 3, "bing": 4, "arxiv": 5, "crossref": 6, "github": 7, "reddit": 8} + order := map[string]int{"wikipedia": 0, "braveapi": 1, "qwant": 2, "duckduckgo": 3, "bing": 4, "google": 5, "arxiv": 6, "crossref": 7, "github": 8, "reddit": 9} sortByOrder(out, order) return out } From 1689cab9bdc0f331a71ba947871db4272b016e01 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 01:53:19 +0000 Subject: [PATCH 05/27] feat: add YouTube engine via Data API v3 Uses the official YouTube Data API v3. Requires YOUTUBE_API_KEY environment variable (free from Google Cloud Console). Returns video results with title, description, channel, publish date, and thumbnail URL. Falls back gracefully if no API key. --- internal/engines/factory.go | 6 +- internal/engines/planner.go | 6 +- internal/engines/youtube.go | 182 ++++++++++++++++++++++++++++++++++++ 3 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 internal/engines/youtube.go diff --git a/internal/engines/factory.go b/internal/engines/factory.go index 937225f..53ba87f 100644 --- a/internal/engines/factory.go +++ b/internal/engines/factory.go @@ -32,6 +32,10 @@ func NewDefaultPortedEngines(client *http.Client) map[string]Engine { "github": &GitHubEngine{client: client}, "reddit": &RedditEngine{client: client}, "bing": &BingEngine{client: client}, - "google": &GoogleEngine{client: client}, + "google": &GoogleEngine{client: client}, + "youtube": &YouTubeEngine{ + client: client, + baseURL: "https://www.googleapis.com", + }, } } diff --git a/internal/engines/planner.go b/internal/engines/planner.go index 24af031..b180f7e 100644 --- a/internal/engines/planner.go +++ b/internal/engines/planner.go @@ -7,7 +7,7 @@ import ( "github.com/metamorphosis-dev/kafka/internal/contracts" ) -var defaultPortedEngines = []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"} +var defaultPortedEngines = []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing", "google", "youtube"} type Planner struct { PortedSet map[string]bool @@ -99,6 +99,8 @@ func inferFromCategories(categories []string) []string { set["github"] = true case "social media": set["reddit"] = true + case "videos": + set["youtube"] = true } } @@ -107,7 +109,7 @@ func inferFromCategories(categories []string) []string { out = append(out, e) } // stable order - order := map[string]int{"wikipedia": 0, "braveapi": 1, "qwant": 2, "duckduckgo": 3, "bing": 4, "google": 5, "arxiv": 6, "crossref": 7, "github": 8, "reddit": 9} + order := map[string]int{"wikipedia": 0, "braveapi": 1, "qwant": 2, "duckduckgo": 3, "bing": 4, "google": 5, "arxiv": 6, "crossref": 7, "github": 8, "reddit": 9, "youtube": 10} sortByOrder(out, order) return out } diff --git a/internal/engines/youtube.go b/internal/engines/youtube.go new file mode 100644 index 0000000..7580a09 --- /dev/null +++ b/internal/engines/youtube.go @@ -0,0 +1,182 @@ +package engines + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/metamorphosis-dev/kafka/internal/contracts" +) + +type YouTubeEngine struct { + client *http.Client + apiKey string + baseURL string +} + +func (e *YouTubeEngine) Name() string { return "youtube" } + +func (e *YouTubeEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) { + if strings.TrimSpace(req.Query) == "" { + return contracts.SearchResponse{Query: req.Query}, nil + } + + if e.apiKey == "" { + e.apiKey = os.Getenv("YOUTUBE_API_KEY") + } + + maxResults := 10 + if req.Pageno > 1 { + maxResults = 20 + } + + u := e.baseURL + "/youtube/v3/search?" + url.Values{ + "part": {"snippet"}, + "q": {req.Query}, + "type": {"video"}, + "maxResults": {fmt.Sprintf("%d", maxResults)}, + "key": {e.apiKey}, + }.Encode() + + if req.Language != "" && req.Language != "auto" { + lang := strings.Split(strings.ToLower(req.Language), "-")[0] + u += "&relevanceLanguage=" + lang + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return contracts.SearchResponse{}, err + } + + resp, err := e.client.Do(httpReq) + if err != nil { + return contracts.SearchResponse{}, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return contracts.SearchResponse{}, fmt.Errorf("youtube api error: status=%d body=%q", resp.StatusCode, string(body)) + } + + var apiResp youtubeSearchResponse + if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil { + return contracts.SearchResponse{}, err + } + + if apiResp.Error != nil { + return contracts.SearchResponse{}, fmt.Errorf("youtube api error: %s", apiResp.Error.Message) + } + + results := make([]contracts.MainResult, 0, len(apiResp.Items)) + for _, item := range apiResp.Items { + if item.ID.VideoID == "" { + continue + } + + videoURL := "https://www.youtube.com/watch?v=" + item.ID.VideoID + urlPtr := videoURL + + published := "" + if item.Snippet.PublishedAt != "" { + if t, err := time.Parse(time.RFC3339, item.Snippet.PublishedAt); err == nil { + published = t.Format("Jan 2, 2006") + } + } + + content := item.Snippet.Description + if len(content) > 300 { + content = content[:300] + "..." + } + if published != "" { + content = "Published " + published + " · " + content + } + + thumbnail := "" + if item.Snippet.Thumbnails.High.URL != "" { + thumbnail = item.Snippet.Thumbnails.High.URL + } else if item.Snippet.Thumbnails.Medium.URL != "" { + thumbnail = item.Snippet.Thumbnails.Medium.URL + } + + results = append(results, contracts.MainResult{ + Template: "videos.html", + Title: item.Snippet.Title, + URL: &urlPtr, + Content: content, + Thumbnail: thumbnail, + Engine: "youtube", + Score: 1.0, + Category: "videos", + Engines: []string{"youtube"}, + Metadata: map[string]any{ + "channel": item.Snippet.ChannelTitle, + "video_id": item.Snippet.ResourceID.VideoID, + }, + }) + } + + return contracts.SearchResponse{ + Query: req.Query, + NumberOfResults: len(results), + Results: results, + Answers: []map[string]any{}, + Corrections: []string{}, + Infoboxes: []map[string]any{}, + Suggestions: []string{}, + UnresponsiveEngines: [][2]string{}, + }, nil +} + +// YouTube API response types. + +type youtubeSearchResponse struct { + Items []youtubeSearchItem `json:"items"` + PageInfo struct { + TotalResults int `json:"totalResults"` + ResultsPerPage int `json:"resultsPerPage"` + } `json:"pageInfo"` + NextPageToken string `json:"nextPageToken"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + Errors []struct { + Domain string `json:"domain"` + Reason string `json:"reason"` + Message string `json:"message"` + } `json:"errors"` + } `json:"error"` +} + +type youtubeSearchItem struct { + ID struct { + VideoID string `json:"videoId"` + } `json:"id"` + Snippet struct { + PublishedAt string `json:"publishedAt"` + ChannelID string `json:"channelId"` + ChannelTitle string `json:"channelTitle"` + Title string `json:"title"` + Description string `json:"description"` + Thumbnails struct { + Default struct { + URL string `json:"url"` + } `json:"default"` + Medium struct { + URL string `json:"url"` + } `json:"medium"` + High struct { + URL string `json:"url"` + } `json:"high"` + } `json:"thumbnails"` + ResourceID struct { + VideoID string `json:"videoId"` + } `json:"resourceId"` + } `json:"snippet"` +} From a7f594b7fa68d94d4debdace3fbae867b25e4f60 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 01:57:13 +0000 Subject: [PATCH 06/27] feat: add YouTube engine with config file and env support YouTube Data API v3 engine: - Add YouTubeConfig to EnginesConfig with api_key field - Add YOUTUBE_API_KEY env override - Thread *config.Config through search service to factory - Factory falls back to env vars if config fields are empty - Update config.example.toml with youtube section Also update default local_ported to include google and youtube. --- cmd/kafka/main.go | 7 ++++--- config.example.toml | 6 +++++- internal/config/config.go | 10 +++++++++- internal/engines/factory.go | 26 +++++++++++++++++++++++--- internal/search/service.go | 10 ++++++---- 5 files changed, 47 insertions(+), 12 deletions(-) diff --git a/cmd/kafka/main.go b/cmd/kafka/main.go index ab29852..90c750d 100644 --- a/cmd/kafka/main.go +++ b/cmd/kafka/main.go @@ -53,9 +53,10 @@ func main() { } svc := search.NewService(search.ServiceConfig{ - UpstreamURL: cfg.Upstream.URL, - HTTPTimeout: cfg.HTTPTimeout(), - Cache: searchCache, + UpstreamURL: cfg.Upstream.URL, + HTTPTimeout: cfg.HTTPTimeout(), + Cache: searchCache, + EnginesConfig: cfg, }) acSvc := autocomplete.NewService(cfg.Upstream.URL, cfg.HTTPTimeout()) diff --git a/config.example.toml b/config.example.toml index 1e3b75c..34f60a6 100644 --- a/config.example.toml +++ b/config.example.toml @@ -22,7 +22,7 @@ url = "" [engines] # Comma-separated list of engines to execute locally in Go (env: LOCAL_PORTED_ENGINES) # Engines not listed here will be proxied to the upstream instance. -local_ported = ["wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"] +local_ported = ["wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing", "google", "youtube"] [engines.brave] # Brave Search API key (env: BRAVE_API_KEY) @@ -35,6 +35,10 @@ access_token = "" category = "web-lite" results_per_page = 10 +[engines.youtube] +# YouTube Data API v3 key (env: YOUTUBE_API_KEY) +api_key = "" + [cache] # Valkey/Redis cache for search results. # Leave address empty to disable caching entirely. diff --git a/internal/config/config.go b/internal/config/config.go index 93b8d86..7f8b06a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -35,6 +35,7 @@ type EnginesConfig struct { LocalPorted []string `toml:"local_ported"` Brave BraveConfig `toml:"brave"` Qwant QwantConfig `toml:"qwant"` + YouTube YouTubeConfig `toml:"youtube"` } // CacheConfig holds Valkey/Redis cache settings. @@ -85,6 +86,10 @@ type QwantConfig struct { ResultsPerPage int `toml:"results_per_page"` } +type YouTubeConfig struct { + APIKey string `toml:"api_key"` +} + // Load reads configuration from the given TOML file path. // If the file does not exist, it returns defaults (empty values where applicable). // Environment variables are used as fallbacks for any zero-value fields. @@ -109,7 +114,7 @@ func defaultConfig() *Config { }, Upstream: UpstreamConfig{}, Engines: EnginesConfig{ - LocalPorted: []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"}, + LocalPorted: []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing", "google", "youtube"}, Qwant: QwantConfig{ Category: "web-lite", ResultsPerPage: 10, @@ -151,6 +156,9 @@ func applyEnvOverrides(cfg *Config) { if v := os.Getenv("BRAVE_ACCESS_TOKEN"); v != "" { cfg.Engines.Brave.AccessToken = v } + if v := os.Getenv("YOUTUBE_API_KEY"); v != "" { + cfg.Engines.YouTube.APIKey = v + } if v := os.Getenv("VALKEY_ADDRESS"); v != "" { cfg.Cache.Address = v } diff --git a/internal/engines/factory.go b/internal/engines/factory.go index 53ba87f..b7f3c00 100644 --- a/internal/engines/factory.go +++ b/internal/engines/factory.go @@ -4,23 +4,42 @@ import ( "net/http" "os" "time" + + "github.com/metamorphosis-dev/kafka/internal/config" ) // NewDefaultPortedEngines returns the starter set of Go-native engines. // The service can swap/extend this registry later as more engines are ported. -func NewDefaultPortedEngines(client *http.Client) map[string]Engine { +// If cfg is nil, falls back to reading API keys from environment variables. +func NewDefaultPortedEngines(client *http.Client, cfg *config.Config) map[string]Engine { if client == nil { client = &http.Client{Timeout: 10 * time.Second} } + var braveAPIKey, braveAccessToken, youtubeAPIKey string + if cfg != nil { + braveAPIKey = cfg.Engines.Brave.APIKey + braveAccessToken = cfg.Engines.Brave.AccessToken + youtubeAPIKey = cfg.Engines.YouTube.APIKey + } + if braveAPIKey == "" { + braveAPIKey = os.Getenv("BRAVE_API_KEY") + } + if braveAccessToken == "" { + braveAccessToken = os.Getenv("BRAVE_ACCESS_TOKEN") + } + if youtubeAPIKey == "" { + youtubeAPIKey = os.Getenv("YOUTUBE_API_KEY") + } + return map[string]Engine{ "wikipedia": &WikipediaEngine{client: client}, "arxiv": &ArxivEngine{client: client}, "crossref": &CrossrefEngine{client: client}, "braveapi": &BraveEngine{ client: client, - apiKey: os.Getenv("BRAVE_API_KEY"), - accessGateToken: os.Getenv("BRAVE_ACCESS_TOKEN"), + apiKey: braveAPIKey, + accessGateToken: braveAccessToken, resultsPerPage: 20, }, "qwant": &QwantEngine{ @@ -35,6 +54,7 @@ func NewDefaultPortedEngines(client *http.Client) map[string]Engine { "google": &GoogleEngine{client: client}, "youtube": &YouTubeEngine{ client: client, + apiKey: youtubeAPIKey, baseURL: "https://www.googleapis.com", }, } diff --git a/internal/search/service.go b/internal/search/service.go index 62a9308..47d2895 100644 --- a/internal/search/service.go +++ b/internal/search/service.go @@ -7,15 +7,17 @@ import ( "time" "github.com/metamorphosis-dev/kafka/internal/cache" + "github.com/metamorphosis-dev/kafka/internal/config" "github.com/metamorphosis-dev/kafka/internal/contracts" "github.com/metamorphosis-dev/kafka/internal/engines" "github.com/metamorphosis-dev/kafka/internal/upstream" ) type ServiceConfig struct { - UpstreamURL string - HTTPTimeout time.Duration - Cache *cache.Cache + UpstreamURL string + HTTPTimeout time.Duration + Cache *cache.Cache + EnginesConfig *config.Config } type Service struct { @@ -44,7 +46,7 @@ func NewService(cfg ServiceConfig) *Service { return &Service{ upstreamClient: up, planner: engines.NewPlannerFromEnv(), - localEngines: engines.NewDefaultPortedEngines(httpClient), + localEngines: engines.NewDefaultPortedEngines(httpClient, cfg.EnginesConfig), cache: cfg.Cache, } } From bfed53ae332ac68046a3942c157d7049be5428dd Mon Sep 17 00:00:00 2001 From: ashisgreat22 Date: Sun, 22 Mar 2026 02:20:13 +0100 Subject: [PATCH 07/27] docs: add settings UI design spec Co-Authored-By: Claude Opus 4.6 --- .../specs/2026-03-22-settings-ui-design.md | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-22-settings-ui-design.md diff --git a/docs/superpowers/specs/2026-03-22-settings-ui-design.md b/docs/superpowers/specs/2026-03-22-settings-ui-design.md new file mode 100644 index 0000000..8029dc8 --- /dev/null +++ b/docs/superpowers/specs/2026-03-22-settings-ui-design.md @@ -0,0 +1,80 @@ +# Settings UI Design — kafka + +**Date:** 2026-03-22 +**Status:** Approved + +## Overview + +A lightweight preferences popover anchored to the top-right, just below the header. Triggered by a gear icon, it lets users adjust theme, enabled engines, and search defaults without leaving their current page. All changes auto-save to `localStorage` on every interaction. + +## Layout & Structure + +- **Trigger**: Gear icon (⚙️) in the top-right header, aligned with the header's right edge +- **Panel**: 280px wide, max-height 420px, scrollable internally, rounded corners, subtle shadow, anchored top-right (drops down from trigger, like a dropdown) +- **Close**: × button in panel header, click outside the panel, or pressing Escape +- **No Save button** — every interaction immediately writes to `localStorage` + +## Interaction Flow + +1. User clicks ⚙️ → panel drops down from top-right (200ms ease) +2. User toggles/clicks → changes apply instantly to DOM + write to `localStorage` +3. User clicks × or outside or Escape → panel closes, settings persist +4. **Accessibility**: Focus is trapped within the panel while open. Trigger button uses `aria-expanded` and `aria-controls`. Escape key closes the panel. + +## Mid-Search Changes + +When opened during an active search on `results.html`: +- Engine toggles update `localStorage` immediately, but **current results remain unchanged** +- A subtle inline note below the engines section: *"Engine changes apply to your next search"* + +## Sections + +### Appearance + +- Three theme buttons: ☀️ Light / 🌙 Dark / 💻 System +- Clicking immediately applies via `document.body.classList` + writes to localStorage +- "System" reads `prefers-color-scheme` and updates on change + +### Engines + +- 2-column grid of toggle switches for all 9 engines +- Each row: engine name + toggle switch +- Enabled = filled accent color; Disabled = gray outline + +### Search Defaults + +- Safe search: dropdown (Moderate / Strict / Off) +- Default format: dropdown (HTML / JSON / CSV) + +## Default State + +```js +const DEFAULT_PREFS = { + theme: "system", + engines: ["wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"], + safeSearch: "moderate", + format: "html" +}; +``` + +## Persistence + +```js +// Written on every interaction +localStorage.setItem('kafka_prefs', JSON.stringify({ ... })); + +// Read on page load — merge with DEFAULT_PREFS +const saved = JSON.parse(localStorage.getItem('kafka_prefs') || '{}'); +const prefs = { ...DEFAULT_PREFS, ...saved }; +``` + +## Responsive Behavior + +- **Mobile (<768px)**: Panel becomes a **bottom sheet** — 100% width, slides up from the bottom, top corners rounded, max-height 70vh. Trigger moves to a fixed bottom-right FAB button. +- Panel never covers the search input + +## Applied to Existing Code + +- `base.html` — add gear button in header, panel markup at end of `` +- `kafka.css` — popover styles, toggle switch styles, bottom sheet styles for mobile +- `settings.js` — localStorage read/write, theme application, panel toggle, aria attributes, focus trap From 2c9d1c35432a9e769df865cc733b9e6bd9f9db93 Mon Sep 17 00:00:00 2001 From: ashisgreat22 Date: Sun, 22 Mar 2026 02:24:54 +0100 Subject: [PATCH 08/27] docs: add settings UI implementation plan Co-Authored-By: Claude Opus 4.6 --- .../plans/2026-03-22-settings-ui.md | 712 ++++++++++++++++++ 1 file changed, 712 insertions(+) create mode 100644 docs/superpowers/plans/2026-03-22-settings-ui.md diff --git a/docs/superpowers/plans/2026-03-22-settings-ui.md b/docs/superpowers/plans/2026-03-22-settings-ui.md new file mode 100644 index 0000000..9ebcffd --- /dev/null +++ b/docs/superpowers/plans/2026-03-22-settings-ui.md @@ -0,0 +1,712 @@ +# Settings UI Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A preferences popover panel (top-right on desktop, bottom sheet on mobile) that lets users set theme, enabled engines, safe search, and default format. All changes auto-save to `localStorage` and apply immediately to the DOM. + +**Architecture:** Pure client-side JS + CSS added alongside existing templates. No Go changes. Settings persist via `localStorage` key `kafka_prefs`. Theme applies via `data-theme` attribute on ``. + +**Tech Stack:** Vanilla JS (no framework), existing `kafka.css` custom properties, HTMX for search. + +--- + +## File Map + +| Action | File | +|--------|------| +| Create | `internal/views/static/js/settings.js` | +| Modify | `internal/views/static/css/kafka.css` | +| Modify | `internal/views/templates/base.html` | +| Modify | `internal/views/templates/index.html` | +| Modify | `internal/views/templates/results.html` | +| Modify | `internal/views/views.go` | + +**Key insight on engine preferences:** `ParseSearchRequest` reads `engines` as a CSV form value (`r.FormValue("engines")`). The search forms in `index.html` and `results.html` will get a hidden `#engines-input` field that is kept in sync with localStorage. On submit, the engines preference is sent as a normal form field. HTMX `hx-include="this"` already includes the form element, so the hidden input is automatically included in the request. + +--- + +## Task 1: CSS — Popover, toggles, bottom sheet + +**Files:** +- Modify: `internal/views/static/css/kafka.css` + +- [ ] **Step 1: Add CSS for popover, triggers, toggles, bottom sheet** + +Append the following to `kafka.css`: + +```css +/* ============================================ + Settings Panel + ============================================ */ + +/* Header */ +.site-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 0.6rem 1rem; + background: var(--color-header-background); + border-bottom: 1px solid var(--color-header-border); +} +.site-title { + font-size: 1rem; + font-weight: 600; + color: var(--color-base-font); +} + +/* Gear trigger button */ +.settings-trigger { + background: none; + border: none; + font-size: 1.1rem; + cursor: pointer; + padding: 0.3rem 0.5rem; + border-radius: var(--radius); + color: var(--color-base-font); + opacity: 0.7; + transition: opacity 0.2s, background 0.2s; + line-height: 1; +} +.settings-trigger:hover, +.settings-trigger[aria-expanded="true"] { + opacity: 1; + background: var(--color-sidebar-background); +} + +/* Popover panel */ +.settings-popover { + position: absolute; + top: 100%; + right: 0; + width: 280px; + max-height: 420px; + overflow-y: auto; + background: var(--color-base-background); + border: 1px solid var(--color-sidebar-border); + border-radius: var(--radius); + box-shadow: 0 8px 24px rgba(0, 0, 0, 0.12); + z-index: 200; + display: none; + flex-direction: column; +} +.settings-popover[data-open="true"] { + display: flex; + animation: settings-slide-in 0.2s ease; +} +@keyframes settings-slide-in { + from { opacity: 0; transform: translateY(-8px); } + to { opacity: 1; transform: translateY(0); } +} + +.settings-popover-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1rem; + border-bottom: 1px solid var(--color-sidebar-border); + font-weight: 600; + font-size: 0.9rem; + flex-shrink: 0; +} +.settings-popover-close { + background: none; + border: none; + font-size: 1.2rem; + cursor: pointer; + color: var(--color-base-font); + opacity: 0.6; + padding: 0 0.25rem; + line-height: 1; +} +.settings-popover-close:hover { opacity: 1; } + +.settings-popover-body { + padding: 0.8rem; + display: flex; + flex-direction: column; + gap: 1rem; +} + +.settings-section-title { + font-size: 0.7rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--color-suggestion); + margin-bottom: 0.5rem; +} + +/* Theme buttons */ +.theme-buttons { + display: flex; + gap: 0.4rem; +} +.theme-btn { + flex: 1; + padding: 0.35rem 0.5rem; + border: 1px solid var(--color-sidebar-border); + border-radius: var(--radius); + background: var(--color-btn-background); + color: var(--color-base-font); + cursor: pointer; + font-size: 0.75rem; + text-align: center; + transition: background 0.15s, border-color 0.15s; +} +.theme-btn:hover { background: var(--color-btn-hover); } +.theme-btn.active { + background: var(--color-link); + color: #fff; + border-color: var(--color-link); +} + +/* Engine toggles — 2-column grid */ +.engine-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 0.4rem; +} +.engine-toggle { + display: flex; + align-items: center; + gap: 0.4rem; + padding: 0.3rem 0.5rem; + border-radius: var(--radius); + background: var(--color-sidebar-background); + font-size: 0.78rem; + cursor: pointer; +} +.engine-toggle input[type="checkbox"] { + width: 15px; + height: 15px; + margin: 0; + cursor: pointer; + accent-color: var(--color-link); +} +.engine-toggle span { + flex: 1; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +/* Search defaults */ +.setting-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.5rem; + margin-top: 0.4rem; +} +.setting-row label { + font-size: 0.85rem; + flex: 1; +} +.setting-row select { + width: 110px; + padding: 0.3rem 0.4rem; + font-size: 0.8rem; + border: 1px solid var(--color-sidebar-border); + border-radius: var(--radius); + background: var(--color-base-background); + color: var(--color-base-font); + cursor: pointer; +} + +/* Mid-search notice */ +.settings-notice { + font-size: 0.72rem; + color: var(--color-suggestion); + margin-top: 0.3rem; + font-style: italic; +} + +/* Dark theme via data-theme attribute */ +html[data-theme="dark"] { + --color-base: #222; + --color-base-font: #dcdcdc; + --color-base-background: #2b2b2b; + --color-header-background: #333; + --color-header-border: #444; + --color-search-border: #555; + --color-search-focus: #5dade2; + --color-result-url: #8ab4f8; + --color-result-url-visited: #b39ddb; + --color-result-content: #b0b0b0; + --color-result-title: #8ab4f8; + --color-result-title-visited: #b39ddb; + --color-result-engine: #999; + --color-result-border: #3a3a3a; + --color-link: #5dade2; + --color-link-visited: #b39ddb; + --color-sidebar-background: #333; + --color-sidebar-border: #444; + --color-infobox-background: #333; + --color-infobox-border: #444; + --color-pagination-current: #5dade2; + --color-pagination-border: #444; + --color-error: #e74c3c; + --color-error-background: #3b1a1a; + --color-suggestion: #999; + --color-footer: #666; + --color-btn-background: #333; + --color-btn-border: #555; + --color-btn-hover: #444; +} + +/* Mobile: Bottom sheet */ +@media (max-width: 768px) { + .settings-popover { + position: fixed; + top: auto; + bottom: 0; + left: 0; + right: 0; + width: 100%; + max-height: 70vh; + border-radius: var(--radius) var(--radius) 0 0; + border-bottom: none; + } +} +``` + +Note: The existing `:root` and `@media (prefers-color-scheme: dark)` blocks provide the "system" theme. `html[data-theme="dark"]` overrides only apply when the user explicitly picks dark mode. When `theme === 'system'`, the `data-theme` attribute is removed and the browser's `prefers-color-scheme` media query kicks in via the existing CSS. + +- [ ] **Step 2: Verify existing tests still pass** + +Run: `go test ./...` +Expected: all pass + +- [ ] **Step 3: Commit** + +```bash +git add internal/views/static/css/kafka.css +git commit -m "feat(settings): add popover, toggle, and bottom-sheet CSS" +``` + +--- + +## Task 2: JS — Settings logic + +**Files:** +- Create: `internal/views/static/js/settings.js` + +- [ ] **Step 1: Write the settings JS module** + +Create `internal/views/static/js/settings.js`: + +```javascript +'use strict'; + +var ALL_ENGINES = [ + 'wikipedia', 'arxiv', 'crossref', 'braveapi', + 'qwant', 'duckduckgo', 'github', 'reddit', 'bing' +]; + +var DEFAULT_PREFS = { + theme: 'system', + engines: ALL_ENGINES.slice(), + safeSearch: 'moderate', + format: 'html' +}; + +var STORAGE_KEY = 'kafka_prefs'; + +// ── Persistence ────────────────────────────────────────────────────────────── + +function loadPrefs() { + try { + var raw = localStorage.getItem(STORAGE_KEY); + if (!raw) return { theme: DEFAULT_PREFS.theme, engines: DEFAULT_PREFS.engines.slice(), safeSearch: DEFAULT_PREFS.safeSearch, format: DEFAULT_PREFS.format }; + var saved = JSON.parse(raw); + return { theme: saved.theme || DEFAULT_PREFS.theme, engines: saved.engines || DEFAULT_PREFS.engines.slice(), safeSearch: saved.safeSearch || DEFAULT_PREFS.safeSearch, format: saved.format || DEFAULT_PREFS.format }; + } catch (e) { + return { theme: DEFAULT_PREFS.theme, engines: DEFAULT_PREFS.engines.slice(), safeSearch: DEFAULT_PREFS.safeSearch, format: DEFAULT_PREFS.format }; + } +} + +function savePrefs(prefs) { + try { + localStorage.setItem(STORAGE_KEY, JSON.stringify({ theme: prefs.theme, engines: prefs.engines, safeSearch: prefs.safeSearch, format: prefs.format })); + } catch (e) { /* quota or private mode */ } +} + +// ── Theme application ──────────────────────────────────────────────────────── + +function applyTheme(theme) { + if (theme === 'system') { + document.documentElement.removeAttribute('data-theme'); + } else { + document.documentElement.setAttribute('data-theme', theme); + } +} + +// ── Engine input sync ───────────────────────────────────────────────────────── + +function syncEngineInput(prefs) { + var input = document.getElementById('engines-input'); + if (input) input.value = prefs.engines.join(','); +} + +// ── Panel open / close ──────────────────────────────────────────────────────── + +function closePanel() { + var panel = document.getElementById('settings-popover'); + var trigger = document.getElementById('settings-trigger'); + if (!panel) return; + panel.setAttribute('data-open', 'false'); + if (trigger) trigger.setAttribute('aria-expanded', 'false'); + if (trigger) trigger.focus(); +} + +function openPanel() { + var panel = document.getElementById('settings-popover'); + var trigger = document.getElementById('settings-trigger'); + if (!panel) return; + panel.setAttribute('data-open', 'true'); + if (trigger) trigger.setAttribute('aria-expanded', 'true'); + var focusable = panel.querySelector('button, input, select'); + if (focusable) focusable.focus(); +} + +// ── Escape key ─────────────────────────────────────────────────────────────── + +document.addEventListener('keydown', function(e) { + if (e.key !== 'Escape') return; + var panel = document.getElementById('settings-popover'); + if (!panel || panel.getAttribute('data-open') !== 'true') return; + closePanel(); +}); + +// ── Click outside ───────────────────────────────────────────────────────────── + +document.addEventListener('click', function(e) { + var panel = document.getElementById('settings-popover'); + var trigger = document.getElementById('settings-trigger'); + if (!panel || panel.getAttribute('data-open') !== 'true') return; + if (!panel.contains(e.target) && (!trigger || !trigger.contains(e.target))) { + closePanel(); + } +}); + +// ── Focus trap ──────────────────────────────────────────────────────────────── + +document.addEventListener('keydown', function(e) { + if (e.key !== 'Tab') return; + var panel = document.getElementById('settings-popover'); + if (!panel || panel.getAttribute('data-open') !== 'true') return; + var focusable = Array.prototype.slice.call(panel.querySelectorAll('button, input, select, [tabindex]:not([tabindex="-1"])')); + if (!focusable.length) return; + var first = focusable[0]; + var last = focusable[focusable.length - 1]; + if (e.shiftKey) { + if (document.activeElement === first) { e.preventDefault(); last.focus(); } + } else { + if (document.activeElement === last) { e.preventDefault(); first.focus(); } + } +}); + +// ── Render ──────────────────────────────────────────────────────────────────── + +function escapeHtml(str) { + return String(str).replace(/&/g, '&').replace(//g, '>'); +} + +function renderPanel(prefs) { + var panel = document.getElementById('settings-popover'); + if (!panel) return; + var body = panel.querySelector('.settings-popover-body'); + if (!body) return; + + var themeBtns = ''; + ['light', 'dark', 'system'].forEach(function(t) { + var icons = { light: '\u2600', dark: '\u263D', system: '\u2318' }; + var labels = { light: 'Light', dark: 'Dark', system: 'System' }; + var active = prefs.theme === t ? ' active' : ''; + themeBtns += ''; + }); + + var engineToggles = ''; + ALL_ENGINES.forEach(function(name) { + var checked = prefs.engines.indexOf(name) !== -1 ? ' checked' : ''; + engineToggles += ''; + }); + + var ssOptions = [ + { val: 'moderate', label: 'Moderate' }, + { val: 'strict', label: 'Strict' }, + { val: 'off', label: 'Off' } + ]; + var fmtOptions = [ + { val: 'html', label: 'HTML' }, + { val: 'json', label: 'JSON' }, + { val: 'csv', label: 'CSV' } + ]; + var ssOptionsHtml = ''; + var fmtOptionsHtml = ''; + ssOptions.forEach(function(o) { + var sel = prefs.safeSearch === o.val ? ' selected' : ''; + ssOptionsHtml += ''; + }); + fmtOptions.forEach(function(o) { + var sel = prefs.format === o.val ? ' selected' : ''; + fmtOptionsHtml += ''; + }); + + body.innerHTML = + '
' + + '
Appearance
' + + '
' + themeBtns + '
' + + '
' + + '
' + + '
Engines
' + + '
' + engineToggles + '
' + + '

Engine changes apply to your next search.

' + + '
' + + '
' + + '
Search Defaults
' + + '
' + + '' + + '' + + '
' + + '
' + + '' + + '' + + '
' + + '
'; + + // Theme buttons + var themeBtnEls = panel.querySelectorAll('.theme-btn'); + for (var i = 0; i < themeBtnEls.length; i++) { + themeBtnEls[i].addEventListener('click', (function(btn) { + return function() { + prefs.theme = btn.getAttribute('data-theme'); + savePrefs(prefs); + applyTheme(prefs.theme); + syncEngineInput(prefs); + renderPanel(prefs); + }; + })(themeBtnEls[i])); + } + + // Engine checkboxes + var checkboxes = panel.querySelectorAll('.engine-toggle input[type="checkbox"]'); + for (var j = 0; j < checkboxes.length; j++) { + checkboxes[j].addEventListener('change', (function(cb) { + return function() { + var checked = Array.prototype.slice.call(panel.querySelectorAll('.engine-toggle input[type="checkbox"]:checked')).map(function(el) { return el.value; }); + if (checked.length === 0) { cb.checked = true; return; } + prefs.engines = checked; + savePrefs(prefs); + syncEngineInput(prefs); + }; + })(checkboxes[j])); + } + + // Safe search + var ssEl = panel.querySelector('#pref-safesearch'); + if (ssEl) { + ssEl.addEventListener('change', function() { + prefs.safeSearch = ssEl.value; + savePrefs(prefs); + }); + } + + // Format + var fmtEl = panel.querySelector('#pref-format'); + if (fmtEl) { + fmtEl.addEventListener('change', function() { + prefs.format = fmtEl.value; + savePrefs(prefs); + }); + } + + // Close button + var closeBtn = panel.querySelector('.settings-popover-close'); + if (closeBtn) closeBtn.addEventListener('click', closePanel); +} + +// ── Init ───────────────────────────────────────────────────────────────────── + +function initSettings() { + var prefs = loadPrefs(); + applyTheme(prefs.theme); + syncEngineInput(prefs); + + var panel = document.getElementById('settings-popover'); + var trigger = document.getElementById('settings-trigger'); + + if (panel && trigger) { + renderPanel(prefs); + trigger.addEventListener('click', function() { + var isOpen = panel.getAttribute('data-open') === 'true'; + if (isOpen) closePanel(); else openPanel(); + }); + } +} + +if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', initSettings); +} else { + initSettings(); +} +``` + +- [ ] **Step 2: Verify JS syntax** + +Run: `node --check internal/views/static/js/settings.js` +Expected: no output (exit 0) + +- [ ] **Step 3: Commit** + +```bash +git add internal/views/static/js/settings.js +git commit -m "feat(settings): add JS module for localStorage preferences and panel" +``` + +--- + +## Task 3: HTML — Gear trigger, panel markup, header in base + +**Files:** +- Modify: `internal/views/templates/base.html` +- Modify: `internal/views/views.go` + +- [ ] **Step 1: Add ShowHeader to PageData** + +In `views.go`, add `ShowHeader bool` to `PageData` struct. + +- [ ] **Step 2: Set ShowHeader in render functions** + +In `RenderIndex` and `RenderSearch`, set `PageData.ShowHeader = true`. + +- [ ] **Step 3: Update base.html — add header and settings markup** + +In `base.html`, update the `` to: + +```html + + {{if .ShowHeader}} + + {{end}} +
+ {{template "content" .}} +
+
+

Powered by kafka — a privacy-respecting, open metasearch engine

+
+ + + + +``` + +**Note:** The existing autocomplete ` + + + From af23a63a73a86d66b00931f363a839ae50243047 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 02:03:30 +0000 Subject: [PATCH 18/27] remove settings-design-playground.html --- .../specs/settings-design-playground.html | 337 ------------------ 1 file changed, 337 deletions(-) delete mode 100644 docs/superpowers/specs/settings-design-playground.html diff --git a/docs/superpowers/specs/settings-design-playground.html b/docs/superpowers/specs/settings-design-playground.html deleted file mode 100644 index 7cd8365..0000000 --- a/docs/superpowers/specs/settings-design-playground.html +++ /dev/null @@ -1,337 +0,0 @@ - - - - - - Settings UI Design Playground - - - -
-
-
- - - -
-
-
Access Pattern
- -
-
-
Theme Preview
- -
-
-
Trigger Button
- -
-
-
Settings Density
- -
Normal
-
-
-
Animation
-
- Enable transitions - -
-
-
-
Sections to Show
-
Theme
-
Engines
-
Search Defaults
-
-
- Click the gear button in the preview to toggle the settings panel. -
-
-
-
Update the settings UI to use a clean, minimal design with a slide-out drawer...
-
-
- kafka - -
- -
-
-

Results

-

About 42 results

-

Powered by kafka

-
-
-
-
github.com/golang/go
-
The Go Programming Language
-
-
-
go.dev
-
Go land
-
-
-
-
-
-
-
- - - From 4a6559be62ab57b04de57758fc261f82e3988837 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 02:06:41 +0000 Subject: [PATCH 19/27] fix: add Thumbnail field and video result template MainResult: add Thumbnail field (used by YouTube, images, etc.) video_item.html: new partial for video results with thumbnail display views.go: add templateForResult func + video_item.html to template parse results_inner.html: dispatch to video_item when Template="videos" kafka.css: add .video-result flex layout with thumbnail styling --- internal/contracts/main_result.go | 43 +++++++++++---------- internal/views/static/css/kafka.css | 23 +++++++++++ internal/views/templates/results_inner.html | 2 +- internal/views/templates/video_item.html | 22 +++++++++++ internal/views/views.go | 12 +++++- 5 files changed, 79 insertions(+), 23 deletions(-) create mode 100644 internal/views/templates/video_item.html diff --git a/internal/contracts/main_result.go b/internal/contracts/main_result.go index 20c9231..b5b568a 100644 --- a/internal/contracts/main_result.go +++ b/internal/contracts/main_result.go @@ -14,16 +14,17 @@ type MainResult struct { raw map[string]any // Common fields used by templates (RSS uses: title, url, content, pubdate). - Template string `json:"template"` - Title string `json:"title"` - Content string `json:"content"` - URL *string `json:"url"` - Pubdate *string `json:"pubdate"` + Template string `json:"template"` + Title string `json:"title"` + Content string `json:"content"` + URL *string `json:"url"` + Pubdate *string `json:"pubdate"` + Thumbnail string `json:"thumbnail"` - Engine string `json:"engine"` - Score float64 `json:"score"` - Category string `json:"category"` - Priority string `json:"priority"` + Engine string `json:"engine"` + Score float64 `json:"score"` + Category string `json:"category"` + Priority string `json:"priority"` Positions []int `json:"positions"` Engines []string `json:"engines"` @@ -54,6 +55,7 @@ func (mr *MainResult) UnmarshalJSON(data []byte) error { mr.Title = stringOrEmpty(m["title"]) mr.Content = stringOrEmpty(m["content"]) mr.Engine = stringOrEmpty(m["engine"]) + mr.Thumbnail = stringOrEmpty(m["thumbnail"]) mr.Category = stringOrEmpty(m["category"]) mr.Priority = stringOrEmpty(m["priority"]) @@ -93,20 +95,21 @@ func (mr MainResult) MarshalJSON() ([]byte, error) { // Otherwise, marshal the known fields. m := map[string]any{ - "template": mr.Template, - "title": mr.Title, - "content": mr.Content, - "url": mr.URL, - "pubdate": mr.Pubdate, - "engine": mr.Engine, - "score": mr.Score, - "category": mr.Category, - "priority": mr.Priority, + "template": mr.Template, + "title": mr.Title, + "content": mr.Content, + "url": mr.URL, + "pubdate": mr.Pubdate, + "thumbnail": mr.Thumbnail, + "engine": mr.Engine, + "score": mr.Score, + "category": mr.Category, + "priority": mr.Priority, "positions": mr.Positions, "engines": mr.Engines, - "open_group": mr.OpenGroup, + "open_group": mr.OpenGroup, "close_group": mr.CloseGroup, - "parsed_url": mr.ParsedURL, + "parsed_url": mr.ParsedURL, } return json.Marshal(m) } diff --git a/internal/views/static/css/kafka.css b/internal/views/static/css/kafka.css index 78f82e2..b637860 100644 --- a/internal/views/static/css/kafka.css +++ b/internal/views/static/css/kafka.css @@ -805,3 +805,26 @@ html[data-theme="light"] { opacity: 1; } } + +/* Video result cards */ +.video-result { + display: flex; + gap: 1rem; + align-items: flex-start; +} + +.video-result .result_thumbnail { + flex-shrink: 0; + width: 180px; +} + +.video-result .result_thumbnail img { + width: 100%; + height: auto; + border-radius: var(--radius); +} + +.video-result .result_content_wrapper { + flex: 1; + min-width: 0; +} diff --git a/internal/views/templates/results_inner.html b/internal/views/templates/results_inner.html index b6ad2e4..97aefb1 100644 --- a/internal/views/templates/results_inner.html +++ b/internal/views/templates/results_inner.html @@ -52,7 +52,7 @@
{{if .Results}} {{range .Results}} - {{template "result_item" .}} + {{template (templateForResult .Template) .}} {{end}} {{else if not .Answers}}
diff --git a/internal/views/templates/video_item.html b/internal/views/templates/video_item.html new file mode 100644 index 0000000..0ca0109 --- /dev/null +++ b/internal/views/templates/video_item.html @@ -0,0 +1,22 @@ +{{define "video_item"}} +
+ {{if .Thumbnail}} +
+ + {{.Title}} + +
+ {{end}} +
+

+ {{.Title}} +

+ {{if .Content}} +

{{.Content}}

+ {{end}} + {{if .Engine}} +
{{.Engine}}
+ {{end}} +
+
+{{end}} diff --git a/internal/views/views.go b/internal/views/views.go index c9e371e..5592235 100644 --- a/internal/views/views.go +++ b/internal/views/views.go @@ -63,16 +63,24 @@ func init() { funcMap := template.FuncMap{ "urlquery": template.URLQueryEscaper, + // templateForResult returns the template name to use for a result. + // Defaults to "result_item"; use "video_item" for video results. + "templateForResult": func(tmpl string) string { + if tmpl == "videos" { + return "video_item" + } + return "result_item" + }, } tmplFull = template.Must(template.New("").Funcs(funcMap).ParseFS(tmplFS, - "base.html", "results.html", "results_inner.html", "result_item.html", + "base.html", "results.html", "results_inner.html", "result_item.html", "video_item.html", )) tmplIndex = template.Must(template.New("").Funcs(funcMap).ParseFS(tmplFS, "base.html", "index.html", )) tmplFragment = template.Must(template.New("").Funcs(funcMap).ParseFS(tmplFS, - "results_inner.html", "result_item.html", + "results_inner.html", "result_item.html", "video_item.html", )) } From 281c327f6010025d81c4e0f10fa936a5ff2bca43 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 02:19:12 +0000 Subject: [PATCH 20/27] fix: correct go.mod to 1.24 (go 1.25 does not exist) Also use go-version-file in CI so go.mod and workflow stay in sync. --- .forgejo/workflows/test.yml | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/test.yml b/.forgejo/workflows/test.yml index 6b09ea7..bd05693 100644 --- a/.forgejo/workflows/test.yml +++ b/.forgejo/workflows/test.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: https://github.com/actions/setup-go@v5 with: - go-version: '1.24' + go-version-file: go.mod - name: Test run: go test -race -v ./... diff --git a/go.mod b/go.mod index f153b2d..d872485 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/metamorphosis-dev/kafka -go 1.25.0 +go 1.24 require ( github.com/BurntSushi/toml v1.5.0 From a03945b0e4acb6dc1797ab73da7fe3c7be1f1fef Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 02:27:03 +0000 Subject: [PATCH 21/27] fix: downgrade goquery to v1.9.0 (v1.12.0 requires Go 1.25) goquery v1.12.0 has a go.mod requirement of Go 1.25, which is not released yet. Downgrade to v1.9.0 which works with Go 1.24. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d872485..f5e61cb 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.24 require ( github.com/BurntSushi/toml v1.5.0 - github.com/PuerkitoBio/goquery v1.12.0 + github.com/PuerkitoBio/goquery v1.9.0 github.com/redis/go-redis/v9 v9.18.0 ) diff --git a/go.sum b/go.sum index 0aad3f0..ea8678a 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= -github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= +github.com/PuerkitoBio/goquery v1.9.0 h1:Q39JTQYMBgkBnIZcjBn4Eg5lyKlUyU3FwxC11fYN1w4= +github.com/PuerkitoBio/goquery v1.9.0/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= From 841526276e0d718bbafa71d9b99c6cac94b8d24d Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 02:33:01 +0000 Subject: [PATCH 22/27] fix: update goquery v1.9.0 checksum in go.sum --- go.sum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.sum b/go.sum index ea8678a..b16354f 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/PuerkitoBio/goquery v1.9.0 h1:Q39JTQYMBgkBnIZcjBn4Eg5lyKlUyU3FwxC11fYN1w4= +github.com/PuerkitoBio/goquery v1.9.0 h1:zgjKkdpRY9T97Q5DCtcXwfqkcylSFIVCocZmn2huTp8= github.com/PuerkitoBio/goquery v1.9.0/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= From 8a4a606dd66c94eeb3d6b6a5b2bc2ee4c9150724 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 02:39:10 +0000 Subject: [PATCH 23/27] fix: replace x/net v0.52.0 (requires go 1.25) with v0.33.0 (go 1.21) Use replace directive to force Go to use v0.33.0 instead of the transitively-pulled v0.52.0, which requires Go 1.25. --- go.mod | 2 ++ go.sum | 2 ++ 2 files changed, 4 insertions(+) diff --git a/go.mod b/go.mod index f5e61cb..e8eddd3 100644 --- a/go.mod +++ b/go.mod @@ -14,4 +14,6 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/net v0.52.0 // indirect + +replace golang.org/x/net v0.52.0 => golang.org/x/net v0.33.0 ) diff --git a/go.sum b/go.sum index b16354f..6b1b78b 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/PuerkitoBio/goquery v1.9.0 h1:zgjKkdpRY9T97Q5DCtcXwfqkcylSFIVCocZmn2h github.com/PuerkitoBio/goquery v1.9.0/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= From 7d3c82214bd7feeafd59b50befaadbd4e556eb37 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 02:40:12 +0000 Subject: [PATCH 24/27] fix: move replace directive outside require block --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e8eddd3..85f1653 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,6 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/net v0.52.0 // indirect +) replace golang.org/x/net v0.52.0 => golang.org/x/net v0.33.0 -) From f0a65e2b8c603e8f3fa08059f9c79836ed980dee Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 02:44:50 +0000 Subject: [PATCH 25/27] fix: compute TemplateName in ResultView instead of using dynamic template function Go html/template doesn't support function calls as template names in {{template (func .Arg) .}}. Instead, precompute TemplateName in FromResponse and use {{template .TemplateName .}} in the template. --- internal/views/templates/results_inner.html | 2 +- internal/views/views.go | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/internal/views/templates/results_inner.html b/internal/views/templates/results_inner.html index 97aefb1..66808f4 100644 --- a/internal/views/templates/results_inner.html +++ b/internal/views/templates/results_inner.html @@ -52,7 +52,7 @@
{{if .Results}} {{range .Results}} - {{template (templateForResult .Template) .}} + {{template .TemplateName .}} {{end}} {{else if not .Answers}}
diff --git a/internal/views/views.go b/internal/views/views.go index 5592235..58ef2eb 100644 --- a/internal/views/views.go +++ b/internal/views/views.go @@ -36,7 +36,12 @@ type PageData struct { } // ResultView is a template-friendly wrapper around a MainResult. -type ResultView contracts.MainResult +type ResultView struct { + contracts.MainResult + // TemplateName is the actual template to dispatch to, computed from Template. + // "videos" maps to "video_item", everything else maps to "result_item". + TemplateName string +} // PageNumber represents a numbered pagination button. type PageNumber struct { @@ -63,14 +68,6 @@ func init() { funcMap := template.FuncMap{ "urlquery": template.URLQueryEscaper, - // templateForResult returns the template name to use for a result. - // Defaults to "result_item"; use "video_item" for video results. - "templateForResult": func(tmpl string) string { - if tmpl == "videos" { - return "video_item" - } - return "result_item" - }, } tmplFull = template.Must(template.New("").Funcs(funcMap).ParseFS(tmplFS, @@ -113,7 +110,11 @@ func FromResponse(resp contracts.SearchResponse, query string, pageno int) PageD // Convert results. pd.Results = make([]ResultView, len(resp.Results)) for i, r := range resp.Results { - pd.Results[i] = ResultView(r) + tmplName := "result_item" + if r.Template == "videos" { + tmplName = "video_item" + } + pd.Results[i] = ResultView{MainResult: r, TemplateName: tmplName} } // Convert answers (they're map[string]any — extract string values). From b499db68f798fa1d1a443e04e8ddbf6785b0387b Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 02:46:28 +0000 Subject: [PATCH 26/27] fix: use explicit if/else template dispatch instead of dynamic name html/template requires template names to be string literals, not field accesses. Use {{if eq .Template "videos"}} to branch and call the appropriate template by literal name. --- internal/views/templates/results_inner.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/views/templates/results_inner.html b/internal/views/templates/results_inner.html index 66808f4..1b93c66 100644 --- a/internal/views/templates/results_inner.html +++ b/internal/views/templates/results_inner.html @@ -52,7 +52,11 @@
{{if .Results}} {{range .Results}} - {{template .TemplateName .}} + {{if eq .Template "videos"}} + {{template "video_item" .}} + {{else}} + {{template "result_item" .}} + {{end}} {{end}} {{else if not .Answers}}
From f1436310eb126d1c1126aa5c5b069d44f907a9f1 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 02:54:12 +0000 Subject: [PATCH 27/27] fix: regexp.DotAll flag in google engine and Metadata field removal - google.go: use inline (?s) flag instead of regexp.DotAll second arg - youtube.go: remove Metadata field (not in MainResult contract) - config_test.go: fix expected engine count from 9 to 11 (google+youtube) --- internal/config/config_test.go | 4 ++-- internal/engines/google.go | 2 +- internal/engines/youtube.go | 4 ---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4a09848..07bddef 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -14,8 +14,8 @@ func TestLoadDefaults(t *testing.T) { if cfg.Server.Port != 8080 { t.Errorf("expected default port 8080, got %d", cfg.Server.Port) } - if len(cfg.Engines.LocalPorted) != 9 { - t.Errorf("expected 9 default engines, got %d", len(cfg.Engines.LocalPorted)) + if len(cfg.Engines.LocalPorted) != 11 { + t.Errorf("expected 11 default engines, got %d", len(cfg.Engines.LocalPorted)) } } diff --git a/internal/engines/google.go b/internal/engines/google.go index 0371283..71b50d1 100644 --- a/internal/engines/google.go +++ b/internal/engines/google.go @@ -209,7 +209,7 @@ func extractGoogleSnippet(block string) string { func extractGoogleSuggestions(body string) []string { var suggestions []string // SearXNG xpath: //div[contains(@class, "ouy7Mc")]//a - suggestionPattern := regexp.MustCompile(`]*class="[^"]*ouy7Mc[^"]*"[^>]*>.*?]*>([^<]+)`, regexp.DotAll) + suggestionPattern := regexp.MustCompile(`(?s)]*class="[^"]*ouy7Mc[^"]*"[^>]*>.*?]*>([^<]+)`) matches := suggestionPattern.FindAllStringSubmatch(body, -1) seen := map[string]bool{} for _, m := range matches { diff --git a/internal/engines/youtube.go b/internal/engines/youtube.go index 7580a09..a18abd8 100644 --- a/internal/engines/youtube.go +++ b/internal/engines/youtube.go @@ -115,10 +115,6 @@ func (e *YouTubeEngine) Search(ctx context.Context, req contracts.SearchRequest) Score: 1.0, Category: "videos", Engines: []string{"youtube"}, - Metadata: map[string]any{ - "channel": item.Snippet.ChannelTitle, - "video_id": item.Snippet.ResourceID.VideoID, - }, }) }