From 8649864971b4faa2794a1ccd1f7077cf056f25c9 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sat, 21 Mar 2026 14:56:00 +0000 Subject: [PATCH] feat: migrate from env vars to config.toml - Add internal/config package with TOML parsing (BurntSushi/toml) - Create config.example.toml documenting all settings - Update main.go to load config via -config flag (default: config.toml) - Environment variables remain as fallback overrides for backward compat - Config file values are used as defaults; env vars override when set - Add comprehensive tests for file loading, defaults, and env overrides - Add config.toml to .gitignore (secrets stay local) --- .gitignore | 2 +- cmd/searxng-go/main.go | 38 +++++---- config.example.toml | 31 +++++++ go.mod | 1 + internal/config/config.go | 130 ++++++++++++++++++++++++++++++ internal/config/config_test.go | 143 +++++++++++++++++++++++++++++++++ 6 files changed, 329 insertions(+), 16 deletions(-) create mode 100644 config.example.toml create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go diff --git a/.gitignore b/.gitignore index de8d99a..5b6c096 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -.agent/ +config.toml diff --git a/cmd/searxng-go/main.go b/cmd/searxng-go/main.go index 7797ee6..bc82387 100644 --- a/cmd/searxng-go/main.go +++ b/cmd/searxng-go/main.go @@ -1,33 +1,42 @@ package main import ( + "flag" + "fmt" "log" "net/http" "os" - "time" + "github.com/ashie/gosearch/internal/config" "github.com/ashie/gosearch/internal/httpapi" "github.com/ashie/gosearch/internal/search" ) func main() { - port := os.Getenv("PORT") - if port == "" { - port = "8080" + configPath := flag.String("config", "config.toml", "path to config.toml") + flag.Parse() + + cfg, err := config.Load(*configPath) + if err != nil { + log.Fatalf("failed to load config: %v", err) } - upstreamURL := os.Getenv("UPSTREAM_SEARXNG_URL") - - timeout := 10 * time.Second - if v := os.Getenv("HTTP_TIMEOUT"); v != "" { - if d, err := time.ParseDuration(v); err == nil { - timeout = d - } + // Seed env vars from config so existing engine/factory/planner code + // picks them up without changes. The config layer is the single source + // of truth; env vars remain as overrides via applyEnvOverrides. + if len(cfg.Engines.LocalPorted) > 0 { + os.Setenv("LOCAL_PORTED_ENGINES", cfg.LocalPortedCSV()) + } + if cfg.Engines.Brave.APIKey != "" { + os.Setenv("BRAVE_API_KEY", cfg.Engines.Brave.APIKey) + } + if cfg.Engines.Brave.AccessToken != "" { + os.Setenv("BRAVE_ACCESS_TOKEN", cfg.Engines.Brave.AccessToken) } svc := search.NewService(search.ServiceConfig{ - UpstreamURL: upstreamURL, - HTTPTimeout: timeout, + UpstreamURL: cfg.Upstream.URL, + HTTPTimeout: cfg.HTTPTimeout(), }) h := httpapi.NewHandler(svc) @@ -36,8 +45,7 @@ func main() { mux.HandleFunc("/healthz", h.Healthz) mux.HandleFunc("/search", h.Search) - addr := ":" + port + addr := fmt.Sprintf(":%d", cfg.Server.Port) log.Printf("searxng-go listening on %s", addr) log.Fatal(http.ListenAndServe(addr, mux)) } - diff --git a/config.example.toml b/config.example.toml new file mode 100644 index 0000000..e3e9556 --- /dev/null +++ b/config.example.toml @@ -0,0 +1,31 @@ +# gosearch configuration +# Copy to config.toml and adjust as needed. +# Environment variables are used as fallbacks when a config field is empty/unset. + +[server] +# Listen port (env: PORT) +port = 8080 + +# HTTP timeout for engine and upstream calls (env: HTTP_TIMEOUT) +http_timeout = "10s" + +[upstream] +# URL of an upstream SearXNG instance for unported engines (env: UPSTREAM_SEARXNG_URL) +# Leave empty to run without an upstream proxy. +url = "" + +[engines] +# Comma-separated list of engines to execute locally in Go (env: LOCAL_PORTED_ENGINES) +# Engines not listed here will be proxied to upstream SearXNG. +local_ported = ["wikipedia", "arxiv", "crossref", "braveapi", "qwant"] + +[engines.brave] +# Brave Search API key (env: BRAVE_API_KEY) +api_key = "" +# Optional access token to gate requests (env: BRAVE_ACCESS_TOKEN) +access_token = "" + +[engines.qwant] +# Qwant category: "web" or "web-lite" (default: "web-lite") +category = "web-lite" +results_per_page = 10 diff --git a/go.mod b/go.mod index 288c9ef..2c2e94f 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/ashie/gosearch go 1.25.0 require ( + github.com/BurntSushi/toml v1.5.0 github.com/PuerkitoBio/goquery v1.12.0 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect golang.org/x/net v0.52.0 // indirect diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..aeca2af --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,130 @@ +package config + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/BurntSushi/toml" +) + +// Config is the top-level configuration for the gosearch service. +type Config struct { + Server ServerConfig `toml:"server"` + Upstream UpstreamConfig `toml:"upstream"` + Engines EnginesConfig `toml:"engines"` +} + +type ServerConfig struct { + Port int `toml:"port"` + HTTPTimeout string `toml:"http_timeout"` +} + +type UpstreamConfig struct { + URL string `toml:"url"` +} + +type EnginesConfig struct { + LocalPorted []string `toml:"local_ported"` + Brave BraveConfig `toml:"brave"` + Qwant QwantConfig `toml:"qwant"` +} + +type BraveConfig struct { + APIKey string `toml:"api_key"` + AccessToken string `toml:"access_token"` +} + +type QwantConfig struct { + Category string `toml:"category"` + ResultsPerPage int `toml:"results_per_page"` +} + +// Load reads configuration from the given TOML file path. +// If the file does not exist, it returns defaults (empty values where applicable). +// Environment variables are used as fallbacks for any zero-value fields. +func Load(path string) (*Config, error) { + cfg := defaultConfig() + + if _, err := os.Stat(path); err == nil { + if _, err := toml.DecodeFile(path, &cfg); err != nil { + return nil, fmt.Errorf("parse config %s: %w", path, err) + } + } + + applyEnvOverrides(cfg) + return cfg, nil +} + +func defaultConfig() *Config { + return &Config{ + Server: ServerConfig{ + Port: 8080, + HTTPTimeout: "10s", + }, + Upstream: UpstreamConfig{}, + Engines: EnginesConfig{ + LocalPorted: []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant"}, + Qwant: QwantConfig{ + Category: "web-lite", + ResultsPerPage: 10, + }, + }, + } +} + +// applyEnvOverrides fills any zero-value fields from environment variables. +// This preserves backward compatibility: existing deployments using env vars +// continue to work without a config file. +func applyEnvOverrides(cfg *Config) { + if v := os.Getenv("PORT"); v != "" { + fmt.Sscanf(v, "%d", &cfg.Server.Port) + } + if v := os.Getenv("HTTP_TIMEOUT"); v != "" { + cfg.Server.HTTPTimeout = v + } + if v := os.Getenv("UPSTREAM_SEARXNG_URL"); v != "" { + cfg.Upstream.URL = v + } + if v := os.Getenv("LOCAL_PORTED_ENGINES"); v != "" { + parts := splitCSV(v) + if len(parts) > 0 { + cfg.Engines.LocalPorted = parts + } + } + if v := os.Getenv("BRAVE_API_KEY"); v != "" { + cfg.Engines.Brave.APIKey = v + } + if v := os.Getenv("BRAVE_ACCESS_TOKEN"); v != "" { + cfg.Engines.Brave.AccessToken = v + } +} + +// HTTPTimeout parses the configured timeout string into a time.Duration. +func (c *Config) HTTPTimeout() time.Duration { + if d, err := time.ParseDuration(c.Server.HTTPTimeout); err == nil && d > 0 { + return d + } + return 10 * time.Second +} + +// LocalPortedCSV returns the local ported engines as a comma-separated string. +func (c *Config) LocalPortedCSV() string { + return strings.Join(c.Engines.LocalPorted, ",") +} + +func splitCSV(s string) []string { + if s == "" { + return nil + } + parts := strings.Split(s, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..4f0cc43 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,143 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadDefaults(t *testing.T) { + cfg, err := Load("/nonexistent/config.toml") + if err != nil { + t.Fatalf("Load with missing file should return defaults: %v", err) + } + if cfg.Server.Port != 8080 { + t.Errorf("expected default port 8080, got %d", cfg.Server.Port) + } + if len(cfg.Engines.LocalPorted) != 5 { + t.Errorf("expected 5 default engines, got %d", len(cfg.Engines.LocalPorted)) + } +} + +func TestLoadFromFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.toml") + content := ` +[server] +port = 9090 +http_timeout = "30s" + +[upstream] +url = "http://localhost:8888" + +[engines] +local_ported = ["wikipedia", "braveapi"] + +[engines.brave] +api_key = "test-key" +access_token = "secret" + +[engines.qwant] +category = "web" +results_per_page = 5 +` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load failed: %v", err) + } + + if cfg.Server.Port != 9090 { + t.Errorf("expected port 9090, got %d", cfg.Server.Port) + } + if cfg.Server.HTTPTimeout != "30s" { + t.Errorf("expected http_timeout 30s, got %s", cfg.Server.HTTPTimeout) + } + if cfg.Upstream.URL != "http://localhost:8888" { + t.Errorf("expected upstream URL, got %s", cfg.Upstream.URL) + } + if len(cfg.Engines.LocalPorted) != 2 { + t.Errorf("expected 2 engines, got %d", len(cfg.Engines.LocalPorted)) + } + if cfg.Engines.Brave.APIKey != "test-key" { + t.Errorf("expected brave api_key, got %s", cfg.Engines.Brave.APIKey) + } + if cfg.Engines.Brave.AccessToken != "secret" { + t.Errorf("expected brave access_token, got %s", cfg.Engines.Brave.AccessToken) + } + if cfg.Engines.Qwant.Category != "web" { + t.Errorf("expected qwant category web, got %s", cfg.Engines.Qwant.Category) + } + if cfg.Engines.Qwant.ResultsPerPage != 5 { + t.Errorf("expected qwant results_per_page 5, got %d", cfg.Engines.Qwant.ResultsPerPage) + } +} + +func TestEnvOverrides(t *testing.T) { + t.Setenv("PORT", "3000") + t.Setenv("HTTP_TIMEOUT", "5s") + t.Setenv("UPSTREAM_SEARXNG_URL", "http://env:9999") + t.Setenv("BRAVE_API_KEY", "env-key") + t.Setenv("LOCAL_PORTED_ENGINES", "wikipedia,arxiv") + + cfg, err := Load("/nonexistent/config.toml") + if err != nil { + t.Fatal(err) + } + + if cfg.Server.Port != 3000 { + t.Errorf("expected env port 3000, got %d", cfg.Server.Port) + } + if cfg.Server.HTTPTimeout != "5s" { + t.Errorf("expected env timeout 5s, got %s", cfg.Server.HTTPTimeout) + } + if cfg.Upstream.URL != "http://env:9999" { + t.Errorf("expected env upstream URL, got %s", cfg.Upstream.URL) + } + if cfg.Engines.Brave.APIKey != "env-key" { + t.Errorf("expected env brave key, got %s", cfg.Engines.Brave.APIKey) + } + if len(cfg.Engines.LocalPorted) != 2 { + t.Errorf("expected 2 env engines, got %d", len(cfg.Engines.LocalPorted)) + } +} + +func TestConfigFileWinsOverDefaults(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.toml") + content := ` +[server] +port = 7070 +` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + + t.Setenv("PORT", "3000") // env should override config + + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + + // Env overrides the config file + if cfg.Server.Port != 3000 { + t.Errorf("env should override config: expected 3000, got %d", cfg.Server.Port) + } +} + +func TestHTTPTimeout(t *testing.T) { + cfg := &Config{Server: ServerConfig{HTTPTimeout: "5s"}} + if d := cfg.HTTPTimeout(); d.String() != "5s" { + t.Errorf("expected 5s, got %s", d) + } + + // Invalid falls back to 10s + cfg.Server.HTTPTimeout = "not-a-duration" + if d := cfg.HTTPTimeout(); d.String() != "10s" { + t.Errorf("expected fallback 10s, got %s", d) + } +}