From 41b80a939a66e9ad1c9ac5594ddd3dce5093cbb3 Mon Sep 17 00:00:00 2001 From: Franz Kafka Date: Sun, 22 Mar 2026 01:57:13 +0000 Subject: [PATCH] feat: add YouTube engine with config file and env support YouTube Data API v3 engine: - Add YouTubeConfig to EnginesConfig with api_key field - Add YOUTUBE_API_KEY env override - Thread *config.Config through search service to factory - Factory falls back to env vars if config fields are empty - Update config.example.toml with youtube section Also update default local_ported to include google and youtube. --- cmd/kafka/main.go | 7 ++++--- config.example.toml | 6 +++++- internal/config/config.go | 10 +++++++++- internal/engines/factory.go | 26 +++++++++++++++++++++++--- internal/search/service.go | 10 ++++++---- 5 files changed, 47 insertions(+), 12 deletions(-) diff --git a/cmd/kafka/main.go b/cmd/kafka/main.go index ab29852..90c750d 100644 --- a/cmd/kafka/main.go +++ b/cmd/kafka/main.go @@ -53,9 +53,10 @@ func main() { } svc := search.NewService(search.ServiceConfig{ - UpstreamURL: cfg.Upstream.URL, - HTTPTimeout: cfg.HTTPTimeout(), - Cache: searchCache, + UpstreamURL: cfg.Upstream.URL, + HTTPTimeout: cfg.HTTPTimeout(), + Cache: searchCache, + EnginesConfig: cfg, }) acSvc := autocomplete.NewService(cfg.Upstream.URL, cfg.HTTPTimeout()) diff --git a/config.example.toml b/config.example.toml index 1e3b75c..34f60a6 100644 --- a/config.example.toml +++ b/config.example.toml @@ -22,7 +22,7 @@ url = "" [engines] # Comma-separated list of engines to execute locally in Go (env: LOCAL_PORTED_ENGINES) # Engines not listed here will be proxied to the upstream instance. -local_ported = ["wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"] +local_ported = ["wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing", "google", "youtube"] [engines.brave] # Brave Search API key (env: BRAVE_API_KEY) @@ -35,6 +35,10 @@ access_token = "" category = "web-lite" results_per_page = 10 +[engines.youtube] +# YouTube Data API v3 key (env: YOUTUBE_API_KEY) +api_key = "" + [cache] # Valkey/Redis cache for search results. # Leave address empty to disable caching entirely. diff --git a/internal/config/config.go b/internal/config/config.go index 93b8d86..7f8b06a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -35,6 +35,7 @@ type EnginesConfig struct { LocalPorted []string `toml:"local_ported"` Brave BraveConfig `toml:"brave"` Qwant QwantConfig `toml:"qwant"` + YouTube YouTubeConfig `toml:"youtube"` } // CacheConfig holds Valkey/Redis cache settings. @@ -85,6 +86,10 @@ type QwantConfig struct { ResultsPerPage int `toml:"results_per_page"` } +type YouTubeConfig struct { + APIKey string `toml:"api_key"` +} + // Load reads configuration from the given TOML file path. // If the file does not exist, it returns defaults (empty values where applicable). // Environment variables are used as fallbacks for any zero-value fields. @@ -109,7 +114,7 @@ func defaultConfig() *Config { }, Upstream: UpstreamConfig{}, Engines: EnginesConfig{ - LocalPorted: []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"}, + LocalPorted: []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing", "google", "youtube"}, Qwant: QwantConfig{ Category: "web-lite", ResultsPerPage: 10, @@ -151,6 +156,9 @@ func applyEnvOverrides(cfg *Config) { if v := os.Getenv("BRAVE_ACCESS_TOKEN"); v != "" { cfg.Engines.Brave.AccessToken = v } + if v := os.Getenv("YOUTUBE_API_KEY"); v != "" { + cfg.Engines.YouTube.APIKey = v + } if v := os.Getenv("VALKEY_ADDRESS"); v != "" { cfg.Cache.Address = v } diff --git a/internal/engines/factory.go b/internal/engines/factory.go index 53ba87f..b7f3c00 100644 --- a/internal/engines/factory.go +++ b/internal/engines/factory.go @@ -4,23 +4,42 @@ import ( "net/http" "os" "time" + + "github.com/metamorphosis-dev/kafka/internal/config" ) // NewDefaultPortedEngines returns the starter set of Go-native engines. // The service can swap/extend this registry later as more engines are ported. -func NewDefaultPortedEngines(client *http.Client) map[string]Engine { +// If cfg is nil, falls back to reading API keys from environment variables. +func NewDefaultPortedEngines(client *http.Client, cfg *config.Config) map[string]Engine { if client == nil { client = &http.Client{Timeout: 10 * time.Second} } + var braveAPIKey, braveAccessToken, youtubeAPIKey string + if cfg != nil { + braveAPIKey = cfg.Engines.Brave.APIKey + braveAccessToken = cfg.Engines.Brave.AccessToken + youtubeAPIKey = cfg.Engines.YouTube.APIKey + } + if braveAPIKey == "" { + braveAPIKey = os.Getenv("BRAVE_API_KEY") + } + if braveAccessToken == "" { + braveAccessToken = os.Getenv("BRAVE_ACCESS_TOKEN") + } + if youtubeAPIKey == "" { + youtubeAPIKey = os.Getenv("YOUTUBE_API_KEY") + } + return map[string]Engine{ "wikipedia": &WikipediaEngine{client: client}, "arxiv": &ArxivEngine{client: client}, "crossref": &CrossrefEngine{client: client}, "braveapi": &BraveEngine{ client: client, - apiKey: os.Getenv("BRAVE_API_KEY"), - accessGateToken: os.Getenv("BRAVE_ACCESS_TOKEN"), + apiKey: braveAPIKey, + accessGateToken: braveAccessToken, resultsPerPage: 20, }, "qwant": &QwantEngine{ @@ -35,6 +54,7 @@ func NewDefaultPortedEngines(client *http.Client) map[string]Engine { "google": &GoogleEngine{client: client}, "youtube": &YouTubeEngine{ client: client, + apiKey: youtubeAPIKey, baseURL: "https://www.googleapis.com", }, } diff --git a/internal/search/service.go b/internal/search/service.go index 62a9308..47d2895 100644 --- a/internal/search/service.go +++ b/internal/search/service.go @@ -7,15 +7,17 @@ import ( "time" "github.com/metamorphosis-dev/kafka/internal/cache" + "github.com/metamorphosis-dev/kafka/internal/config" "github.com/metamorphosis-dev/kafka/internal/contracts" "github.com/metamorphosis-dev/kafka/internal/engines" "github.com/metamorphosis-dev/kafka/internal/upstream" ) type ServiceConfig struct { - UpstreamURL string - HTTPTimeout time.Duration - Cache *cache.Cache + UpstreamURL string + HTTPTimeout time.Duration + Cache *cache.Cache + EnginesConfig *config.Config } type Service struct { @@ -44,7 +46,7 @@ func NewService(cfg ServiceConfig) *Service { return &Service{ upstreamClient: up, planner: engines.NewPlannerFromEnv(), - localEngines: engines.NewDefaultPortedEngines(httpClient), + localEngines: engines.NewDefaultPortedEngines(httpClient, cfg.EnginesConfig), cache: cfg.Cache, } }