feat: add YouTube engine with config file and env support
YouTube Data API v3 engine: - Add YouTubeConfig to EnginesConfig with api_key field - Add YOUTUBE_API_KEY env override - Thread *config.Config through search service to factory - Factory falls back to env vars if config fields are empty - Update config.example.toml with youtube section Also update default local_ported to include google and youtube.
This commit is contained in:
parent
1689cab9bd
commit
a7f594b7fa
5 changed files with 47 additions and 12 deletions
|
|
@ -56,6 +56,7 @@ func main() {
|
||||||
UpstreamURL: cfg.Upstream.URL,
|
UpstreamURL: cfg.Upstream.URL,
|
||||||
HTTPTimeout: cfg.HTTPTimeout(),
|
HTTPTimeout: cfg.HTTPTimeout(),
|
||||||
Cache: searchCache,
|
Cache: searchCache,
|
||||||
|
EnginesConfig: cfg,
|
||||||
})
|
})
|
||||||
|
|
||||||
acSvc := autocomplete.NewService(cfg.Upstream.URL, cfg.HTTPTimeout())
|
acSvc := autocomplete.NewService(cfg.Upstream.URL, cfg.HTTPTimeout())
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ url = ""
|
||||||
[engines]
|
[engines]
|
||||||
# Comma-separated list of engines to execute locally in Go (env: LOCAL_PORTED_ENGINES)
|
# Comma-separated list of engines to execute locally in Go (env: LOCAL_PORTED_ENGINES)
|
||||||
# Engines not listed here will be proxied to the upstream instance.
|
# Engines not listed here will be proxied to the upstream instance.
|
||||||
local_ported = ["wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"]
|
local_ported = ["wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing", "google", "youtube"]
|
||||||
|
|
||||||
[engines.brave]
|
[engines.brave]
|
||||||
# Brave Search API key (env: BRAVE_API_KEY)
|
# Brave Search API key (env: BRAVE_API_KEY)
|
||||||
|
|
@ -35,6 +35,10 @@ access_token = ""
|
||||||
category = "web-lite"
|
category = "web-lite"
|
||||||
results_per_page = 10
|
results_per_page = 10
|
||||||
|
|
||||||
|
[engines.youtube]
|
||||||
|
# YouTube Data API v3 key (env: YOUTUBE_API_KEY)
|
||||||
|
api_key = ""
|
||||||
|
|
||||||
[cache]
|
[cache]
|
||||||
# Valkey/Redis cache for search results.
|
# Valkey/Redis cache for search results.
|
||||||
# Leave address empty to disable caching entirely.
|
# Leave address empty to disable caching entirely.
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,7 @@ type EnginesConfig struct {
|
||||||
LocalPorted []string `toml:"local_ported"`
|
LocalPorted []string `toml:"local_ported"`
|
||||||
Brave BraveConfig `toml:"brave"`
|
Brave BraveConfig `toml:"brave"`
|
||||||
Qwant QwantConfig `toml:"qwant"`
|
Qwant QwantConfig `toml:"qwant"`
|
||||||
|
YouTube YouTubeConfig `toml:"youtube"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CacheConfig holds Valkey/Redis cache settings.
|
// CacheConfig holds Valkey/Redis cache settings.
|
||||||
|
|
@ -85,6 +86,10 @@ type QwantConfig struct {
|
||||||
ResultsPerPage int `toml:"results_per_page"`
|
ResultsPerPage int `toml:"results_per_page"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type YouTubeConfig struct {
|
||||||
|
APIKey string `toml:"api_key"`
|
||||||
|
}
|
||||||
|
|
||||||
// Load reads configuration from the given TOML file path.
|
// Load reads configuration from the given TOML file path.
|
||||||
// If the file does not exist, it returns defaults (empty values where applicable).
|
// If the file does not exist, it returns defaults (empty values where applicable).
|
||||||
// Environment variables are used as fallbacks for any zero-value fields.
|
// Environment variables are used as fallbacks for any zero-value fields.
|
||||||
|
|
@ -109,7 +114,7 @@ func defaultConfig() *Config {
|
||||||
},
|
},
|
||||||
Upstream: UpstreamConfig{},
|
Upstream: UpstreamConfig{},
|
||||||
Engines: EnginesConfig{
|
Engines: EnginesConfig{
|
||||||
LocalPorted: []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"},
|
LocalPorted: []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing", "google", "youtube"},
|
||||||
Qwant: QwantConfig{
|
Qwant: QwantConfig{
|
||||||
Category: "web-lite",
|
Category: "web-lite",
|
||||||
ResultsPerPage: 10,
|
ResultsPerPage: 10,
|
||||||
|
|
@ -151,6 +156,9 @@ func applyEnvOverrides(cfg *Config) {
|
||||||
if v := os.Getenv("BRAVE_ACCESS_TOKEN"); v != "" {
|
if v := os.Getenv("BRAVE_ACCESS_TOKEN"); v != "" {
|
||||||
cfg.Engines.Brave.AccessToken = v
|
cfg.Engines.Brave.AccessToken = v
|
||||||
}
|
}
|
||||||
|
if v := os.Getenv("YOUTUBE_API_KEY"); v != "" {
|
||||||
|
cfg.Engines.YouTube.APIKey = v
|
||||||
|
}
|
||||||
if v := os.Getenv("VALKEY_ADDRESS"); v != "" {
|
if v := os.Getenv("VALKEY_ADDRESS"); v != "" {
|
||||||
cfg.Cache.Address = v
|
cfg.Cache.Address = v
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,23 +4,42 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/metamorphosis-dev/kafka/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewDefaultPortedEngines returns the starter set of Go-native engines.
|
// NewDefaultPortedEngines returns the starter set of Go-native engines.
|
||||||
// The service can swap/extend this registry later as more engines are ported.
|
// The service can swap/extend this registry later as more engines are ported.
|
||||||
func NewDefaultPortedEngines(client *http.Client) map[string]Engine {
|
// If cfg is nil, falls back to reading API keys from environment variables.
|
||||||
|
func NewDefaultPortedEngines(client *http.Client, cfg *config.Config) map[string]Engine {
|
||||||
if client == nil {
|
if client == nil {
|
||||||
client = &http.Client{Timeout: 10 * time.Second}
|
client = &http.Client{Timeout: 10 * time.Second}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var braveAPIKey, braveAccessToken, youtubeAPIKey string
|
||||||
|
if cfg != nil {
|
||||||
|
braveAPIKey = cfg.Engines.Brave.APIKey
|
||||||
|
braveAccessToken = cfg.Engines.Brave.AccessToken
|
||||||
|
youtubeAPIKey = cfg.Engines.YouTube.APIKey
|
||||||
|
}
|
||||||
|
if braveAPIKey == "" {
|
||||||
|
braveAPIKey = os.Getenv("BRAVE_API_KEY")
|
||||||
|
}
|
||||||
|
if braveAccessToken == "" {
|
||||||
|
braveAccessToken = os.Getenv("BRAVE_ACCESS_TOKEN")
|
||||||
|
}
|
||||||
|
if youtubeAPIKey == "" {
|
||||||
|
youtubeAPIKey = os.Getenv("YOUTUBE_API_KEY")
|
||||||
|
}
|
||||||
|
|
||||||
return map[string]Engine{
|
return map[string]Engine{
|
||||||
"wikipedia": &WikipediaEngine{client: client},
|
"wikipedia": &WikipediaEngine{client: client},
|
||||||
"arxiv": &ArxivEngine{client: client},
|
"arxiv": &ArxivEngine{client: client},
|
||||||
"crossref": &CrossrefEngine{client: client},
|
"crossref": &CrossrefEngine{client: client},
|
||||||
"braveapi": &BraveEngine{
|
"braveapi": &BraveEngine{
|
||||||
client: client,
|
client: client,
|
||||||
apiKey: os.Getenv("BRAVE_API_KEY"),
|
apiKey: braveAPIKey,
|
||||||
accessGateToken: os.Getenv("BRAVE_ACCESS_TOKEN"),
|
accessGateToken: braveAccessToken,
|
||||||
resultsPerPage: 20,
|
resultsPerPage: 20,
|
||||||
},
|
},
|
||||||
"qwant": &QwantEngine{
|
"qwant": &QwantEngine{
|
||||||
|
|
@ -35,6 +54,7 @@ func NewDefaultPortedEngines(client *http.Client) map[string]Engine {
|
||||||
"google": &GoogleEngine{client: client},
|
"google": &GoogleEngine{client: client},
|
||||||
"youtube": &YouTubeEngine{
|
"youtube": &YouTubeEngine{
|
||||||
client: client,
|
client: client,
|
||||||
|
apiKey: youtubeAPIKey,
|
||||||
baseURL: "https://www.googleapis.com",
|
baseURL: "https://www.googleapis.com",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/kafka/internal/cache"
|
"github.com/metamorphosis-dev/kafka/internal/cache"
|
||||||
|
"github.com/metamorphosis-dev/kafka/internal/config"
|
||||||
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
"github.com/metamorphosis-dev/kafka/internal/engines"
|
"github.com/metamorphosis-dev/kafka/internal/engines"
|
||||||
"github.com/metamorphosis-dev/kafka/internal/upstream"
|
"github.com/metamorphosis-dev/kafka/internal/upstream"
|
||||||
|
|
@ -16,6 +17,7 @@ type ServiceConfig struct {
|
||||||
UpstreamURL string
|
UpstreamURL string
|
||||||
HTTPTimeout time.Duration
|
HTTPTimeout time.Duration
|
||||||
Cache *cache.Cache
|
Cache *cache.Cache
|
||||||
|
EnginesConfig *config.Config
|
||||||
}
|
}
|
||||||
|
|
||||||
type Service struct {
|
type Service struct {
|
||||||
|
|
@ -44,7 +46,7 @@ func NewService(cfg ServiceConfig) *Service {
|
||||||
return &Service{
|
return &Service{
|
||||||
upstreamClient: up,
|
upstreamClient: up,
|
||||||
planner: engines.NewPlannerFromEnv(),
|
planner: engines.NewPlannerFromEnv(),
|
||||||
localEngines: engines.NewDefaultPortedEngines(httpClient),
|
localEngines: engines.NewDefaultPortedEngines(httpClient, cfg.EnginesConfig),
|
||||||
cache: cfg.Cache,
|
cache: cfg.Cache,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue