feat: add DuckDuckGo, GitHub, Reddit, and Bing engines
- DuckDuckGo: scrapes Lite HTML endpoint for results - Language-aware region mapping (de→de-de, ja→jp-jp, etc.) - HTML parser extracts result links and snippets from DDG Lite markup - Shared html_helpers.go with extractAttr, stripHTML, htmlUnescape - GitHub: uses public Search API (repos, sorted by stars) - No auth required (10 req/min unauthenticated) - Shows stars, language, topics, last updated date - Paginated via GitHub's page parameter - Reddit: uses public JSON search API - Respects safesearch (skips over_18 posts) - Shows subreddit, score, comment count - Links self-posts to the thread URL - Bing: scrapes web search HTML (b_algo containers) - Extracts titles, URLs, and snippets from Bing's result markup - Handles Bing's tracking URL encoding - Updated factory, config defaults, and config.example.toml - Full test suite: unit tests for all engines, HTML parsing tests, region mapping tests, live request tests (skipped in short mode) 9 engines total: wikipedia, arxiv, crossref, braveapi, qwant, duckduckgo, github, reddit, bing
This commit is contained in:
parent
28b61ff251
commit
df8fe9474b
14 changed files with 1030 additions and 5 deletions
|
|
@ -92,7 +92,7 @@ func defaultConfig() *Config {
|
|||
},
|
||||
Upstream: UpstreamConfig{},
|
||||
Engines: EnginesConfig{
|
||||
LocalPorted: []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant"},
|
||||
LocalPorted: []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"},
|
||||
Qwant: QwantConfig{
|
||||
Category: "web-lite",
|
||||
ResultsPerPage: 10,
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ func TestLoadDefaults(t *testing.T) {
|
|||
if cfg.Server.Port != 8080 {
|
||||
t.Errorf("expected default port 8080, got %d", cfg.Server.Port)
|
||||
}
|
||||
if len(cfg.Engines.LocalPorted) != 5 {
|
||||
t.Errorf("expected 5 default engines, got %d", len(cfg.Engines.LocalPorted))
|
||||
if len(cfg.Engines.LocalPorted) != 9 {
|
||||
t.Errorf("expected 9 default engines, got %d", len(cfg.Engines.LocalPorted))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue