refactor: clean up verbose and redundant comments
Trim or remove comments that: - State the obvious (function names already convey purpose) - Repeat what the code clearly shows - Are excessively long without adding value Keep comments that explain *why*, not *what*.
This commit is contained in:
parent
805e7ffdc2
commit
5b942a5fd6
11 changed files with 16 additions and 102 deletions
|
|
@ -30,13 +30,9 @@ import (
|
|||
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||
)
|
||||
|
||||
// BraveEngine implements the `braveapi` engine (Brave Web Search API).
|
||||
//
|
||||
// Config / gating:
|
||||
// - BRAVE_API_KEY: required to call Brave
|
||||
// - BRAVE_ACCESS_TOKEN (optional): if set, the request must include a token
|
||||
// that matches the env var (via Authorization Bearer, X-Search-Token,
|
||||
// X-Brave-Access-Token, or form field `token`).
|
||||
// BraveEngine implements the Brave Web Search API.
|
||||
// Required: BRAVE_API_KEY env var or config.
|
||||
// Optional: BRAVE_ACCESS_TOKEN to gate requests.
|
||||
type BraveEngine struct {
|
||||
client *http.Client
|
||||
apiKey string
|
||||
|
|
@ -51,8 +47,6 @@ func (e *BraveEngine) Search(ctx context.Context, req contracts.SearchRequest) (
|
|||
return contracts.SearchResponse{}, errors.New("brave engine not initialized")
|
||||
}
|
||||
|
||||
// Gate / config checks should not be treated as fatal errors; the reference
|
||||
// implementation treats misconfigured engines as unresponsive.
|
||||
if strings.TrimSpace(e.apiKey) == "" {
|
||||
return contracts.SearchResponse{
|
||||
Query: req.Query,
|
||||
|
|
@ -109,8 +103,6 @@ func (e *BraveEngine) Search(ctx context.Context, req contracts.SearchRequest) (
|
|||
}
|
||||
}
|
||||
|
||||
// The reference implementation checks `if params["safesearch"]:` which treats any
|
||||
// non-zero (moderate/strict) as strict.
|
||||
if req.Safesearch > 0 {
|
||||
args.Set("safesearch", "strict")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,9 +24,8 @@ import (
|
|||
"github.com/metamorphosis-dev/kafka/internal/config"
|
||||
)
|
||||
|
||||
// NewDefaultPortedEngines returns the starter set of Go-native engines.
|
||||
// The service can swap/extend this registry later as more engines are ported.
|
||||
// If cfg is nil, falls back to reading API keys from environment variables.
|
||||
// NewDefaultPortedEngines returns the Go-native engine registry.
|
||||
// If cfg is nil, API keys fall back to environment variables.
|
||||
func NewDefaultPortedEngines(client *http.Client, cfg *config.Config) map[string]Engine {
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 10 * time.Second}
|
||||
|
|
|
|||
|
|
@ -57,7 +57,6 @@ func (e *GoogleEngine) Search(ctx context.Context, req contracts.SearchRequest)
|
|||
start := (req.Pageno - 1) * 10
|
||||
query := url.QueryEscape(req.Query)
|
||||
|
||||
// Build URL like SearXNG does.
|
||||
u := fmt.Sprintf(
|
||||
"https://www.google.com/search?q=%s&filter=0&start=%d&hl=%s&lr=%s&safe=%s",
|
||||
query,
|
||||
|
|
@ -118,7 +117,6 @@ func (e *GoogleEngine) Search(ctx context.Context, req contracts.SearchRequest)
|
|||
}, nil
|
||||
}
|
||||
|
||||
// detectGoogleSorry returns true if the response is a Google block/CAPTCHA page.
|
||||
func detectGoogleSorry(resp *http.Response) bool {
|
||||
if resp.Request != nil {
|
||||
if resp.Request.URL.Host == "sorry.google.com" || strings.HasPrefix(resp.Request.URL.Path, "/sorry") {
|
||||
|
|
@ -128,16 +126,9 @@ func detectGoogleSorry(resp *http.Response) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
// parseGoogleResults extracts search results from Google's HTML.
|
||||
// Uses the same selectors as SearXNG: div.MjjYud for result containers.
|
||||
func parseGoogleResults(body, query string) []contracts.MainResult {
|
||||
var results []contracts.MainResult
|
||||
|
||||
// SearXNG selector: .//div[contains(@class, "MjjYud")]
|
||||
// Each result block contains a title link and snippet.
|
||||
// We simulate the XPath matching with regex-based extraction.
|
||||
|
||||
// Find all MjjYud div blocks.
|
||||
mjjPattern := regexp.MustCompile(`<div[^>]*class="[^"]*MjjYud[^"]*"[^>]*>(.*?)</div>\s*(?=<div[^>]*class="[^"]*MjjYud|$)`)
|
||||
matches := mjjPattern.FindAllStringSubmatch(body, -1)
|
||||
|
||||
|
|
@ -147,15 +138,12 @@ func parseGoogleResults(body, query string) []contracts.MainResult {
|
|||
}
|
||||
block := match[1]
|
||||
|
||||
// Extract title and URL from the result link.
|
||||
// Pattern: <a href="/url?q=ACTUAL_URL&sa=..." ...>TITLE</a>
|
||||
urlPattern := regexp.MustCompile(`<a[^>]+href="(/url\?q=[^"&]+)`)
|
||||
urlMatch := urlPattern.FindStringSubmatch(block)
|
||||
if len(urlMatch) < 2 {
|
||||
continue
|
||||
}
|
||||
rawURL := urlMatch[1]
|
||||
// Remove /url?q= prefix and decode.
|
||||
actualURL := strings.TrimPrefix(rawURL, "/url?q=")
|
||||
if amp := strings.Index(actualURL, "&"); amp != -1 {
|
||||
actualURL = actualURL[:amp]
|
||||
|
|
@ -168,14 +156,12 @@ func parseGoogleResults(body, query string) []contracts.MainResult {
|
|||
continue
|
||||
}
|
||||
|
||||
// Extract title from the title tag.
|
||||
titlePattern := regexp.MustCompile(`<span[^>]*class="[^"]*qrStP[^"]*"[^>]*>([^<]+)</span>`)
|
||||
titleMatch := titlePattern.FindStringSubmatch(block)
|
||||
title := query
|
||||
if len(titleMatch) >= 2 {
|
||||
title = stripTags(titleMatch[1])
|
||||
} else {
|
||||
// Fallback: extract visible text from an <a> with data-title or role="link"
|
||||
linkTitlePattern := regexp.MustCompile(`<a[^>]+role="link"[^>]*>([^<]+)<`)
|
||||
ltMatch := linkTitlePattern.FindStringSubmatch(block)
|
||||
if len(ltMatch) >= 2 {
|
||||
|
|
@ -183,7 +169,6 @@ func parseGoogleResults(body, query string) []contracts.MainResult {
|
|||
}
|
||||
}
|
||||
|
||||
// Extract snippet from data-sncf divs (SearXNG's approach).
|
||||
snippet := extractGoogleSnippet(block)
|
||||
|
||||
urlPtr := actualURL
|
||||
|
|
@ -202,10 +187,7 @@ func parseGoogleResults(body, query string) []contracts.MainResult {
|
|||
return results
|
||||
}
|
||||
|
||||
// extractGoogleSnippet extracts the snippet text from a Google result block.
|
||||
func extractGoogleSnippet(block string) string {
|
||||
// Google's snippets live in divs with data-sncf attribute.
|
||||
// SearXNG looks for: .//div[contains(@data-sncf, "1")]
|
||||
snippetPattern := regexp.MustCompile(`<div[^>]+data-sncf="1"[^>]*>(.*?)</div>`)
|
||||
matches := snippetPattern.FindAllStringSubmatch(block, -1)
|
||||
var parts []string
|
||||
|
|
@ -221,10 +203,8 @@ func extractGoogleSnippet(block string) string {
|
|||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// extractGoogleSuggestions extracts search suggestions from Google result cards.
|
||||
func extractGoogleSuggestions(body string) []string {
|
||||
var suggestions []string
|
||||
// SearXNG xpath: //div[contains(@class, "ouy7Mc")]//a
|
||||
suggestionPattern := regexp.MustCompile(`(?s)<div[^>]*class="[^"]*ouy7Mc[^"]*"[^>]*>.*?<a[^>]*>([^<]+)</a>`)
|
||||
matches := suggestionPattern.FindAllStringSubmatch(body, -1)
|
||||
seen := map[string]bool{}
|
||||
|
|
@ -241,8 +221,6 @@ func extractGoogleSuggestions(body string) []string {
|
|||
return suggestions
|
||||
}
|
||||
|
||||
// googleHL maps SearXNG locale to Google hl (host language) parameter.
|
||||
// e.g. "en-US" -> "en-US"
|
||||
func googleHL(lang string) string {
|
||||
lang = strings.ToLower(strings.TrimSpace(lang))
|
||||
if lang == "" || lang == "auto" {
|
||||
|
|
@ -251,8 +229,6 @@ func googleHL(lang string) string {
|
|||
return lang
|
||||
}
|
||||
|
||||
// googleUILanguage maps SearXNG language to Google lr (language restrict) parameter.
|
||||
// e.g. "en" -> "lang_en", "de" -> "lang_de"
|
||||
func googleUILanguage(lang string) string {
|
||||
lang = strings.ToLower(strings.Split(lang, "-")[0])
|
||||
if lang == "" || lang == "auto" {
|
||||
|
|
@ -261,7 +237,6 @@ func googleUILanguage(lang string) string {
|
|||
return "lang_" + lang
|
||||
}
|
||||
|
||||
// googleSafeSearchLevel maps safesearch (0-2) to Google's safe parameter.
|
||||
func googleSafeSearchLevel(safesearch int) string {
|
||||
switch safesearch {
|
||||
case 0:
|
||||
|
|
@ -275,7 +250,6 @@ func googleSafeSearchLevel(safesearch int) string {
|
|||
}
|
||||
}
|
||||
|
||||
// stripTags removes HTML tags from a string.
|
||||
func stripTags(s string) string {
|
||||
stripper := regexp.MustCompile(`<[^>]*>`)
|
||||
s = stripper.ReplaceAllString(s, "")
|
||||
|
|
|
|||
|
|
@ -95,9 +95,6 @@ func (p *Planner) Plan(req contracts.SearchRequest) (localEngines, upstreamEngin
|
|||
}
|
||||
|
||||
func inferFromCategories(categories []string) []string {
|
||||
// Minimal mapping for the initial porting subset.
|
||||
// This mirrors the idea of selecting from engine categories without
|
||||
// embedding the whole engine registry.
|
||||
set := map[string]bool{}
|
||||
for _, c := range categories {
|
||||
switch strings.TrimSpace(strings.ToLower(c)) {
|
||||
|
|
@ -131,7 +128,6 @@ func inferFromCategories(categories []string) []string {
|
|||
}
|
||||
|
||||
func sortByOrder(list []string, order map[string]int) {
|
||||
// simple insertion sort (list is tiny)
|
||||
for i := 1; i < len(list); i++ {
|
||||
j := i
|
||||
for j > 0 && order[list[j-1]] > order[list[j]] {
|
||||
|
|
|
|||
|
|
@ -30,11 +30,7 @@ import (
|
|||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
// QwantEngine implements a `qwant` (web) adapter using
|
||||
// Qwant v3 endpoint: https://api.qwant.com/v3/search/web.
|
||||
//
|
||||
// Qwant's API is not fully documented; this implements parsing logic
|
||||
// for the `web` category.
|
||||
// QwantEngine implements the Qwant v3 API (web and web-lite modes).
|
||||
type QwantEngine struct {
|
||||
client *http.Client
|
||||
category string // "web" (JSON API) or "web-lite" (HTML fallback)
|
||||
|
|
@ -53,8 +49,6 @@ func (e *QwantEngine) Search(ctx context.Context, req contracts.SearchRequest) (
|
|||
return contracts.SearchResponse{Query: req.Query}, nil
|
||||
}
|
||||
|
||||
// For API parity we use web defaults: count=10, offset=(pageno-1)*count.
|
||||
// The engine's config field exists so we can expand to news/images/videos later.
|
||||
count := e.resultsPerPage
|
||||
if count <= 0 {
|
||||
count = 10
|
||||
|
|
@ -271,9 +265,7 @@ func (e *QwantEngine) searchWebLite(ctx context.Context, req contracts.SearchReq
|
|||
results := make([]contracts.MainResult, 0)
|
||||
seen := map[string]bool{}
|
||||
|
||||
// Pattern 1: legacy/known qwant-lite structure.
|
||||
doc.Find("section article").Each(func(_ int, item *goquery.Selection) {
|
||||
// ignore randomly interspersed advertising adds
|
||||
if item.Find("span.tooltip").Length() > 0 {
|
||||
return
|
||||
}
|
||||
|
|
@ -307,19 +299,14 @@ func (e *QwantEngine) searchWebLite(ctx context.Context, req contracts.SearchReq
|
|||
})
|
||||
})
|
||||
|
||||
// Pattern 2: broader fallback for updated lite markup:
|
||||
// any article/list item/div block containing an external anchor.
|
||||
// We keep this conservative by requiring non-empty title + URL.
|
||||
doc.Find("article, li, div").Each(func(_ int, item *goquery.Selection) {
|
||||
if len(results) >= 20 {
|
||||
return
|
||||
}
|
||||
// Skip ad-like blocks in fallback pass too.
|
||||
if item.Find("span.tooltip").Length() > 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Skip obvious nav/footer blocks.
|
||||
classAttr, _ := item.Attr("class")
|
||||
classLower := strings.ToLower(classAttr)
|
||||
if strings.Contains(classLower, "nav") || strings.Contains(classLower, "footer") {
|
||||
|
|
@ -368,13 +355,10 @@ func (e *QwantEngine) searchWebLite(ctx context.Context, req contracts.SearchReq
|
|||
}
|
||||
seen[href] = true
|
||||
|
||||
// Best-effort snippet extraction from nearby paragraph/span text.
|
||||
content := strings.TrimSpace(item.Find("p").First().Text())
|
||||
if content == "" {
|
||||
content = strings.TrimSpace(item.Find("span").First().Text())
|
||||
}
|
||||
// If there is no snippet, still keep clearly external result links.
|
||||
// Qwant-lite frequently omits rich snippets for some entries.
|
||||
|
||||
u := href
|
||||
results = append(results, contracts.MainResult{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue