Franz Kafka
b3e3123612
- Fix config validation: upstream URLs allow private IPs (self-hosted) - Fix util.SafeURLScheme to return parsed URL - Replace spoofed GSA User-Agent with honest Kafka UA - Sanitize all engine error messages (strip response bodies) - Replace unused body reads with io.Copy(io.Discard, ...) for reuse - Fix pre-existing braveapi_test using wrong struct type - Fix ratelimit test reference to limiter variable - Update ratelimit tests for new trusted proxy behavior
103 lines
3.2 KiB
Go
103 lines
3.2 KiB
Go
// kafka — a privacy-respecting metasearch engine
|
|
// Copyright (C) 2026-present metamorphosis-dev
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
package engines
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
|
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
|
)
|
|
|
|
// DuckDuckGoEngine searches DuckDuckGo's Lite/HTML endpoint.
|
|
// DuckDuckGo Lite returns a simple HTML page that can be scraped for results.
|
|
type DuckDuckGoEngine struct {
|
|
client *http.Client
|
|
}
|
|
|
|
func (e *DuckDuckGoEngine) Name() string { return "duckduckgo" }
|
|
|
|
func (e *DuckDuckGoEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
|
if strings.TrimSpace(req.Query) == "" {
|
|
return contracts.SearchResponse{Query: req.Query}, nil
|
|
}
|
|
if e == nil || e.client == nil {
|
|
return contracts.SearchResponse{}, errors.New("duckduckgo engine not initialized")
|
|
}
|
|
|
|
endpoint := fmt.Sprintf(
|
|
"https://lite.duckduckgo.com/lite/?q=%s&kl=%s",
|
|
url.QueryEscape(req.Query),
|
|
duckduckgoRegion(req.Language),
|
|
)
|
|
|
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
|
if err != nil {
|
|
return contracts.SearchResponse{}, err
|
|
}
|
|
httpReq.Header.Set("User-Agent", "kafka/0.1 (compatible; +https://git.ashisgreat.xyz/penal-colony/gosearch)")
|
|
|
|
resp, err := e.client.Do(httpReq)
|
|
if err != nil {
|
|
return contracts.SearchResponse{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
|
return contracts.SearchResponse{}, fmt.Errorf("duckduckgo upstream error: status %d", resp.StatusCode)
|
|
}
|
|
|
|
results, err := parseDuckDuckGoHTML(resp.Body)
|
|
if err != nil {
|
|
return contracts.SearchResponse{}, err
|
|
}
|
|
|
|
return contracts.SearchResponse{
|
|
Query: req.Query,
|
|
NumberOfResults: len(results),
|
|
Results: results,
|
|
Answers: []map[string]any{},
|
|
Corrections: []string{},
|
|
Infoboxes: []map[string]any{},
|
|
Suggestions: []string{},
|
|
UnresponsiveEngines: [][2]string{},
|
|
}, nil
|
|
}
|
|
|
|
// duckduckgoRegion maps language codes to DDG region parameters.
|
|
func duckduckgoRegion(lang string) string {
|
|
lang = strings.ToLower(strings.TrimSpace(lang))
|
|
if lang == "" || lang == "auto" {
|
|
return "us-en"
|
|
}
|
|
langCode := strings.SplitN(lang, "-", 2)[0]
|
|
regionMap := map[string]string{
|
|
"en": "us-en", "de": "de-de", "fr": "fr-fr", "es": "es-es",
|
|
"pt": "br-pt", "ru": "ru-ru", "ja": "jp-jp", "zh": "cn-zh",
|
|
"ko": "kr-kr", "it": "it-it", "nl": "nl-nl", "pl": "pl-pl",
|
|
}
|
|
if region, ok := regionMap[langCode]; ok {
|
|
return region
|
|
}
|
|
return "wt-wt"
|
|
}
|