kafka/internal/search/merge.go

package search

import (
	"encoding/json"
	"net/url"
	"strings"

	"github.com/metamorphosis-dev/kafka/internal/contracts"
)

// MergeResponses merges multiple SearXNG-compatible JSON responses.
//
// MVP merge semantics:
// - results are concatenated with a simple de-dup key (engine|title|url)
// - suggestions/corrections are de-duplicated as sets
// - answers/infoboxes/unresponsive_engines are concatenated (best-effort)
func MergeResponses(responses []contracts.SearchResponse) contracts.SearchResponse {
	var merged contracts.SearchResponse

	mergedResultSeen := map[string]struct{}{}
	mergedAnswerSeen := map[string]struct{}{}
	mergedCorrectionsSeen := map[string]struct{}{}
	mergedSuggestionsSeen := map[string]struct{}{}

	for _, r := range responses {
		if merged.Query == "" {
			merged.Query = r.Query
		}

		merged.NumberOfResults = maxInt(merged.NumberOfResults, r.NumberOfResults)

		for _, mr := range r.Results {
			key := resultDedupKey(mr)
			if _, ok := mergedResultSeen[key]; ok {
				continue
			}
			mergedResultSeen[key] = struct{}{}
			merged.Results = append(merged.Results, mr)
		}

		for _, ans := range r.Answers {
			// De-dup by normalized JSON when possible.
			b, err := json.Marshal(ans)
			if err != nil {
				merged.Answers = append(merged.Answers, ans)
				continue
			}
			key := string(b)
			if _, ok := mergedAnswerSeen[key]; ok {
				continue
			}
			mergedAnswerSeen[key] = struct{}{}
			merged.Answers = append(merged.Answers, ans)
		}

		merged.Corrections = unionStrings(merged.Corrections, r.Corrections, &mergedCorrectionsSeen)
		merged.Suggestions = unionStrings(merged.Suggestions, r.Suggestions, &mergedSuggestionsSeen)

		merged.Infoboxes = append(merged.Infoboxes, r.Infoboxes...)
		merged.UnresponsiveEngines = append(merged.UnresponsiveEngines, r.UnresponsiveEngines...)
	}

	// Ensure non-nil slices to keep JSON shape stable.
	if merged.Results == nil {
		merged.Results = []contracts.MainResult{}
	}
	if merged.Answers == nil {
		merged.Answers = []map[string]any{}
	}
	if merged.Corrections == nil {
		merged.Corrections = []string{}
	}
	if merged.Infoboxes == nil {
		merged.Infoboxes = []map[string]any{}
	}
	if merged.Suggestions == nil {
		merged.Suggestions = []string{}
	}
	if merged.UnresponsiveEngines == nil {
		merged.UnresponsiveEngines = [][2]string{}
	}

	return merged
}

func resultDedupKey(r contracts.MainResult) string {
	urlStr := ""
	if r.URL != nil {
		urlStr = *r.URL
	}
	// Normalize host to reduce duplicates.
	if u, err := url.Parse(urlStr); err == nil {
		if u.Host != "" {
			urlStr = u.Host + u.Path
		}
	}
	return strings.ToLower(r.Engine) + "|" + strings.ToLower(r.Title) + "|" + urlStr
}

func unionStrings(dst []string, src []string, seen *map[string]struct{}) []string {
	if *seen == nil {
		*seen = map[string]struct{}{}
	}
	out := dst
	for _, s := range src {
		if _, ok := (*seen)[s]; ok {
			continue
		}
		(*seen)[s] = struct{}{}
		out = append(out, s)
	}
	return out
}

func maxInt(a, b int) int {
	if a > b {
		return a
	}
	return b
}