Compare commits
No commits in common. "main" and "feat/google-engine" have entirely different histories.
main
...
feat/googl
96 changed files with 1338 additions and 8864 deletions
|
|
@ -11,15 +11,12 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: https://github.com/actions/checkout@v5
|
uses: https://github.com/actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: https://github.com/actions/setup-go@v5
|
uses: https://github.com/actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version-file: go.mod
|
go-version: '1.24'
|
||||||
|
|
||||||
- name: Clean vendor
|
|
||||||
run: rm -rf vendor
|
|
||||||
|
|
||||||
- name: Test
|
- name: Test
|
||||||
run: go test -race -v ./...
|
run: go test -race -v ./...
|
||||||
|
|
|
||||||
25
.github/workflows/test.yml
vendored
25
.github/workflows/test.yml
vendored
|
|
@ -1,25 +0,0 @@
|
||||||
name: Tests
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
branches: [main]
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
test:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v5
|
|
||||||
|
|
||||||
- name: Set up Go
|
|
||||||
uses: actions/setup-go@v5
|
|
||||||
with:
|
|
||||||
go-version-file: go.mod
|
|
||||||
|
|
||||||
- name: Clean vendor
|
|
||||||
run: rm -rf vendor
|
|
||||||
|
|
||||||
- name: Test
|
|
||||||
run: go test -race -v ./...
|
|
||||||
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -1,11 +1,5 @@
|
||||||
node_modules/
|
node_modules/
|
||||||
.agent/
|
.agent/
|
||||||
internal/spa/dist/
|
|
||||||
frontend/node_modules/
|
|
||||||
frontend/dist/
|
|
||||||
frontend/bun.lock
|
|
||||||
frontend/bun.lockb
|
|
||||||
frontend/package-lock.json
|
|
||||||
*.exe
|
*.exe
|
||||||
*.exe~
|
*.exe~
|
||||||
*.dll
|
*.dll
|
||||||
|
|
|
||||||
12
CLAUDE.md
12
CLAUDE.md
|
|
@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
||||||
|
|
||||||
## Project Overview
|
## Project Overview
|
||||||
|
|
||||||
samsa is a privacy-respecting metasearch engine written in Go. It provides a SearXNG-compatible `/search` API and an HTML frontend (HTMX + Go templates). 9 engines are implemented natively in Go; unlisted engines can be proxied to an upstream metasearch instance. Responses from multiple engines are merged into a single JSON/CSV/RSS/HTML response.
|
kafka is a privacy-respecting metasearch engine written in Go. It provides a SearXNG-compatible `/search` API and an HTML frontend (HTMX + Go templates). 9 engines are implemented natively in Go; unlisted engines can be proxied to an upstream SearXNG instance. Responses from multiple engines are merged into a single JSON/CSV/RSS/HTML response.
|
||||||
|
|
||||||
## Build & Run Commands
|
## Build & Run Commands
|
||||||
|
|
||||||
|
|
@ -22,7 +22,7 @@ go test -run TestWikipedia ./internal/engines/
|
||||||
go test -v ./internal/engines/
|
go test -v ./internal/engines/
|
||||||
|
|
||||||
# Run the server (requires config.toml)
|
# Run the server (requires config.toml)
|
||||||
go run ./cmd/samsa -config config.toml
|
go run ./cmd/searxng-go -config config.toml
|
||||||
```
|
```
|
||||||
|
|
||||||
There is no Makefile. There is no linter configured.
|
There is no Makefile. There is no linter configured.
|
||||||
|
|
@ -37,13 +37,13 @@ There is no Makefile. There is no linter configured.
|
||||||
- `internal/config` — TOML-based configuration with env var fallbacks. `Load(path)` reads `config.toml`; env vars override zero-value fields. See `config.example.toml` for all settings.
|
- `internal/config` — TOML-based configuration with env var fallbacks. `Load(path)` reads `config.toml`; env vars override zero-value fields. See `config.example.toml` for all settings.
|
||||||
- `internal/engines` — `Engine` interface and all 9 Go-native implementations. `factory.go` registers engines via `NewDefaultPortedEngines()`. `planner.go` routes engines to local or upstream based on `LOCAL_PORTED_ENGINES` env var.
|
- `internal/engines` — `Engine` interface and all 9 Go-native implementations. `factory.go` registers engines via `NewDefaultPortedEngines()`. `planner.go` routes engines to local or upstream based on `LOCAL_PORTED_ENGINES` env var.
|
||||||
- `internal/search` — `Service` orchestrates the pipeline: cache check, planning, parallel engine execution via goroutines/WaitGroup, upstream proxying, response merging. Individual engine failures are reported as `unresponsive_engines` rather than aborting the search. Qwant has fallback logic to upstream on empty results.
|
- `internal/search` — `Service` orchestrates the pipeline: cache check, planning, parallel engine execution via goroutines/WaitGroup, upstream proxying, response merging. Individual engine failures are reported as `unresponsive_engines` rather than aborting the search. Qwant has fallback logic to upstream on empty results.
|
||||||
- `internal/autocomplete` — Fetches search suggestions. Proxies to upstream `/autocompleter` if configured, falls back to Wikipedia OpenSearch API otherwise.
|
- `internal/autocomplete` — Fetches search suggestions. Proxies to upstream SearXNG `/autocompleter` if configured, falls back to Wikipedia OpenSearch API otherwise.
|
||||||
- `internal/httpapi` — HTTP handlers for `/`, `/search`, `/autocompleter`, `/healthz`, `/opensearch.xml`. Detects HTMX requests via `HX-Request` header to return fragments instead of full pages.
|
- `internal/httpapi` — HTTP handlers for `/`, `/search`, `/autocompleter`, `/healthz`, `/opensearch.xml`. Detects HTMX requests via `HX-Request` header to return fragments instead of full pages.
|
||||||
- `internal/upstream` — Client that proxies requests to an upstream metasearch instance via POST.
|
- `internal/upstream` — Client that proxies requests to an upstream SearXNG instance via POST.
|
||||||
- `internal/cache` — Valkey/Redis-backed cache with SHA-256 cache keys. No-op if unconfigured.
|
- `internal/cache` — Valkey/Redis-backed cache with SHA-256 cache keys. No-op if unconfigured.
|
||||||
- `internal/middleware` — Three rate limiters (per-IP sliding window, burst+sustained, global) and CORS. All disabled by default.
|
- `internal/middleware` — Three rate limiters (per-IP sliding window, burst+sustained, global) and CORS. All disabled by default.
|
||||||
- `internal/views` — HTML templates and static files embedded via `//go:embed`. Renders full pages or HTMX fragments. Templates: `base.html`, `index.html`, `results.html`, `results_inner.html`, `result_item.html`.
|
- `internal/views` — HTML templates and static files embedded via `//go:embed`. Renders full pages or HTMX fragments. Templates: `base.html`, `index.html`, `results.html`, `results_inner.html`, `result_item.html`.
|
||||||
- `cmd/samsa` — Entry point. Loads TOML config, seeds env vars for engine code, wires up middleware chain, starts HTTP server.
|
- `cmd/searxng-go` — Entry point. Loads TOML config, seeds env vars for engine code, wires up middleware chain, starts HTTP server.
|
||||||
|
|
||||||
**Engine interface** (`internal/engines/engine.go`):
|
**Engine interface** (`internal/engines/engine.go`):
|
||||||
```go
|
```go
|
||||||
|
|
@ -66,7 +66,7 @@ Config is loaded from `config.toml` (see `config.example.toml`). All fields can
|
||||||
|
|
||||||
## Conventions
|
## Conventions
|
||||||
|
|
||||||
- Module path: `github.com/metamorphosis-dev/samsa`
|
- Module path: `github.com/metamorphosis-dev/kafka`
|
||||||
- Tests use shared mock helpers in `internal/engines/http_mock_test.go` (`roundTripperFunc`, `httpResponse`)
|
- Tests use shared mock helpers in `internal/engines/http_mock_test.go` (`roundTripperFunc`, `httpResponse`)
|
||||||
- Engine implementations are single files under `internal/engines/` (e.g., `wikipedia.go`, `duckduckgo.go`)
|
- Engine implementations are single files under `internal/engines/` (e.g., `wikipedia.go`, `duckduckgo.go`)
|
||||||
- Response merging de-duplicates by `engine|title|url` key; suggestions/corrections are merged as sets
|
- Response merging de-duplicates by `engine|title|url` key; suggestions/corrections are merged as sets
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ RUN go mod download
|
||||||
|
|
||||||
# Copy source and build
|
# Copy source and build
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /kafka ./cmd/kafka
|
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /kafka ./cmd/searxng-go
|
||||||
|
|
||||||
# Runtime stage
|
# Runtime stage
|
||||||
FROM alpine:3.21
|
FROM alpine:3.21
|
||||||
|
|
@ -21,7 +21,7 @@ RUN apk add --no-cache ca-certificates tzdata
|
||||||
COPY --from=builder /kafka /usr/local/bin/kafka
|
COPY --from=builder /kafka /usr/local/bin/kafka
|
||||||
COPY config.example.toml /etc/kafka/config.example.toml
|
COPY config.example.toml /etc/kafka/config.example.toml
|
||||||
|
|
||||||
EXPOSE 5355
|
EXPOSE 8080
|
||||||
|
|
||||||
ENTRYPOINT ["kafka"]
|
ENTRYPOINT ["kafka"]
|
||||||
CMD ["-config", "/etc/kafka/config.toml"]
|
CMD ["-config", "/etc/kafka/config.toml"]
|
||||||
|
|
|
||||||
683
LICENSE
683
LICENSE
|
|
@ -1,662 +1,21 @@
|
||||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
MIT License
|
||||||
Version 3, 19 November 2007
|
|
||||||
|
Copyright (c) 2026-present metamorphosis-dev
|
||||||
Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
Everyone is permitted to copy and distribute verbatim copies
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
of this license document, but changing it is not allowed.
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
Preamble
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
The GNU Affero General Public License is a free, copyleft license for
|
|
||||||
software and other kinds of works, specifically designed to ensure
|
The above copyright notice and this permission notice shall be included in all
|
||||||
cooperation with the community in the case of network server software.
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
The licenses for most software and other practical works are designed
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
to take away your freedom to share and change the works. By contrast,
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
our General Public Licenses are intended to guarantee your freedom to
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
share and change all versions of a program--to make sure it remains free
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
software for all its users.
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
When we speak of free software, we are referring to freedom, not
|
SOFTWARE.
|
||||||
price. Our General Public Licenses are designed to make sure that you
|
|
||||||
have the freedom to distribute copies of free software (and charge for
|
|
||||||
them if you wish), that you receive source code or can get it if you
|
|
||||||
want it, that you can change the software or use pieces of it in new
|
|
||||||
free programs, and that you know you can do these things.
|
|
||||||
|
|
||||||
Developers that use our General Public Licenses protect your rights
|
|
||||||
with two steps: (1) assert copyright on the software, and (2) offer
|
|
||||||
you this License which gives you legal permission to copy, distribute
|
|
||||||
and/or modify the software.
|
|
||||||
|
|
||||||
A secondary benefit of defending all users' freedom is that
|
|
||||||
improvements made in alternate versions of the program, if they
|
|
||||||
receive widespread use, become available for other developers to
|
|
||||||
incorporate. Many developers of free software are heartened and
|
|
||||||
encouraged by the resulting cooperation. However, in the case of
|
|
||||||
software used on network servers, this result may fail to come about.
|
|
||||||
The GNU General Public License permits making a modified version and
|
|
||||||
letting the public access it on a server without ever releasing its
|
|
||||||
source code to the public.
|
|
||||||
|
|
||||||
The GNU Affero General Public License is designed specifically to
|
|
||||||
ensure that, in such cases, the modified source code becomes available
|
|
||||||
to the community. It requires the operator of a network server to
|
|
||||||
provide the source code of the modified version running there to the
|
|
||||||
users of that server. Therefore, public use of a modified version, on
|
|
||||||
a publicly accessible server, gives the public access to the source
|
|
||||||
code of the modified version.
|
|
||||||
|
|
||||||
An older license, called the Affero General Public License and
|
|
||||||
published by Affero, was designed to accomplish similar goals. This is
|
|
||||||
a different license, not a version of the Affero GPL, but Affero has
|
|
||||||
released a new version of the Affero GPL which permits relicensing under
|
|
||||||
this license.
|
|
||||||
|
|
||||||
The precise terms and conditions for copying, distribution and
|
|
||||||
modification follow.
|
|
||||||
|
|
||||||
TERMS AND CONDITIONS
|
|
||||||
|
|
||||||
0. Definitions.
|
|
||||||
|
|
||||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
|
||||||
|
|
||||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
|
||||||
works, such as semiconductor masks.
|
|
||||||
|
|
||||||
"The Program" refers to any copyrightable work licensed under this
|
|
||||||
License. Each licensee is addressed as "you". "Licensees" and
|
|
||||||
"recipients" may be individuals or organizations.
|
|
||||||
|
|
||||||
To "modify" a work means to copy from or adapt all or part of the work
|
|
||||||
in a fashion requiring copyright permission, other than the making of an
|
|
||||||
exact copy. The resulting work is called a "modified version" of the
|
|
||||||
earlier work or a work "based on" the earlier work.
|
|
||||||
|
|
||||||
A "covered work" means either the unmodified Program or a work based
|
|
||||||
on the Program.
|
|
||||||
|
|
||||||
To "propagate" a work means to do anything with it that, without
|
|
||||||
permission, would make you directly or secondarily liable for
|
|
||||||
infringement under applicable copyright law, except executing it on a
|
|
||||||
computer or modifying a private copy. Propagation includes copying,
|
|
||||||
distribution (with or without modification), making available to the
|
|
||||||
public, and in some countries other activities as well.
|
|
||||||
|
|
||||||
To "convey" a work means any kind of propagation that enables other
|
|
||||||
parties to make or receive copies. Mere interaction with a user through
|
|
||||||
a computer network, with no transfer of a copy, is not conveying.
|
|
||||||
|
|
||||||
An interactive user interface displays "Appropriate Legal Notices"
|
|
||||||
to the extent that it includes a convenient and prominently visible
|
|
||||||
feature that (1) displays an appropriate copyright notice, and (2)
|
|
||||||
tells the user that there is no warranty for the work (except to the
|
|
||||||
extent that warranties are provided), that licensees may convey the
|
|
||||||
work under this License, and how to view a copy of this License. If
|
|
||||||
the interface presents a list of user commands or options, such as a
|
|
||||||
menu, a prominent item in the list meets this criterion.
|
|
||||||
|
|
||||||
1. Source Code.
|
|
||||||
|
|
||||||
The "source code" for a work means the preferred form of the work
|
|
||||||
for making modifications to it. "Object code" means any non-source
|
|
||||||
form of a work.
|
|
||||||
|
|
||||||
A "Standard Interface" means an interface that either is an official
|
|
||||||
standard defined by a recognized standards body or, in the case of
|
|
||||||
interfaces specified for a particular programming language, one that
|
|
||||||
is widely used among developers working in that language.
|
|
||||||
|
|
||||||
The "System Libraries" of an executable work include anything, other
|
|
||||||
than the work as a whole, that (a) is included in the normal form of
|
|
||||||
packaging a Major Component, but which is not part of that Major
|
|
||||||
Component, and (b) serves only to enable use of the work with that
|
|
||||||
Major Component, or to implement a Standard Interface for which an
|
|
||||||
implementation is available to the public in source code form. A
|
|
||||||
"Major Component", in this context, means a major essential component
|
|
||||||
(kernel, window system, and so on) of the specific operating system
|
|
||||||
(if any) on which the executable work runs, or a compiler used to
|
|
||||||
produce the work, or an object code interpreter used to run it.
|
|
||||||
|
|
||||||
The "Corresponding Source" for a work in object code form means all
|
|
||||||
the source code needed to generate, install, and (for an executable
|
|
||||||
work) run the object code and to modify the work, including scripts to
|
|
||||||
control those activities. However, it does not include the work's
|
|
||||||
System Libraries, or general-purpose tools or generally available free
|
|
||||||
programs which are used unmodified in performing those activities but
|
|
||||||
which are not part of the work. For example, Corresponding Source
|
|
||||||
includes interface definition files associated with source files for
|
|
||||||
the work, and the source code for shared libraries and dynamically
|
|
||||||
linked subprograms that the work is specifically designed to require,
|
|
||||||
such as by intimate data communication or control flow between those
|
|
||||||
subprograms and other parts of the work.
|
|
||||||
|
|
||||||
The Corresponding Source need not include anything that users
|
|
||||||
can regenerate automatically from other parts of the Corresponding
|
|
||||||
Source.
|
|
||||||
|
|
||||||
The Corresponding Source for a work in source code form is that
|
|
||||||
same work.
|
|
||||||
|
|
||||||
2. Basic Permissions.
|
|
||||||
|
|
||||||
All rights granted under this License are granted for the term of
|
|
||||||
copyright on the Program, and are irrevocable provided the stated
|
|
||||||
conditions are met. This License explicitly affirms your unlimited
|
|
||||||
permission to run the unmodified Program. The output from running a
|
|
||||||
covered work is covered by this License only if the output, given its
|
|
||||||
content, constitutes a covered work. This License acknowledges your
|
|
||||||
rights of fair use or other equivalent, as provided by copyright law.
|
|
||||||
|
|
||||||
You may make, run and propagate covered works that you do not
|
|
||||||
convey, without conditions so long as your license otherwise remains
|
|
||||||
in force. You may convey covered works to others for the sole purpose
|
|
||||||
of having them make modifications exclusively for you, or provide you
|
|
||||||
with facilities for running those works, provided that you comply with
|
|
||||||
the terms of this License in conveying all material for which you do
|
|
||||||
not control copyright. Those thus making or running the covered works
|
|
||||||
for you must do so exclusively on your behalf, under your direction
|
|
||||||
and control, on terms that prohibit them from making any copies of
|
|
||||||
your copyrighted material outside their relationship with you.
|
|
||||||
|
|
||||||
Conveying under any other circumstances is permitted solely under
|
|
||||||
the conditions stated below. Sublicensing is not allowed; section 10
|
|
||||||
makes it unnecessary.
|
|
||||||
|
|
||||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
|
||||||
|
|
||||||
No covered work shall be deemed part of an effective technological
|
|
||||||
measure under any applicable law fulfilling obligations under article
|
|
||||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
|
||||||
similar laws prohibiting or restricting circumvention of such
|
|
||||||
measures.
|
|
||||||
|
|
||||||
When you convey a covered work, you waive any legal power to forbid
|
|
||||||
circumvention of technological measures to the extent such circumvention
|
|
||||||
is effected by exercising rights under this License with respect to
|
|
||||||
the covered work, and you disclaim any intention to limit operation or
|
|
||||||
modification of the work as a means of enforcing, against the work's
|
|
||||||
users, your or third parties' legal rights to forbid circumvention of
|
|
||||||
technological measures.
|
|
||||||
|
|
||||||
4. Conveying Verbatim Copies.
|
|
||||||
|
|
||||||
You may convey verbatim copies of the Program's source code as you
|
|
||||||
receive it, in any medium, provided that you conspicuously and
|
|
||||||
appropriately publish on each copy an appropriate copyright notice;
|
|
||||||
keep intact all notices stating that this License and any
|
|
||||||
non-permissive terms added in accord with section 7 apply to the code;
|
|
||||||
keep intact all notices of the absence of any warranty; and give all
|
|
||||||
recipients a copy of this License along with the Program.
|
|
||||||
|
|
||||||
You may charge any price or no price for each copy that you convey,
|
|
||||||
and you may offer support or warranty protection for a fee.
|
|
||||||
|
|
||||||
5. Conveying Modified Source Versions.
|
|
||||||
|
|
||||||
You may convey a work based on the Program, or the modifications to
|
|
||||||
produce it from the Program, in the form of source code under the
|
|
||||||
terms of section 4, provided that you also meet all of these conditions:
|
|
||||||
|
|
||||||
a) The work must carry prominent notices stating that you modified
|
|
||||||
it, and giving a relevant date.
|
|
||||||
|
|
||||||
b) The work must carry prominent notices stating that it is
|
|
||||||
released under this License and any conditions added under section
|
|
||||||
7. This requirement modifies the requirement in section 4 to
|
|
||||||
"keep intact all notices".
|
|
||||||
|
|
||||||
c) You must license the entire work, as a whole, under this
|
|
||||||
License to anyone who comes into possession of a copy. This
|
|
||||||
License will therefore apply, along with any applicable section 7
|
|
||||||
additional terms, to the whole of the work, and all its parts,
|
|
||||||
regardless of how they are packaged. This License gives no
|
|
||||||
permission to license the work in any other way, but it does not
|
|
||||||
invalidate such permission if you have separately received it.
|
|
||||||
|
|
||||||
d) If the work has interactive user interfaces, each must display
|
|
||||||
Appropriate Legal Notices; however, if the Program has interactive
|
|
||||||
interfaces that do not display Appropriate Legal Notices, your
|
|
||||||
work need not make them do so.
|
|
||||||
|
|
||||||
A compilation of a covered work with other separate and independent
|
|
||||||
works, which are not by their nature extensions of the covered work,
|
|
||||||
and which are not combined with it such as to form a larger program,
|
|
||||||
in or on a volume of a storage or distribution medium, is called an
|
|
||||||
"aggregate" if the compilation and its resulting copyright are not
|
|
||||||
used to limit the access or legal rights of the compilation's users
|
|
||||||
beyond what the individual works permit. Inclusion of a covered work
|
|
||||||
in an aggregate does not cause this License to apply to the other
|
|
||||||
parts of the aggregate.
|
|
||||||
|
|
||||||
6. Conveying Non-Source Forms.
|
|
||||||
|
|
||||||
You may convey a covered work in object code form under the terms
|
|
||||||
of sections 4 and 5, provided that you also convey the
|
|
||||||
machine-readable Corresponding Source under the terms of this License,
|
|
||||||
in one of these ways:
|
|
||||||
|
|
||||||
a) Convey the object code in, or embodied in, a physical product
|
|
||||||
(including a physical distribution medium), accompanied by the
|
|
||||||
Corresponding Source fixed on a durable physical medium
|
|
||||||
customarily used for software interchange.
|
|
||||||
|
|
||||||
b) Convey the object code in, or embodied in, a physical product
|
|
||||||
(including a physical distribution medium), accompanied by a
|
|
||||||
written offer, valid for at least three years and valid for as
|
|
||||||
long as you offer spare parts or customer support for that product
|
|
||||||
model, to give anyone who possesses the object code either (1) a
|
|
||||||
copy of the Corresponding Source for all the software in the
|
|
||||||
product that is covered by this License, on a durable physical
|
|
||||||
medium customarily used for software interchange, for a price no
|
|
||||||
more than your reasonable cost of physically performing this
|
|
||||||
conveying of source, or (2) access to copy the Corresponding
|
|
||||||
Source from a network server at no charge.
|
|
||||||
|
|
||||||
c) Convey individual copies of the object code with a copy of the
|
|
||||||
written offer to provide the Corresponding Source. This
|
|
||||||
alternative is allowed only occasionally and noncommercially, and
|
|
||||||
only if you received the object code with such an offer, in accord
|
|
||||||
with subsection 6b.
|
|
||||||
|
|
||||||
d) Convey the object code by offering access from a designated
|
|
||||||
place (gratis or for a charge), and offer equivalent access to the
|
|
||||||
Corresponding Source in the same way through the same place at no
|
|
||||||
further charge. You need not require recipients to copy the
|
|
||||||
Corresponding Source along with the object code. If the place to
|
|
||||||
copy the object code is a network server, the Corresponding Source
|
|
||||||
may be on a different server (operated by you or a third party)
|
|
||||||
that supports equivalent copying facilities, provided you maintain
|
|
||||||
clear directions next to the object code saying where to find the
|
|
||||||
Corresponding Source. Regardless of what server hosts the
|
|
||||||
Corresponding Source, you remain obligated to ensure that it is
|
|
||||||
available for as long as needed to satisfy these requirements.
|
|
||||||
|
|
||||||
e) Convey the object code using peer-to-peer transmission, provided
|
|
||||||
you inform other peers where the object code and Corresponding
|
|
||||||
Source of the work are being offered to the general public at no
|
|
||||||
charge under subsection 6d.
|
|
||||||
|
|
||||||
A separable portion of the object code, whose source code is excluded
|
|
||||||
from the Corresponding Source as a System Library, need not be
|
|
||||||
included in conveying the object code work.
|
|
||||||
|
|
||||||
A "User Product" is either (1) a "consumer product", which means any
|
|
||||||
tangible personal property which is normally used for personal, family,
|
|
||||||
or household purposes, or (2) anything designed or sold for incorporation
|
|
||||||
into a dwelling. In determining whether a product is a consumer product,
|
|
||||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
|
||||||
product received by a particular user, "normally used" refers to a
|
|
||||||
typical or common use of that class of product, regardless of the status
|
|
||||||
of the particular user or of the way in which the particular user
|
|
||||||
actually uses, or expects or is expected to use, the product. A product
|
|
||||||
is a consumer product regardless of whether the product has substantial
|
|
||||||
commercial, industrial or non-consumer uses, unless such uses represent
|
|
||||||
the only significant mode of use of the product.
|
|
||||||
|
|
||||||
"Installation Information" for a User Product means any methods,
|
|
||||||
procedures, authorization keys, or other information required to install
|
|
||||||
and execute modified versions of a covered work in that User Product from
|
|
||||||
a modified version of its Corresponding Source. The information must
|
|
||||||
suffice to ensure that the continued functioning of the modified object
|
|
||||||
code is in no case prevented or interfered with solely because
|
|
||||||
modification has been made.
|
|
||||||
|
|
||||||
If you convey an object code work under this section in, or with, or
|
|
||||||
specifically for use in, a User Product, and the conveying occurs as
|
|
||||||
part of a transaction in which the right of possession and use of the
|
|
||||||
User Product is transferred to the recipient in perpetuity or for a
|
|
||||||
fixed term (regardless of how the transaction is characterized), the
|
|
||||||
Corresponding Source conveyed under this section must be accompanied
|
|
||||||
by the Installation Information. But this requirement does not apply
|
|
||||||
if neither you nor any third party retains the ability to install
|
|
||||||
modified object code on the User Product (for example, the work has
|
|
||||||
been installed in ROM).
|
|
||||||
|
|
||||||
The requirement to provide Installation Information does not include a
|
|
||||||
requirement to continue to provide support service, warranty, or updates
|
|
||||||
for a work that has been modified or installed by the recipient, or for
|
|
||||||
the User Product in which it has been modified or installed. Access to a
|
|
||||||
network may be denied when the modification itself materially and
|
|
||||||
adversely affects the operation of the network or violates the rules and
|
|
||||||
protocols for communication across the network.
|
|
||||||
|
|
||||||
Corresponding Source conveyed, and Installation Information provided,
|
|
||||||
in accord with this section must be in a format that is publicly
|
|
||||||
documented (and with an implementation available to the public in
|
|
||||||
source code form), and must require no special password or key for
|
|
||||||
unpacking, reading or copying.
|
|
||||||
|
|
||||||
7. Additional Terms.
|
|
||||||
|
|
||||||
"Additional permissions" are terms that supplement the terms of this
|
|
||||||
License by making exceptions from one or more of its conditions.
|
|
||||||
Additional permissions that are applicable to the entire Program shall
|
|
||||||
be treated as though they were included in this License, to the extent
|
|
||||||
that they are valid under applicable law. If additional permissions
|
|
||||||
apply only to part of the Program, that part may be used separately
|
|
||||||
under those permissions, but the entire Program remains governed by
|
|
||||||
this License without regard to the additional permissions.
|
|
||||||
|
|
||||||
When you convey a copy of a covered work, you may at your option
|
|
||||||
remove any additional permissions from that copy, or from any part of
|
|
||||||
it. (Additional permissions may be written to require their own
|
|
||||||
removal in certain cases when you modify the work.) You may place
|
|
||||||
additional permissions on material, added by you to a covered work,
|
|
||||||
for which you have or can give appropriate copyright permission.
|
|
||||||
|
|
||||||
Notwithstanding any other provision of this License, for material you
|
|
||||||
add to a covered work, you may (if authorized by the copyright holders of
|
|
||||||
that material) supplement the terms of this License with terms:
|
|
||||||
|
|
||||||
a) Disclaiming warranty or limiting liability differently from the
|
|
||||||
terms of sections 15 and 16 of this License; or
|
|
||||||
|
|
||||||
b) Requiring preservation of specified reasonable legal notices or
|
|
||||||
author attributions in that material or in the Appropriate Legal
|
|
||||||
Notices displayed by works containing it; or
|
|
||||||
|
|
||||||
c) Prohibiting misrepresentation of the origin of that material, or
|
|
||||||
requiring that modified versions of such material be marked in
|
|
||||||
reasonable ways as different from the original version; or
|
|
||||||
|
|
||||||
d) Limiting the use for publicity purposes of names of licensors or
|
|
||||||
authors of the material; or
|
|
||||||
|
|
||||||
e) Declining to grant rights under trademark law for use of some
|
|
||||||
trade names, trademarks, or service marks; or
|
|
||||||
|
|
||||||
f) Requiring indemnification of licensors and authors of that
|
|
||||||
material by anyone who conveys the material (or modified versions of
|
|
||||||
it) with contractual assumptions of liability to the recipient, for
|
|
||||||
any liability that these contractual assumptions directly impose on
|
|
||||||
those licensors and authors.
|
|
||||||
|
|
||||||
All other non-permissive additional terms are considered "further
|
|
||||||
restrictions" within the meaning of section 10. If the Program as you
|
|
||||||
received it, or any part of it, contains a notice stating that it is
|
|
||||||
governed by this License along with a term that is a further
|
|
||||||
restriction, you may remove that term. If a license document contains
|
|
||||||
a further restriction but permits relicensing or conveying under this
|
|
||||||
License, you may add to a covered work material governed by the terms
|
|
||||||
of that license document, provided that the further restriction does
|
|
||||||
not survive such relicensing or conveying.
|
|
||||||
|
|
||||||
If you add terms to a covered work in accord with this section, you
|
|
||||||
must place, in the relevant source files, a statement of the
|
|
||||||
additional terms that apply to those files, or a notice indicating
|
|
||||||
where to find the applicable terms.
|
|
||||||
|
|
||||||
Additional terms, permissive or non-permissive, may be stated in the
|
|
||||||
form of a separately written license, or stated as exceptions;
|
|
||||||
the above requirements apply either way.
|
|
||||||
|
|
||||||
8. Termination.
|
|
||||||
|
|
||||||
You may not propagate or modify a covered work except as expressly
|
|
||||||
provided under this License. Any attempt otherwise to propagate or
|
|
||||||
modify it is void, and will automatically terminate your rights under
|
|
||||||
this License (including any patent licenses granted under the third
|
|
||||||
paragraph of section 11).
|
|
||||||
|
|
||||||
However, if you cease all violation of this License, then your
|
|
||||||
license from a particular copyright holder is reinstated (a)
|
|
||||||
provisionally, unless and until the copyright holder explicitly and
|
|
||||||
finally terminates your license, and (b) permanently, if the copyright
|
|
||||||
holder fails to notify you of the violation by some reasonable means
|
|
||||||
prior to 60 days after the cessation.
|
|
||||||
|
|
||||||
Moreover, your license from a particular copyright holder is
|
|
||||||
reinstated permanently if the copyright holder notifies you of the
|
|
||||||
violation by some reasonable means, this is the first time you have
|
|
||||||
received notice of violation of this License (for any work) from that
|
|
||||||
copyright holder, and you cure the violation prior to 30 days after
|
|
||||||
your receipt of the notice.
|
|
||||||
|
|
||||||
Termination of your rights under this section does not terminate the
|
|
||||||
licenses of parties who have received copies or rights from you under
|
|
||||||
this License. If your rights have been terminated and not permanently
|
|
||||||
reinstated, you do not qualify to receive new licenses for the same
|
|
||||||
material under section 10.
|
|
||||||
|
|
||||||
9. Acceptance Not Required for Having Copies.
|
|
||||||
|
|
||||||
You are not required to accept this License in order to receive or
|
|
||||||
run a copy of the Program. Ancillary propagation of a covered work
|
|
||||||
occurring solely as a consequence of using peer-to-peer transmission
|
|
||||||
to receive a copy likewise does not require acceptance. However,
|
|
||||||
nothing other than this License grants you permission to propagate or
|
|
||||||
modify any covered work. These actions infringe copyright if you do
|
|
||||||
not accept this License. Therefore, by modifying or propagating a
|
|
||||||
covered work, you indicate your acceptance of this License to do so.
|
|
||||||
|
|
||||||
10. Automatic Licensing of Downstream Recipients.
|
|
||||||
|
|
||||||
Each time you convey a covered work, the recipient automatically
|
|
||||||
receives a license from the original licensors, to run, modify and
|
|
||||||
propagate that work, subject to this License. You are not responsible
|
|
||||||
for enforcing compliance by third parties with this License.
|
|
||||||
|
|
||||||
An "entity transaction" is a transaction transferring control of an
|
|
||||||
organization, or substantially all assets of one, or subdividing an
|
|
||||||
organization, or merging organizations. If propagation of a covered
|
|
||||||
work results from an entity transaction, each party to that
|
|
||||||
transaction who receives a copy of the work also receives whatever
|
|
||||||
licenses to the work the party's predecessor in interest had or could
|
|
||||||
give under the previous paragraph, plus a right to possession of the
|
|
||||||
Corresponding Source of the work from the predecessor in interest, if
|
|
||||||
the predecessor has it or can get it with reasonable efforts.
|
|
||||||
|
|
||||||
You may not impose any further restrictions on the exercise of the
|
|
||||||
rights granted or affirmed under this License. For example, you may
|
|
||||||
not impose a license fee, royalty, or other charge for exercise of
|
|
||||||
rights granted under this License, and you may not initiate litigation
|
|
||||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
|
||||||
any patent claim is infringed by making, using, selling, offering for
|
|
||||||
sale, or importing the Program or any portion of it.
|
|
||||||
|
|
||||||
11. Patents.
|
|
||||||
|
|
||||||
A "contributor" is a copyright holder who authorizes use under this
|
|
||||||
License of the Program or a work on which the Program is based. The
|
|
||||||
work thus licensed is called the contributor's "contributor version".
|
|
||||||
|
|
||||||
A contributor's "essential patent claims" are all patent claims
|
|
||||||
owned or controlled by the contributor, whether already acquired or
|
|
||||||
hereafter acquired, that would be infringed by some manner, permitted
|
|
||||||
by this License, of making, using, or selling its contributor version,
|
|
||||||
but do not include claims that would be infringed only as a
|
|
||||||
consequence of further modification of the contributor version. For
|
|
||||||
purposes of this definition, "control" includes the right to grant
|
|
||||||
patent sublicenses in a manner consistent with the requirements of
|
|
||||||
this License.
|
|
||||||
|
|
||||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
|
||||||
patent license under the contributor's essential patent claims, to
|
|
||||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
|
||||||
propagate the contents of its contributor version.
|
|
||||||
|
|
||||||
In the following three paragraphs, a "patent license" is any express
|
|
||||||
agreement or commitment, however denominated, not to enforce a patent
|
|
||||||
(such as an express permission to practice a patent or covenant not to
|
|
||||||
sue for patent infringement). To "grant" such a patent license to a
|
|
||||||
party means to make such an agreement or commitment not to enforce a
|
|
||||||
patent against the party.
|
|
||||||
|
|
||||||
If you convey a covered work, knowingly relying on a patent license,
|
|
||||||
and the Corresponding Source of the work is not available for anyone
|
|
||||||
to copy, free of charge and under the terms of this License, through a
|
|
||||||
publicly available network server or other readily accessible means,
|
|
||||||
then you must either (1) cause the Corresponding Source to be so
|
|
||||||
available, or (2) arrange to deprive yourself of the benefit of the
|
|
||||||
patent license for this particular work, or (3) arrange, in a manner
|
|
||||||
consistent with the requirements of this License, to extend the patent
|
|
||||||
license to downstream recipients. "Knowingly relying" means you have
|
|
||||||
actual knowledge that, but for the patent license, your conveying the
|
|
||||||
covered work in a country, or your recipient's use of the covered work
|
|
||||||
in a country, would infringe one or more identifiable patents in that
|
|
||||||
country that you have reason to believe are valid.
|
|
||||||
|
|
||||||
If, pursuant to or in connection with a single transaction or
|
|
||||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
|
||||||
covered work, and grant a patent license to some of the parties
|
|
||||||
receiving the covered work authorizing them to use, propagate, modify
|
|
||||||
or convey a specific copy of the covered work, then the patent license
|
|
||||||
you grant is automatically extended to all recipients of the covered
|
|
||||||
work and works based on it.
|
|
||||||
|
|
||||||
A patent license is "discriminatory" if it does not include within
|
|
||||||
the scope of its coverage, prohibits the exercise of, or is
|
|
||||||
conditioned on the non-exercise of one or more of the rights that are
|
|
||||||
specifically granted under this License. You may not convey a covered
|
|
||||||
work if you are a party to an arrangement with a third party that is
|
|
||||||
in the business of distributing software, under which you make payment
|
|
||||||
to the third party based on the extent of your activity of conveying
|
|
||||||
the work, and under which the third party grants, to any of the
|
|
||||||
parties who would receive the covered work from you, a discriminatory
|
|
||||||
patent license (a) in connection with copies of the covered work
|
|
||||||
conveyed by you (or copies made from those copies), or (b) primarily
|
|
||||||
for and in connection with specific products or compilations that
|
|
||||||
contain the covered work, unless you entered into that arrangement,
|
|
||||||
or that patent license was granted, prior to 28 March 2007.
|
|
||||||
|
|
||||||
Nothing in this License shall be construed as excluding or limiting
|
|
||||||
any implied license or other defenses to infringement that may
|
|
||||||
otherwise be available to you under applicable patent law.
|
|
||||||
|
|
||||||
12. No Surrender of Others' Freedom.
|
|
||||||
|
|
||||||
If conditions are imposed on you (whether by court order, agreement or
|
|
||||||
otherwise) that contradict the conditions of this License, they do not
|
|
||||||
excuse you from the conditions of this License. If you cannot convey a
|
|
||||||
covered work so as to satisfy simultaneously your obligations under this
|
|
||||||
License and any other pertinent obligations, then as a consequence you may
|
|
||||||
not convey it at all. For example, if you agree to terms that obligate you
|
|
||||||
to collect a royalty for further conveying from those to whom you convey
|
|
||||||
the Program, the only way you could satisfy both those terms and this
|
|
||||||
License would be to refrain entirely from conveying the Program.
|
|
||||||
|
|
||||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
|
||||||
|
|
||||||
Notwithstanding any other provision of this License, if you modify the
|
|
||||||
Program, your modified version must prominently offer all users
|
|
||||||
interacting with it remotely through a computer network (if your version
|
|
||||||
supports such interaction) an opportunity to receive the Corresponding
|
|
||||||
Source of your version by providing access to the Corresponding Source
|
|
||||||
from a network server at no charge, through some standard or customary
|
|
||||||
means of facilitating copying of software. This Corresponding Source
|
|
||||||
shall include the Corresponding Source for any work covered by version 3
|
|
||||||
of the GNU General Public License that is incorporated pursuant to the
|
|
||||||
following paragraph.
|
|
||||||
|
|
||||||
Notwithstanding any other provision of this License, you have
|
|
||||||
permission to link or combine any covered work with a work licensed
|
|
||||||
under version 3 of the GNU General Public License into a single
|
|
||||||
combined work, and to convey the resulting work. The terms of this
|
|
||||||
License will continue to apply to the part which is the covered work,
|
|
||||||
but the work with which it is combined will remain governed by version
|
|
||||||
3 of the GNU General Public License.
|
|
||||||
|
|
||||||
14. Revised Versions of this License.
|
|
||||||
|
|
||||||
The Free Software Foundation may publish revised and/or new versions of
|
|
||||||
the GNU Affero General Public License from time to time. Such new versions
|
|
||||||
will be similar in spirit to the present version, but may differ in detail to
|
|
||||||
address new problems or concerns.
|
|
||||||
|
|
||||||
Each version is given a distinguishing version number. If the
|
|
||||||
Program specifies that a certain numbered version of the GNU Affero General
|
|
||||||
Public License "or any later version" applies to it, you have the
|
|
||||||
option of following the terms and conditions either of that numbered
|
|
||||||
version or of any later version published by the Free Software
|
|
||||||
Foundation. If the Program does not specify a version number of the
|
|
||||||
GNU Affero General Public License, you may choose any version ever published
|
|
||||||
by the Free Software Foundation.
|
|
||||||
|
|
||||||
If the Program specifies that a proxy can decide which future
|
|
||||||
versions of the GNU Affero General Public License can be used, that proxy's
|
|
||||||
public statement of acceptance of a version permanently authorizes you
|
|
||||||
to choose that version for the Program.
|
|
||||||
|
|
||||||
Later license versions may give you additional or different
|
|
||||||
permissions. However, no additional obligations are imposed on any
|
|
||||||
author or copyright holder as a result of your choosing to follow a
|
|
||||||
later version.
|
|
||||||
|
|
||||||
15. Disclaimer of Warranty.
|
|
||||||
|
|
||||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
|
||||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
|
||||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
|
||||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
|
||||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
||||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
|
||||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
|
||||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
|
||||||
|
|
||||||
16. Limitation of Liability.
|
|
||||||
|
|
||||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
|
||||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
|
||||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
|
||||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
|
||||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
|
||||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
|
||||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
|
||||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
|
||||||
SUCH DAMAGES.
|
|
||||||
|
|
||||||
17. Interpretation of Sections 15 and 16.
|
|
||||||
|
|
||||||
If the disclaimer of warranty and limitation of liability provided
|
|
||||||
above cannot be given local legal effect according to their terms,
|
|
||||||
reviewing courts shall apply local law that most closely approximates
|
|
||||||
an absolute waiver of all civil liability in connection with the
|
|
||||||
Program, unless a warranty or assumption of liability accompanies a
|
|
||||||
copy of the Program in return for a fee.
|
|
||||||
|
|
||||||
END OF TERMS AND CONDITIONS
|
|
||||||
|
|
||||||
How to Apply These Terms to Your New Programs
|
|
||||||
|
|
||||||
If you develop a new program, and you want it to be of the greatest
|
|
||||||
possible use to the public, the best way to achieve this is to make it
|
|
||||||
free software which everyone can redistribute and change under these terms.
|
|
||||||
|
|
||||||
To do so, attach the following notices to the program. It is safest
|
|
||||||
to attach them to the start of each source file to most effectively
|
|
||||||
state the exclusion of warranty; and each file should have at least
|
|
||||||
the "copyright" line and a pointer to where the full notice is found.
|
|
||||||
|
|
||||||
kafka — a privacy-respecting metasearch engine
|
|
||||||
Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU Affero General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU Affero General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU Affero General Public License
|
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Also add information on how to contact you by electronic and paper mail.
|
|
||||||
|
|
||||||
If your software can interact with users remotely through a computer
|
|
||||||
network, you should also make sure that it provides a way for users to
|
|
||||||
get its source. For example, if your program is a web application, its
|
|
||||||
interface could display a "Source" link that leads users to an archive
|
|
||||||
of the code. There are many ways you could offer source, and different
|
|
||||||
solutions will be better for different programs; see section 13 for the
|
|
||||||
specific requirements.
|
|
||||||
|
|
||||||
You should also get your employer (if you work as a programmer) or school,
|
|
||||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
|
||||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
|
||||||
<https://www.gnu.org/licenses/>.
|
|
||||||
|
|
|
||||||
97
README.md
97
README.md
|
|
@ -1,23 +1,20 @@
|
||||||
# samsa
|
# kafka
|
||||||
|
|
||||||
*samsa — named for Gregor Samsa, who woke to find himself transformed. You wanted results; you got a metasearch engine.*
|
|
||||||
|
|
||||||
A privacy-respecting, open metasearch engine written in Go. SearXNG-compatible API with an HTML frontend, designed to be fast, lightweight, and deployable anywhere.
|
A privacy-respecting, open metasearch engine written in Go. SearXNG-compatible API with an HTML frontend, designed to be fast, lightweight, and deployable anywhere.
|
||||||
|
|
||||||
**11 engines. No JavaScript required. No tracking. One binary.**
|
**9 engines. No JavaScript. No tracking. One binary.**
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
- **SearXNG-compatible API** — drop-in replacement for existing integrations
|
- **SearXNG-compatible API** — drop-in replacement for existing integrations
|
||||||
- **11 search engines** — Wikipedia, arXiv, Crossref, Brave Search API, Brave (scraping), Qwant, DuckDuckGo, GitHub, Reddit, Bing, Google, YouTube
|
- **9 search engines** — Wikipedia, arXiv, Crossref, Brave, Qwant, DuckDuckGo, GitHub, Reddit, Bing
|
||||||
- **Stack Overflow** — bonus engine, not enabled by default
|
- **HTML frontend** — HTMX + Go templates with instant search, dark mode, responsive design
|
||||||
- **HTML frontend** — Go templates + HTMX with instant search, dark mode, responsive design
|
|
||||||
- **Valkey cache** — optional Redis-compatible caching with configurable TTL
|
- **Valkey cache** — optional Redis-compatible caching with configurable TTL
|
||||||
- **Rate limiting** — three layers: per-IP, burst, and global (all disabled by default)
|
- **Rate limiting** — three layers: per-IP, burst, and global (all disabled by default)
|
||||||
- **CORS** — configurable origins for browser-based clients
|
- **CORS** — configurable origins for browser-based clients
|
||||||
- **OpenSearch** — browsers can add samsa as a search engine from the address bar
|
- **OpenSearch** — browsers can add kafka as a search engine from the address bar
|
||||||
- **Graceful degradation** — individual engine failures don't kill the whole search
|
- **Graceful degradation** — individual engine failures don't kill the whole search
|
||||||
- **Docker** — multi-stage build, static binary, ~20MB runtime image
|
- **Docker** — multi-stage build, ~20MB runtime image
|
||||||
- **NixOS** — native NixOS module with systemd service
|
- **NixOS** — native NixOS module with systemd service
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
@ -25,17 +22,17 @@ A privacy-respecting, open metasearch engine written in Go. SearXNG-compatible A
|
||||||
### Binary
|
### Binary
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://git.ashisgreat.xyz/penal-colony/samsa.git
|
git clone https://git.ashisgreat.xyz/penal-colony/gosearch.git
|
||||||
cd samsa
|
cd kafka
|
||||||
go build ./cmd/samsa
|
go build ./cmd/searxng-go
|
||||||
./samsa -config config.toml
|
./searxng-go -config config.toml
|
||||||
```
|
```
|
||||||
|
|
||||||
### Docker Compose
|
### Docker Compose
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cp config.example.toml config.toml
|
cp config.example.toml config.toml
|
||||||
# Edit config.toml — set your Brave API key, YouTube API key, etc.
|
# Edit config.toml — set your Brave API key, etc.
|
||||||
docker compose up -d
|
docker compose up -d
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -44,28 +41,28 @@ docker compose up -d
|
||||||
Add to your flake inputs:
|
Add to your flake inputs:
|
||||||
|
|
||||||
```nix
|
```nix
|
||||||
inputs.samsa.url = "git+https://git.ashisgreat.xyz/penal-colony/samsa.git";
|
inputs.kafka.url = "git+https://git.ashisgreat.xyz/penal-colony/gosearch.git";
|
||||||
```
|
```
|
||||||
|
|
||||||
Enable in your configuration:
|
Enable in your configuration:
|
||||||
|
|
||||||
```nix
|
```nix
|
||||||
imports = [ inputs.samsa.nixosModules.default ];
|
imports = [ inputs.kafka.nixosModules.default ];
|
||||||
|
|
||||||
services.samsa = {
|
services.kafka = {
|
||||||
enable = true;
|
enable = true;
|
||||||
openFirewall = true;
|
openFirewall = true;
|
||||||
baseUrl = "https://search.example.com";
|
baseUrl = "https://search.example.com";
|
||||||
# config = "/etc/samsa/config.toml"; # default
|
# config = "/etc/kafka/config.toml"; # default
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
Write your config:
|
Write your config:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo mkdir -p /etc/samsa
|
sudo mkdir -p /etc/kafka
|
||||||
sudo cp config.example.toml /etc/samsa/config.toml
|
sudo cp config.example.toml /etc/kafka/config.toml
|
||||||
sudo $EDITOR /etc/samsa/config.toml
|
sudo $EDITOR /etc/kafka/config.toml
|
||||||
```
|
```
|
||||||
|
|
||||||
Deploy:
|
Deploy:
|
||||||
|
|
@ -79,7 +76,7 @@ sudo nixos-rebuild switch --flake .#
|
||||||
```bash
|
```bash
|
||||||
nix develop
|
nix develop
|
||||||
go test ./...
|
go test ./...
|
||||||
go run ./cmd/samsa -config config.toml
|
go run ./cmd/searxng-go -config config.toml
|
||||||
```
|
```
|
||||||
|
|
||||||
## Endpoints
|
## Endpoints
|
||||||
|
|
@ -110,7 +107,7 @@ go run ./cmd/samsa -config config.toml
|
||||||
### Example
|
### Example
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl "http://localhost:5355/search?q=golang&format=json&engines=github,duckduckgo"
|
curl "http://localhost:8080/search?q=golang&format=json&engines=github,duckduckgo"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Response (JSON)
|
### Response (JSON)
|
||||||
|
|
@ -141,10 +138,8 @@ Copy `config.example.toml` to `config.toml` and edit. All settings can also be o
|
||||||
### Key Sections
|
### Key Sections
|
||||||
|
|
||||||
- **`[server]`** — port, timeout, public base URL for OpenSearch
|
- **`[server]`** — port, timeout, public base URL for OpenSearch
|
||||||
- **`[upstream]`** — optional upstream metasearch proxy for unported engines
|
- **`[upstream]`** — optional upstream SearXNG proxy for unported engines
|
||||||
- **`[engines]`** — which engines run locally, engine-specific settings
|
- **`[engines]`** — which engines run locally, engine-specific settings
|
||||||
- **`[engines.brave]`** — Brave Search API key
|
|
||||||
- **`[engines.youtube]`** — YouTube Data API v3 key
|
|
||||||
- **`[cache]`** — Valkey/Redis address, password, TTL
|
- **`[cache]`** — Valkey/Redis address, password, TTL
|
||||||
- **`[cors]`** — allowed origins and methods
|
- **`[cors]`** — allowed origins and methods
|
||||||
- **`[rate_limit]`** — per-IP sliding window (30 req/min default)
|
- **`[rate_limit]`** — per-IP sliding window (30 req/min default)
|
||||||
|
|
@ -155,14 +150,13 @@ Copy `config.example.toml` to `config.toml` and edit. All settings can also be o
|
||||||
|
|
||||||
| Variable | Description |
|
| Variable | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `PORT` | Listen port (default: 5355) |
|
| `PORT` | Listen port (default: 8080) |
|
||||||
| `BASE_URL` | Public URL for OpenSearch XML |
|
| `BASE_URL` | Public URL for OpenSearch XML |
|
||||||
| `UPSTREAM_SEARXNG_URL` | Upstream instance URL |
|
| `UPSTREAM_SEARXNG_URL` | Upstream SearXNG instance URL |
|
||||||
| `LOCAL_PORTED_ENGINES` | Comma-separated local engine list |
|
| `LOCAL_PORTED_ENGINES` | Comma-separated local engine list |
|
||||||
| `HTTP_TIMEOUT` | Upstream request timeout |
|
| `HTTP_TIMEOUT` | Upstream request timeout |
|
||||||
| `BRAVE_API_KEY` | Brave Search API key |
|
| `BRAVE_API_KEY` | Brave Search API key |
|
||||||
| `BRAVE_ACCESS_TOKEN` | Gate requests with token |
|
| `BRAVE_ACCESS_TOKEN` | Gate requests with token |
|
||||||
| `YOUTUBE_API_KEY` | YouTube Data API v3 key |
|
|
||||||
| `VALKEY_ADDRESS` | Valkey/Redis address |
|
| `VALKEY_ADDRESS` | Valkey/Redis address |
|
||||||
| `VALKEY_PASSWORD` | Valkey/Redis password |
|
| `VALKEY_PASSWORD` | Valkey/Redis password |
|
||||||
| `VALKEY_CACHE_TTL` | Cache TTL |
|
| `VALKEY_CACHE_TTL` | Cache TTL |
|
||||||
|
|
@ -176,64 +170,55 @@ See `config.example.toml` for the full list including rate limiting and CORS var
|
||||||
| Wikipedia | MediaWiki API | General knowledge |
|
| Wikipedia | MediaWiki API | General knowledge |
|
||||||
| arXiv | arXiv API | Academic papers |
|
| arXiv | arXiv API | Academic papers |
|
||||||
| Crossref | Crossref API | Academic metadata |
|
| Crossref | Crossref API | Academic metadata |
|
||||||
| Brave Search API | Brave API | General web (requires API key) |
|
| Brave | Brave Search API | General web (requires API key) |
|
||||||
| Brave | Brave Lite HTML | General web (no key needed) |
|
|
||||||
| Qwant | Qwant Lite HTML | General web |
|
| Qwant | Qwant Lite HTML | General web |
|
||||||
| DuckDuckGo | DDG Lite HTML | General web |
|
| DuckDuckGo | DDG Lite HTML | General web |
|
||||||
| GitHub | GitHub Search API v3 | Code and repositories |
|
| GitHub | GitHub Search API v3 | Code and repositories |
|
||||||
| Reddit | Reddit JSON API | Discussions |
|
| Reddit | Reddit JSON API | Discussions |
|
||||||
| Bing | Bing RSS | General web |
|
| Bing | Bing RSS | General web |
|
||||||
| Google | GSA User-Agent scraping | General web (no API key) |
|
|
||||||
| YouTube | YouTube Data API v3 | Videos (requires API key) |
|
|
||||||
| Stack Overflow | Stack Exchange API | Q&A (registered, not enabled by default) |
|
|
||||||
|
|
||||||
Engines not listed in `engines.local_ported` are proxied to an upstream metasearch instance if `upstream.url` is configured.
|
Engines not listed in `engines.local_ported` are proxied to an upstream SearXNG instance if `upstream.url` is configured.
|
||||||
|
|
||||||
### API Keys
|
|
||||||
|
|
||||||
Brave Search API and YouTube Data API require keys. If omitted, those engines are silently skipped. Brave Lite (scraping) and Google (GSA UA scraping) work without keys.
|
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
┌───────────────────────────────────────┐
|
┌─────────────────────────────────────┐
|
||||||
│ HTTP Handler │
|
│ HTTP Handler │
|
||||||
│ /search / /opensearch.xml │
|
│ /search / /opensearch.xml │
|
||||||
├───────────────────────────────────────┤
|
├─────────────────────────────────────┤
|
||||||
│ Middleware Chain │
|
│ Middleware Chain │
|
||||||
│ Global → Burst → Per-IP → CORS │
|
│ Global → Burst → Per-IP → CORS │
|
||||||
├───────────────────────────────────────┤
|
├─────────────────────────────────────┤
|
||||||
│ Search Service │
|
│ Search Service │
|
||||||
│ Parallel engine execution │
|
│ Parallel engine execution │
|
||||||
│ WaitGroup + graceful degradation │
|
│ WaitGroup + graceful degradation │
|
||||||
├───────────────────────────────────────┤
|
├─────────────────────────────────────┤
|
||||||
│ Cache Layer │
|
│ Cache Layer │
|
||||||
│ Valkey/Redis (optional; no-op if │
|
│ Valkey/Redis (optional, no-op if │
|
||||||
│ unconfigured) │
|
│ unconfigured) │
|
||||||
├───────────────────────────────────────┤
|
├─────────────────────────────────────┤
|
||||||
│ Engines (×11 default) │
|
│ Engines (×9) │
|
||||||
│ Each runs in its own goroutine │
|
│ Each runs in its own goroutine │
|
||||||
│ Failures → unresponsive_engines │
|
│ Failures → unresponsive_engines │
|
||||||
└───────────────────────────────────────┘
|
└─────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
## Docker
|
## Docker
|
||||||
|
|
||||||
The Dockerfile uses a multi-stage build with a static Go binary on alpine Linux:
|
The Dockerfile uses a multi-stage build:
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
# Build stage: golang:1.24-alpine
|
||||||
|
# Runtime stage: alpine:3.21 (~20MB)
|
||||||
|
# CGO_ENABLED=0 — static binary
|
||||||
|
```
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Build: golang:1.24-alpine
|
|
||||||
# Runtime: alpine:3.21 (~20MB)
|
|
||||||
# CGO_ENABLED=0 — fully static
|
|
||||||
docker compose up -d
|
docker compose up -d
|
||||||
```
|
```
|
||||||
|
|
||||||
Includes Valkey 8 with health checks out of the box.
|
Includes Valkey 8 with health checks out of the box.
|
||||||
|
|
||||||
## Contributing
|
|
||||||
|
|
||||||
See [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md) for a walkthrough of adding a new engine. The interface is two methods: `Name()` and `Search(context, request)`.
|
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
[AGPLv3](https://www.gnu.org/licenses/agpl-3.0.html)
|
MIT
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -25,13 +9,13 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/autocomplete"
|
"github.com/metamorphosis-dev/kafka/internal/autocomplete"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/cache"
|
"github.com/metamorphosis-dev/kafka/internal/cache"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/config"
|
"github.com/metamorphosis-dev/kafka/internal/config"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/httpapi"
|
"github.com/metamorphosis-dev/kafka/internal/httpapi"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/middleware"
|
"github.com/metamorphosis-dev/kafka/internal/middleware"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/search"
|
"github.com/metamorphosis-dev/kafka/internal/search"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/views"
|
"github.com/metamorphosis-dev/kafka/internal/views"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
|
@ -72,25 +56,18 @@ func main() {
|
||||||
UpstreamURL: cfg.Upstream.URL,
|
UpstreamURL: cfg.Upstream.URL,
|
||||||
HTTPTimeout: cfg.HTTPTimeout(),
|
HTTPTimeout: cfg.HTTPTimeout(),
|
||||||
Cache: searchCache,
|
Cache: searchCache,
|
||||||
EnginesConfig: cfg,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
acSvc := autocomplete.NewService(cfg.Upstream.URL, cfg.HTTPTimeout())
|
acSvc := autocomplete.NewService(cfg.Upstream.URL, cfg.HTTPTimeout())
|
||||||
|
|
||||||
h := httpapi.NewHandler(svc, acSvc.Suggestions, cfg.Server.SourceURL, searchCache)
|
h := httpapi.NewHandler(svc, acSvc.Suggestions)
|
||||||
|
|
||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
// HTML template routes
|
|
||||||
mux.HandleFunc("/", h.Index)
|
mux.HandleFunc("/", h.Index)
|
||||||
mux.HandleFunc("/search", h.Search)
|
|
||||||
mux.HandleFunc("/preferences", h.Preferences)
|
|
||||||
|
|
||||||
// API routes
|
|
||||||
mux.HandleFunc("/healthz", h.Healthz)
|
mux.HandleFunc("/healthz", h.Healthz)
|
||||||
|
mux.HandleFunc("/search", h.Search)
|
||||||
mux.HandleFunc("/autocompleter", h.Autocompleter)
|
mux.HandleFunc("/autocompleter", h.Autocompleter)
|
||||||
mux.HandleFunc("/opensearch.xml", h.OpenSearch(cfg.Server.BaseURL))
|
mux.HandleFunc("/opensearch.xml", h.OpenSearch(cfg.Server.BaseURL))
|
||||||
mux.HandleFunc("/favicon/", h.Favicon)
|
|
||||||
|
|
||||||
// Serve embedded static files (CSS, JS, images).
|
// Serve embedded static files (CSS, JS, images).
|
||||||
staticFS, err := views.StaticFS()
|
staticFS, err := views.StaticFS()
|
||||||
|
|
@ -100,9 +77,8 @@ func main() {
|
||||||
var subFS fs.FS = staticFS
|
var subFS fs.FS = staticFS
|
||||||
mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.FS(subFS))))
|
mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.FS(subFS))))
|
||||||
|
|
||||||
// Apply middleware: global rate limit → burst rate limit → per-IP rate limit → CORS → security headers → handler.
|
// Apply middleware: global rate limit → burst rate limit → per-IP rate limit → CORS → handler.
|
||||||
var handler http.Handler = mux
|
var handler http.Handler = mux
|
||||||
handler = middleware.SecurityHeaders(middleware.SecurityHeadersConfig{})(handler)
|
|
||||||
handler = middleware.CORS(middleware.CORSConfig{
|
handler = middleware.CORS(middleware.CORSConfig{
|
||||||
AllowedOrigins: cfg.CORS.AllowedOrigins,
|
AllowedOrigins: cfg.CORS.AllowedOrigins,
|
||||||
AllowedMethods: cfg.CORS.AllowedMethods,
|
AllowedMethods: cfg.CORS.AllowedMethods,
|
||||||
|
|
@ -114,7 +90,6 @@ func main() {
|
||||||
Requests: cfg.RateLimit.Requests,
|
Requests: cfg.RateLimit.Requests,
|
||||||
Window: cfg.RateLimitWindow(),
|
Window: cfg.RateLimitWindow(),
|
||||||
CleanupInterval: cfg.RateLimitCleanupInterval(),
|
CleanupInterval: cfg.RateLimitCleanupInterval(),
|
||||||
TrustedProxies: cfg.RateLimit.TrustedProxies,
|
|
||||||
}, logger)(handler)
|
}, logger)(handler)
|
||||||
handler = middleware.GlobalRateLimit(middleware.GlobalRateLimitConfig{
|
handler = middleware.GlobalRateLimit(middleware.GlobalRateLimitConfig{
|
||||||
Requests: cfg.GlobalRateLimit.Requests,
|
Requests: cfg.GlobalRateLimit.Requests,
|
||||||
|
|
@ -128,7 +103,7 @@ func main() {
|
||||||
}, logger)(handler)
|
}, logger)(handler)
|
||||||
|
|
||||||
addr := fmt.Sprintf(":%d", cfg.Server.Port)
|
addr := fmt.Sprintf(":%d", cfg.Server.Port)
|
||||||
logger.Info("samsa starting",
|
logger.Info("searxng-go starting",
|
||||||
"addr", addr,
|
"addr", addr,
|
||||||
"cache", searchCache.Enabled(),
|
"cache", searchCache.Enabled(),
|
||||||
"rate_limit", cfg.RateLimit.Requests > 0,
|
"rate_limit", cfg.RateLimit.Requests > 0,
|
||||||
|
|
@ -1,34 +1,28 @@
|
||||||
# samsa configuration
|
# kafka configuration
|
||||||
# Copy to config.toml and adjust as needed.
|
# Copy to config.toml and adjust as needed.
|
||||||
# Environment variables are used as fallbacks when a config field is empty/unset.
|
# Environment variables are used as fallbacks when a config field is empty/unset.
|
||||||
|
|
||||||
[server]
|
[server]
|
||||||
# Listen port (env: PORT)
|
# Listen port (env: PORT)
|
||||||
port = 5355
|
port = 8080
|
||||||
|
|
||||||
# HTTP timeout for engine and upstream calls (env: HTTP_TIMEOUT)
|
# HTTP timeout for engine and upstream calls (env: HTTP_TIMEOUT)
|
||||||
http_timeout = "10s"
|
http_timeout = "10s"
|
||||||
|
|
||||||
# Public base URL for OpenSearch XML (env: BASE_URL)
|
# Public base URL for OpenSearch XML (env: BASE_URL)
|
||||||
# Set this so browsers can add samsa as a search engine.
|
# Set this so browsers can add kafka as a search engine.
|
||||||
# Example: "https://search.example.com"
|
# Example: "https://search.example.com"
|
||||||
base_url = ""
|
base_url = ""
|
||||||
|
|
||||||
# Link to the source code (shown in footer as "Source" link)
|
|
||||||
# Defaults to the upstream samsa repo if not set.
|
|
||||||
# Example: "https://git.example.com/my-samsa-fork"
|
|
||||||
source_url = ""
|
|
||||||
|
|
||||||
[upstream]
|
[upstream]
|
||||||
# URL of an upstream metasearch instance for unported engines (env: UPSTREAM_SEARXNG_URL)
|
# URL of an upstream SearXNG instance for unported engines (env: UPSTREAM_SEARXNG_URL)
|
||||||
# Leave empty to run without an upstream proxy.
|
# Leave empty to run without an upstream proxy.
|
||||||
url = ""
|
url = ""
|
||||||
|
|
||||||
[engines]
|
[engines]
|
||||||
# Comma-separated list of engines to execute locally in Go (env: LOCAL_PORTED_ENGINES)
|
# Comma-separated list of engines to execute locally in Go (env: LOCAL_PORTED_ENGINES)
|
||||||
# Engines not listed here will be proxied to the upstream instance.
|
# Engines not listed here will be proxied to upstream SearXNG.
|
||||||
# Include bing_images, ddg_images, qwant_images for image search when [upstream].url is empty.
|
local_ported = ["wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"]
|
||||||
local_ported = ["wikipedia", "wikidata", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing", "google", "youtube", "bing_images", "ddg_images", "qwant_images"]
|
|
||||||
|
|
||||||
[engines.brave]
|
[engines.brave]
|
||||||
# Brave Search API key (env: BRAVE_API_KEY)
|
# Brave Search API key (env: BRAVE_API_KEY)
|
||||||
|
|
@ -41,10 +35,6 @@ access_token = ""
|
||||||
category = "web-lite"
|
category = "web-lite"
|
||||||
results_per_page = 10
|
results_per_page = 10
|
||||||
|
|
||||||
[engines.youtube]
|
|
||||||
# YouTube Data API v3 key (env: YOUTUBE_API_KEY)
|
|
||||||
api_key = ""
|
|
||||||
|
|
||||||
[cache]
|
[cache]
|
||||||
# Valkey/Redis cache for search results.
|
# Valkey/Redis cache for search results.
|
||||||
# Leave address empty to disable caching entirely.
|
# Leave address empty to disable caching entirely.
|
||||||
|
|
@ -57,12 +47,6 @@ db = 0
|
||||||
# Cache TTL for search results (env: VALKEY_CACHE_TTL)
|
# Cache TTL for search results (env: VALKEY_CACHE_TTL)
|
||||||
default_ttl = "5m"
|
default_ttl = "5m"
|
||||||
|
|
||||||
[cache.ttl_overrides]
|
|
||||||
# Per-engine TTL overrides (uncomment to use):
|
|
||||||
# wikipedia = "48h"
|
|
||||||
# reddit = "15m"
|
|
||||||
# braveapi = "2h"
|
|
||||||
|
|
||||||
[cors]
|
[cors]
|
||||||
# CORS configuration for browser-based clients.
|
# CORS configuration for browser-based clients.
|
||||||
# Allowed origins: use "*" for all, or specific domains (env: CORS_ALLOWED_ORIGINS)
|
# Allowed origins: use "*" for all, or specific domains (env: CORS_ALLOWED_ORIGINS)
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ services:
|
||||||
kafka:
|
kafka:
|
||||||
build: .
|
build: .
|
||||||
ports:
|
ports:
|
||||||
- "5355:5355"
|
- "8080:8080"
|
||||||
volumes:
|
volumes:
|
||||||
- ./config.toml:/etc/kafka/config.toml:ro
|
- ./config.toml:/etc/kafka/config.toml:ro
|
||||||
depends_on:
|
depends_on:
|
||||||
|
|
|
||||||
|
|
@ -1,218 +0,0 @@
|
||||||
# Contributing — Adding a New Engine
|
|
||||||
|
|
||||||
This guide walks through adding a new search engine to samsa. The minimal engine needs only an HTTP client, a query, and a result parser.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 1. Create the engine file
|
|
||||||
|
|
||||||
Place it in `internal/engines/`:
|
|
||||||
|
|
||||||
```
|
|
||||||
internal/engines/
|
|
||||||
myengine.go ← your engine
|
|
||||||
myengine_test.go ← tests (required)
|
|
||||||
```
|
|
||||||
|
|
||||||
Name the struct after the engine, e.g. `WolframEngine` for "wolfram". The `Name()` method returns the engine key used throughout samsa.
|
|
||||||
|
|
||||||
## 2. Implement the Engine interface
|
|
||||||
|
|
||||||
```go
|
|
||||||
package engines
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
type MyEngine struct {
|
|
||||||
client *http.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *MyEngine) Name() string { return "myengine" }
|
|
||||||
|
|
||||||
func (e *MyEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
|
||||||
// ...
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### The SearchRequest fields you'll use most:
|
|
||||||
|
|
||||||
| Field | Type | Description |
|
|
||||||
|-------|------|-------------|
|
|
||||||
| `Query` | `string` | The search query |
|
|
||||||
| `Pageno` | `int` | Current page number (1-based) |
|
|
||||||
| `Safesearch` | `int` | 0=off, 1=moderate, 2=strict |
|
|
||||||
| `Language` | `string` | ISO language code (e.g. `"en"`) |
|
|
||||||
|
|
||||||
### The SearchResponse to return:
|
|
||||||
|
|
||||||
```go
|
|
||||||
contracts.SearchResponse{
|
|
||||||
Query: req.Query,
|
|
||||||
NumberOfResults: len(results),
|
|
||||||
Results: results, // []MainResult
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
UnresponsiveEngines: [][2]string{},
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Empty query — return early:
|
|
||||||
|
|
||||||
```go
|
|
||||||
if strings.TrimSpace(req.Query) == "" {
|
|
||||||
return contracts.SearchResponse{Query: req.Query}, nil
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Engine unavailable / error — graceful degradation:
|
|
||||||
|
|
||||||
```go
|
|
||||||
// Rate limited or blocked
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: req.Query,
|
|
||||||
UnresponsiveEngines: [][2]string{{"myengine", "reason"}},
|
|
||||||
Results: []contracts.MainResult{},
|
|
||||||
// ... empty other fields
|
|
||||||
}, nil
|
|
||||||
|
|
||||||
// Hard error — return it
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("myengine upstream error: status %d", resp.StatusCode)
|
|
||||||
```
|
|
||||||
|
|
||||||
## 3. Build the result
|
|
||||||
|
|
||||||
```go
|
|
||||||
urlPtr := "https://example.com/result"
|
|
||||||
result := contracts.MainResult{
|
|
||||||
Title: "Result Title",
|
|
||||||
Content: "Snippet or description text",
|
|
||||||
URL: &urlPtr, // pointer to string, required
|
|
||||||
Engine: "myengine",
|
|
||||||
Category: "general", // or "it", "science", "videos", "images", "social media"
|
|
||||||
Score: 0, // used for relevance ranking during merge
|
|
||||||
Engines: []string{"myengine"},
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Template field
|
|
||||||
|
|
||||||
The template system checks for `"videos"` and `"images"`. Everything else renders via `result_item.html`. Set `Template` only if you have a custom template; omit it for the default result card.
|
|
||||||
|
|
||||||
### Category field
|
|
||||||
|
|
||||||
Controls which category tab the result appears under and which engines are triggered:
|
|
||||||
|
|
||||||
| Category | Engines used |
|
|
||||||
|----------|-------------|
|
|
||||||
| `general` | google, bing, ddg, brave, braveapi, qwant, wikipedia |
|
|
||||||
| `it` | github, stackoverflow |
|
|
||||||
| `science` | arxiv, crossref |
|
|
||||||
| `videos` | youtube |
|
|
||||||
| `images` | bing_images, ddg_images, qwant_images |
|
|
||||||
| `social media` | reddit |
|
|
||||||
|
|
||||||
## 4. Wire it into the factory
|
|
||||||
|
|
||||||
In `internal/engines/factory.go`, add your engine to the map returned by `NewDefaultPortedEngines`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
"myengine": &MyEngine{client: client},
|
|
||||||
```
|
|
||||||
|
|
||||||
If your engine needs an API key, read it from config or the environment (see `braveapi` or `youtube` in factory.go for the pattern).
|
|
||||||
|
|
||||||
## 5. Register defaults
|
|
||||||
|
|
||||||
In `internal/engines/planner.go`:
|
|
||||||
|
|
||||||
**Add to `defaultPortedEngines`:**
|
|
||||||
```go
|
|
||||||
var defaultPortedEngines = []string{
|
|
||||||
// ... existing ...
|
|
||||||
"myengine",
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Add to category mapping in `inferFromCategories`** (if applicable):
|
|
||||||
```go
|
|
||||||
case "general":
|
|
||||||
set["myengine"] = true
|
|
||||||
```
|
|
||||||
|
|
||||||
**Update the sort order map** so results maintain consistent ordering:
|
|
||||||
```go
|
|
||||||
order := map[string]int{
|
|
||||||
// ... existing ...
|
|
||||||
"myengine": N, // pick a slot
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 6. Add tests
|
|
||||||
|
|
||||||
At minimum, test:
|
|
||||||
- `Name()` returns the correct string
|
|
||||||
- Nil engine returns an error
|
|
||||||
- Empty query returns zero results
|
|
||||||
- Successful API response parses correctly
|
|
||||||
- Rate limit / error cases return `UnresponsiveEngines` with a reason
|
|
||||||
|
|
||||||
Use `httptest.NewServer` to mock the upstream API. See `arxiv_test.go` or `reddit_test.go` for examples.
|
|
||||||
|
|
||||||
## 7. Build and test
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go build ./...
|
|
||||||
go test ./internal/engines/ -run MyEngine -v
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
## Example: Adding an RSS-based engine
|
|
||||||
|
|
||||||
If the engine provides an RSS feed, the parsing is straightforward:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type rssItem struct {
|
|
||||||
Title string `xml:"title"`
|
|
||||||
Link string `xml:"link"`
|
|
||||||
Description string `xml:"description"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rssFeed struct {
|
|
||||||
Channel struct {
|
|
||||||
Items []rssItem `xml:"item"`
|
|
||||||
} `xml:"channel"`
|
|
||||||
}
|
|
||||||
|
|
||||||
dec := xml.NewDecoder(resp.Body)
|
|
||||||
var feed rssFeed
|
|
||||||
dec.Decode(&feed)
|
|
||||||
|
|
||||||
for _, item := range feed.Channel.Items {
|
|
||||||
urlPtr := item.Link
|
|
||||||
results = append(results, contracts.MainResult{
|
|
||||||
Title: item.Title,
|
|
||||||
Content: stripHTML(item.Description),
|
|
||||||
URL: &urlPtr,
|
|
||||||
Engine: "myengine",
|
|
||||||
// ...
|
|
||||||
})
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Checklist
|
|
||||||
|
|
||||||
- [ ] Engine file created in `internal/engines/`
|
|
||||||
- [ ] `Engine` interface implemented (`Name()` + `Search()`)
|
|
||||||
- [ ] Empty query handled (return early, no error)
|
|
||||||
- [ ] Graceful degradation for errors and rate limits
|
|
||||||
- [ ] Results use `Category` to group with related engines
|
|
||||||
- [ ] Factory updated with new engine
|
|
||||||
- [ ] Planner updated (defaults + category mapping + sort order)
|
|
||||||
- [ ] Tests written covering main paths
|
|
||||||
- [ ] `go build ./...` succeeds
|
|
||||||
- [ ] `go test ./...` passes
|
|
||||||
|
|
@ -1,789 +0,0 @@
|
||||||
# Per-Engine TTL Cache — Implementation Plan
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
|
||||||
|
|
||||||
**Goal:** Replace the merged-response cache with per-engine response caching, enabling tier-based TTLs and stale-while-revalidate semantics.
|
|
||||||
|
|
||||||
**Architecture:** Each engine's raw response is cached independently with its tier-based TTL. On stale hits, return cached data immediately and refresh in background. Query hash is computed from shared params (query, pageno, safesearch, language, time_range) and prefixed with engine name for the cache key.
|
|
||||||
|
|
||||||
**Tech Stack:** Go 1.24, Valkey/Redis (go-redis/v9), existing samsa contracts
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## File Map
|
|
||||||
|
|
||||||
| Action | File | Responsibility |
|
|
||||||
|--------|------|----------------|
|
|
||||||
| Create | `internal/cache/tiers.go` | Tier definitions, `EngineTier()` function |
|
|
||||||
| Create | `internal/cache/tiers_test.go` | Tests for EngineTier |
|
|
||||||
| Create | `internal/cache/engine_cache.go` | `EngineCache` struct with tier-aware Get/Set |
|
|
||||||
| Create | `internal/cache/engine_cache_test.go` | Tests for EngineCache |
|
|
||||||
| Modify | `internal/cache/cache.go` | Add `QueryHash()`, add `CachedEngineResponse` type |
|
|
||||||
| Modify | `internal/cache/cache_test.go` | Add tests for `QueryHash()` |
|
|
||||||
| Modify | `internal/config/config.go` | Add `TTLOverrides` to `CacheConfig` |
|
|
||||||
| Modify | `internal/search/service.go` | Use `EngineCache`, parallel lookups, background refresh |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task 1: Add QueryHash and CachedEngineResponse to cache.go
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `internal/cache/cache.go`
|
|
||||||
- Modify: `internal/cache/cache_test.go`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing test for QueryHash()**
|
|
||||||
|
|
||||||
```go
|
|
||||||
// In cache_test.go, add:
|
|
||||||
|
|
||||||
func TestQueryHash(t *testing.T) {
|
|
||||||
// Same params should produce same hash
|
|
||||||
hash1 := QueryHash("golang", 1, 0, "en", "")
|
|
||||||
hash2 := QueryHash("golang", 1, 0, "en", "")
|
|
||||||
if hash1 != hash2 {
|
|
||||||
t.Errorf("QueryHash: same params should produce same hash, got %s != %s", hash1, hash2)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Different query should produce different hash
|
|
||||||
hash3 := QueryHash("rust", 1, 0, "en", "")
|
|
||||||
if hash1 == hash3 {
|
|
||||||
t.Errorf("QueryHash: different queries should produce different hash")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Different pageno should produce different hash
|
|
||||||
hash4 := QueryHash("golang", 2, 0, "en", "")
|
|
||||||
if hash1 == hash4 {
|
|
||||||
t.Errorf("QueryHash: different pageno should produce different hash")
|
|
||||||
}
|
|
||||||
|
|
||||||
// time_range should affect hash
|
|
||||||
hash5 := QueryHash("golang", 1, 0, "en", "day")
|
|
||||||
if hash1 == hash5 {
|
|
||||||
t.Errorf("QueryHash: different time_range should produce different hash")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Hash should be 16 characters (truncated SHA-256)
|
|
||||||
if len(hash1) != 16 {
|
|
||||||
t.Errorf("QueryHash: expected 16 char hash, got %d", len(hash1))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run test to verify it fails**
|
|
||||||
|
|
||||||
Run: `nix develop --command bash -c "go test -run TestQueryHash ./internal/cache/ -v"`
|
|
||||||
Expected: FAIL — "QueryHash not defined"
|
|
||||||
|
|
||||||
- [ ] **Step 3: Implement QueryHash() and CachedEngineResponse in cache.go**
|
|
||||||
|
|
||||||
Add to `cache.go` (the imports `crypto/sha256` and `encoding/hex` are already present in cache.go from the existing `Key()` function):
|
|
||||||
|
|
||||||
```go
|
|
||||||
// QueryHash computes a deterministic hash from shared request parameters
|
|
||||||
// (query, pageno, safesearch, language, time_range) for use as a cache key suffix.
|
|
||||||
// The hash is a truncated SHA-256 (16 hex chars).
|
|
||||||
func QueryHash(query string, pageno int, safesearch int, language, timeRange string) string {
|
|
||||||
h := sha256.New()
|
|
||||||
fmt.Fprintf(h, "q=%s|", query)
|
|
||||||
fmt.Fprintf(h, "pageno=%d|", pageno)
|
|
||||||
fmt.Fprintf(h, "safesearch=%d|", safesearch)
|
|
||||||
fmt.Fprintf(h, "lang=%s|", language)
|
|
||||||
if timeRange != "" {
|
|
||||||
fmt.Fprintf(h, "tr=%s|", timeRange)
|
|
||||||
}
|
|
||||||
return hex.EncodeToString(h.Sum(nil))[:16]
|
|
||||||
}
|
|
||||||
|
|
||||||
// CachedEngineResponse wraps an engine's cached response with metadata.
|
|
||||||
type CachedEngineResponse struct {
|
|
||||||
Engine string
|
|
||||||
Response []byte
|
|
||||||
StoredAt time.Time
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run test to verify it passes**
|
|
||||||
|
|
||||||
Run: `nix develop --command bash -c "go test -run TestQueryHash ./internal/cache/ -v"`
|
|
||||||
Expected: PASS
|
|
||||||
|
|
||||||
- [ ] **Step 5: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add internal/cache/cache.go internal/cache/cache_test.go
|
|
||||||
git commit -m "cache: add QueryHash and CachedEngineResponse type"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task 2: Create tiers.go with tier definitions
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `internal/cache/tiers.go`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Create tiers.go with tier definitions and EngineTier function**
|
|
||||||
|
|
||||||
```go
|
|
||||||
package cache
|
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
// TTLTier represents a cache TTL tier with a name and duration.
|
|
||||||
type TTLTier struct {
|
|
||||||
Name string
|
|
||||||
Duration time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// defaultTiers maps engine names to their default TTL tiers.
|
|
||||||
var defaultTiers = map[string]TTLTier{
|
|
||||||
// Static knowledge engines — rarely change
|
|
||||||
"wikipedia": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
"wikidata": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
"arxiv": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
"crossref": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
"stackoverflow": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
"github": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
|
|
||||||
// API-based general search — fresher data
|
|
||||||
"braveapi": {Name: "api_general", Duration: 1 * time.Hour},
|
|
||||||
"youtube": {Name: "api_general", Duration: 1 * time.Hour},
|
|
||||||
|
|
||||||
// Scraped general search — moderately stable
|
|
||||||
"google": {Name: "scraped_general", Duration: 2 * time.Hour},
|
|
||||||
"bing": {Name: "scraped_general", Duration: 2 * time.Hour},
|
|
||||||
"duckduckgo": {Name: "scraped_general", Duration: 2 * time.Hour},
|
|
||||||
"qwant": {Name: "scraped_general", Duration: 2 * time.Hour},
|
|
||||||
"brave": {Name: "scraped_general", Duration: 2 * time.Hour},
|
|
||||||
|
|
||||||
// News/social — changes frequently
|
|
||||||
"reddit": {Name: "news_social", Duration: 30 * time.Minute},
|
|
||||||
|
|
||||||
// Image search
|
|
||||||
"bing_images": {Name: "images", Duration: 1 * time.Hour},
|
|
||||||
"ddg_images": {Name: "images", Duration: 1 * time.Hour},
|
|
||||||
"qwant_images": {Name: "images", Duration: 1 * time.Hour},
|
|
||||||
}
|
|
||||||
|
|
||||||
// EngineTier returns the TTL tier for an engine, applying overrides if provided.
|
|
||||||
// If the engine has no defined tier, returns a default of 1 hour.
|
|
||||||
func EngineTier(engineName string, overrides map[string]time.Duration) TTLTier {
|
|
||||||
// Check override first — override tier name is just the engine name
|
|
||||||
if override, ok := overrides[engineName]; ok && override > 0 {
|
|
||||||
return TTLTier{Name: engineName, Duration: override}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fall back to default tier
|
|
||||||
if tier, ok := defaultTiers[engineName]; ok {
|
|
||||||
return tier
|
|
||||||
}
|
|
||||||
|
|
||||||
// Unknown engines get a sensible default
|
|
||||||
return TTLTier{Name: "unknown", Duration: 1 * time.Hour}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run go vet to verify it compiles**
|
|
||||||
|
|
||||||
Run: `nix develop --command bash -c "go vet ./internal/cache/tiers.go"`
|
|
||||||
Expected: no output (success)
|
|
||||||
|
|
||||||
- [ ] **Step 3: Write a basic test for EngineTier**
|
|
||||||
|
|
||||||
```go
|
|
||||||
// In internal/cache/tiers_test.go:
|
|
||||||
|
|
||||||
package cache
|
|
||||||
|
|
||||||
import "testing"
|
|
||||||
|
|
||||||
func TestEngineTier(t *testing.T) {
|
|
||||||
// Test default static tier
|
|
||||||
tier := EngineTier("wikipedia", nil)
|
|
||||||
if tier.Name != "static" || tier.Duration != 24*time.Hour {
|
|
||||||
t.Errorf("wikipedia: expected static/24h, got %s/%v", tier.Name, tier.Duration)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test default api_general tier
|
|
||||||
tier = EngineTier("braveapi", nil)
|
|
||||||
if tier.Name != "api_general" || tier.Duration != 1*time.Hour {
|
|
||||||
t.Errorf("braveapi: expected api_general/1h, got %s/%v", tier.Name, tier.Duration)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test override takes precedence — override tier name is just the engine name
|
|
||||||
override := 48 * time.Hour
|
|
||||||
tier = EngineTier("wikipedia", map[string]time.Duration{"wikipedia": override})
|
|
||||||
if tier.Name != "wikipedia" || tier.Duration != 48*time.Hour {
|
|
||||||
t.Errorf("wikipedia override: expected wikipedia/48h, got %s/%v", tier.Name, tier.Duration)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test unknown engine gets default
|
|
||||||
tier = EngineTier("unknown_engine", nil)
|
|
||||||
if tier.Name != "unknown" || tier.Duration != 1*time.Hour {
|
|
||||||
t.Errorf("unknown engine: expected unknown/1h, got %s/%v", tier.Name, tier.Duration)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run test to verify it passes**
|
|
||||||
|
|
||||||
Run: `nix develop --command bash -c "go test -run TestEngineTier ./internal/cache/ -v"`
|
|
||||||
Expected: PASS
|
|
||||||
|
|
||||||
- [ ] **Step 5: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add internal/cache/tiers.go internal/cache/tiers_test.go
|
|
||||||
git commit -m "cache: add tier definitions and EngineTier function"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task 3: Create EngineCache in engine_cache.go
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `internal/cache/engine_cache.go`
|
|
||||||
- Create: `internal/cache/engine_cache_test.go`
|
|
||||||
|
|
||||||
**Note:** The existing `Key()` function in `cache.go` is still used for favicon caching. The new `QueryHash()` and `EngineCache` are separate and only for per-engine search response caching.
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing test for EngineCache.Get/Set**
|
|
||||||
|
|
||||||
```go
|
|
||||||
package cache
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestEngineCacheGetSet(t *testing.T) {
|
|
||||||
// Create a disabled cache for unit testing (nil client)
|
|
||||||
c := &Cache{logger: slog.Default()}
|
|
||||||
ec := NewEngineCache(c, nil)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
cached, ok := ec.Get(ctx, "wikipedia", "abc123")
|
|
||||||
if ok {
|
|
||||||
t.Errorf("Get on disabled cache: expected false, got %v", ok)
|
|
||||||
}
|
|
||||||
_ = cached // unused when ok=false
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEngineCacheKeyFormat(t *testing.T) {
|
|
||||||
key := engineCacheKey("wikipedia", "abc123")
|
|
||||||
if key != "samsa:resp:wikipedia:abc123" {
|
|
||||||
t.Errorf("engineCacheKey: expected samsa:resp:wikipedia:abc123, got %s", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEngineCacheIsStale(t *testing.T) {
|
|
||||||
c := &Cache{logger: slog.Default()}
|
|
||||||
ec := NewEngineCache(c, nil)
|
|
||||||
|
|
||||||
// Fresh response (stored 1 minute ago, wikipedia has 24h TTL)
|
|
||||||
fresh := CachedEngineResponse{
|
|
||||||
Engine: "wikipedia",
|
|
||||||
Response: []byte(`{}`),
|
|
||||||
StoredAt: time.Now().Add(-1 * time.Minute),
|
|
||||||
}
|
|
||||||
if ec.IsStale(fresh, "wikipedia") {
|
|
||||||
t.Errorf("IsStale: 1-minute-old wikipedia should NOT be stale")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stale response (stored 25 hours ago)
|
|
||||||
stale := CachedEngineResponse{
|
|
||||||
Engine: "wikipedia",
|
|
||||||
Response: []byte(`{}`),
|
|
||||||
StoredAt: time.Now().Add(-25 * time.Hour),
|
|
||||||
}
|
|
||||||
if !ec.IsStale(stale, "wikipedia") {
|
|
||||||
t.Errorf("IsStale: 25-hour-old wikipedia SHOULD be stale (24h TTL)")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Override: 30 minute TTL for reddit
|
|
||||||
overrides := map[string]time.Duration{"reddit": 30 * time.Minute}
|
|
||||||
ec2 := NewEngineCache(c, overrides)
|
|
||||||
|
|
||||||
// 20 minutes old with 30m override should NOT be stale
|
|
||||||
redditFresh := CachedEngineResponse{
|
|
||||||
Engine: "reddit",
|
|
||||||
Response: []byte(`{}`),
|
|
||||||
StoredAt: time.Now().Add(-20 * time.Minute),
|
|
||||||
}
|
|
||||||
if ec2.IsStale(redditFresh, "reddit") {
|
|
||||||
t.Errorf("IsStale: 20-min reddit with 30m override should NOT be stale")
|
|
||||||
}
|
|
||||||
|
|
||||||
// 45 minutes old with 30m override SHOULD be stale
|
|
||||||
redditStale := CachedEngineResponse{
|
|
||||||
Engine: "reddit",
|
|
||||||
Response: []byte(`{}`),
|
|
||||||
StoredAt: time.Now().Add(-45 * time.Minute),
|
|
||||||
}
|
|
||||||
if !ec2.IsStale(redditStale, "reddit") {
|
|
||||||
t.Errorf("IsStale: 45-min reddit with 30m override SHOULD be stale")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run test to verify it fails**
|
|
||||||
|
|
||||||
Run: `nix develop --command bash -c "go test -run TestEngineCache ./internal/cache/ -v"`
|
|
||||||
Expected: FAIL — "EngineCache not defined" or "CachedEngineResponse not defined"
|
|
||||||
|
|
||||||
- [ ] **Step 3: Implement EngineCache using GetBytes/SetBytes**
|
|
||||||
|
|
||||||
The `EngineCache` uses the existing `GetBytes`/`SetBytes` public methods on `Cache` (the `client` field is unexported so we must use those methods).
|
|
||||||
|
|
||||||
```go
|
|
||||||
package cache
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"log/slog"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
// EngineCache wraps Cache with per-engine tier-aware Get/Set operations.
|
|
||||||
type EngineCache struct {
|
|
||||||
cache *Cache
|
|
||||||
overrides map[string]time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewEngineCache creates a new EngineCache with optional TTL overrides.
|
|
||||||
// If overrides is nil, default tier durations are used.
|
|
||||||
func NewEngineCache(cache *Cache, overrides map[string]time.Duration) *EngineCache {
|
|
||||||
return &EngineCache{
|
|
||||||
cache: cache,
|
|
||||||
overrides: overrides,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get retrieves a cached engine response. Returns (zero value, false) if not
|
|
||||||
// found or if cache is disabled.
|
|
||||||
func (ec *EngineCache) Get(ctx context.Context, engine, queryHash string) (CachedEngineResponse, bool) {
|
|
||||||
key := engineCacheKey(engine, queryHash)
|
|
||||||
|
|
||||||
data, ok := ec.cache.GetBytes(ctx, key)
|
|
||||||
if !ok {
|
|
||||||
return CachedEngineResponse{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
var cached CachedEngineResponse
|
|
||||||
if err := json.Unmarshal(data, &cached); err != nil {
|
|
||||||
ec.cache.logger.Warn("engine cache hit but unmarshal failed", "key", key, "error", err)
|
|
||||||
return CachedEngineResponse{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
ec.cache.logger.Debug("engine cache hit", "key", key, "engine", engine)
|
|
||||||
return cached, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set stores an engine response in the cache with the engine's tier TTL.
|
|
||||||
func (ec *EngineCache) Set(ctx context.Context, engine, queryHash string, resp contracts.SearchResponse) {
|
|
||||||
if !ec.cache.Enabled() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := json.Marshal(resp)
|
|
||||||
if err != nil {
|
|
||||||
ec.cache.logger.Warn("engine cache set: marshal failed", "engine", engine, "error", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
tier := EngineTier(engine, ec.overrides)
|
|
||||||
key := engineCacheKey(engine, queryHash)
|
|
||||||
|
|
||||||
cached := CachedEngineResponse{
|
|
||||||
Engine: engine,
|
|
||||||
Response: data,
|
|
||||||
StoredAt: time.Now(),
|
|
||||||
}
|
|
||||||
|
|
||||||
cachedData, err := json.Marshal(cached)
|
|
||||||
if err != nil {
|
|
||||||
ec.cache.logger.Warn("engine cache set: wrap marshal failed", "key", key, "error", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
ec.cache.SetBytes(ctx, key, cachedData, tier.Duration)
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsStale returns true if the cached response is older than the tier's TTL.
|
|
||||||
func (ec *EngineCache) IsStale(cached CachedEngineResponse, engine string) bool {
|
|
||||||
tier := EngineTier(engine, ec.overrides)
|
|
||||||
return time.Since(cached.StoredAt) > tier.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// Logger returns the logger for background refresh logging.
|
|
||||||
func (ec *EngineCache) Logger() *slog.Logger {
|
|
||||||
return ec.cache.logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// engineCacheKey builds the cache key for an engine+query combination.
|
|
||||||
func engineCacheKey(engine, queryHash string) string {
|
|
||||||
return "samsa:resp:" + engine + ":" + queryHash
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run tests to verify they pass**
|
|
||||||
|
|
||||||
Run: `nix develop --command bash -c "go test -run TestEngineCache ./internal/cache/ -v"`
|
|
||||||
Expected: PASS
|
|
||||||
|
|
||||||
- [ ] **Step 5: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add internal/cache/engine_cache.go internal/cache/engine_cache_test.go
|
|
||||||
git commit -m "cache: add EngineCache with tier-aware Get/Set"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task 4: Add TTLOverrides to config
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `internal/config/config.go`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Add TTLOverrides to CacheConfig**
|
|
||||||
|
|
||||||
In `CacheConfig` struct, add:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type CacheConfig struct {
|
|
||||||
Address string `toml:"address"`
|
|
||||||
Password string `toml:"password"`
|
|
||||||
DB int `toml:"db"`
|
|
||||||
DefaultTTL string `toml:"default_ttl"`
|
|
||||||
TTLOverrides map[string]string `toml:"ttl_overrides"` // engine -> duration string
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Add TTLOverridesParsed() method to Config**
|
|
||||||
|
|
||||||
Add after `CacheTTL()`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
// CacheTTLOverrides returns parsed TTL overrides from config.
|
|
||||||
func (c *Config) CacheTTLOverrides() map[string]time.Duration {
|
|
||||||
if len(c.Cache.TTLOverrides) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make(map[string]time.Duration, len(c.Cache.TTLOverrides))
|
|
||||||
for engine, durStr := range c.Cache.TTLOverrides {
|
|
||||||
if d, err := time.ParseDuration(durStr); err == nil && d > 0 {
|
|
||||||
out[engine] = d
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Run tests to verify nothing breaks**
|
|
||||||
|
|
||||||
Run: `nix develop --command bash -c "go test ./internal/config/ -v"`
|
|
||||||
Expected: PASS
|
|
||||||
|
|
||||||
- [ ] **Step 4: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add internal/config/config.go
|
|
||||||
git commit -m "config: add TTLOverrides to CacheConfig"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task 5: Wire EngineCache into search service
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `internal/search/service.go`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Read the current service.go to understand wiring**
|
|
||||||
|
|
||||||
The service currently takes `*Cache` in `ServiceConfig`. We need to change it to take `*EngineCache` or change the field type.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Modify Service struct and NewService to use EngineCache**
|
|
||||||
|
|
||||||
Change `Service`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type Service struct {
|
|
||||||
upstreamClient *upstream.Client
|
|
||||||
planner *engines.Planner
|
|
||||||
localEngines map[string]engines.Engine
|
|
||||||
engineCache *cache.EngineCache
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Change `NewService`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
func NewService(cfg ServiceConfig) *Service {
|
|
||||||
timeout := cfg.HTTPTimeout
|
|
||||||
if timeout <= 0 {
|
|
||||||
timeout = 10 * time.Second
|
|
||||||
}
|
|
||||||
|
|
||||||
httpClient := httpclient.NewClient(timeout)
|
|
||||||
|
|
||||||
var up *upstream.Client
|
|
||||||
if cfg.UpstreamURL != "" {
|
|
||||||
c, err := upstream.NewClient(cfg.UpstreamURL, timeout)
|
|
||||||
if err == nil {
|
|
||||||
up = c
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var engineCache *cache.EngineCache
|
|
||||||
if cfg.Cache != nil {
|
|
||||||
engineCache = cache.NewEngineCache(cfg.Cache, cfg.CacheTTLOverrides)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Service{
|
|
||||||
upstreamClient: up,
|
|
||||||
planner: engines.NewPlannerFromEnv(),
|
|
||||||
localEngines: engines.NewDefaultPortedEngines(httpClient, cfg.EnginesConfig),
|
|
||||||
engineCache: engineCache,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Add `CacheTTLOverrides` to `ServiceConfig`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type ServiceConfig struct {
|
|
||||||
UpstreamURL string
|
|
||||||
HTTPTimeout time.Duration
|
|
||||||
Cache *cache.Cache
|
|
||||||
CacheTTLOverrides map[string]time.Duration
|
|
||||||
EnginesConfig *config.Config
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Rewrite Search() with correct stale-while-revalidate logic**
|
|
||||||
|
|
||||||
The stale-while-revalidate flow:
|
|
||||||
|
|
||||||
1. **Cache lookup (Phase 1)**: Check cache for each engine in parallel. Classify each as:
|
|
||||||
- Fresh hit: cache has data AND not stale → deserialize, mark as `fresh`
|
|
||||||
- Stale hit: cache has data AND stale → keep in `cached`, no `fresh` yet
|
|
||||||
- Miss: cache has no data → `hit=false`, no `cached` or `fresh`
|
|
||||||
|
|
||||||
2. **Fetch (Phase 2)**: For each engine:
|
|
||||||
- Fresh hit: return immediately, no fetch needed
|
|
||||||
- Stale hit: return stale data immediately, fetch fresh in background
|
|
||||||
- Miss: fetch fresh synchronously, cache result
|
|
||||||
|
|
||||||
3. **Collect (Phase 3)**: Collect all responses for merge.
|
|
||||||
|
|
||||||
```go
|
|
||||||
// Search executes the request against local engines (in parallel) and
|
|
||||||
// optionally the upstream instance for unported engines.
|
|
||||||
func (s *Service) Search(ctx context.Context, req SearchRequest) (SearchResponse, error) {
|
|
||||||
queryHash := cache.QueryHash(
|
|
||||||
req.Query,
|
|
||||||
int(req.Pageno),
|
|
||||||
int(req.Safesearch),
|
|
||||||
req.Language,
|
|
||||||
derefString(req.TimeRange),
|
|
||||||
)
|
|
||||||
|
|
||||||
localEngineNames, upstreamEngineNames, _ := s.planner.Plan(req)
|
|
||||||
|
|
||||||
// Phase 1: Parallel cache lookups — classify each engine as fresh/stale/miss
|
|
||||||
type cacheResult struct {
|
|
||||||
engine string
|
|
||||||
cached cache.CachedEngineResponse
|
|
||||||
hit bool
|
|
||||||
fresh contracts.SearchResponse
|
|
||||||
fetchErr error
|
|
||||||
unmarshalErr bool // true if hit but unmarshal failed (treat as miss)
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheResults := make([]cacheResult, len(localEngineNames))
|
|
||||||
|
|
||||||
var lookupWg sync.WaitGroup
|
|
||||||
for i, name := range localEngineNames {
|
|
||||||
lookupWg.Add(1)
|
|
||||||
go func(i int, name string) {
|
|
||||||
defer lookupWg.Done()
|
|
||||||
|
|
||||||
result := cacheResult{engine: name}
|
|
||||||
|
|
||||||
if s.engineCache != nil {
|
|
||||||
cached, ok := s.engineCache.Get(ctx, name, queryHash)
|
|
||||||
if ok {
|
|
||||||
result.hit = true
|
|
||||||
result.cached = cached
|
|
||||||
if !s.engineCache.IsStale(cached, name) {
|
|
||||||
// Fresh cache hit — deserialize and use directly
|
|
||||||
var resp contracts.SearchResponse
|
|
||||||
if err := json.Unmarshal(cached.Response, &resp); err == nil {
|
|
||||||
result.fresh = resp
|
|
||||||
} else {
|
|
||||||
// Unmarshal failed — treat as cache miss (will fetch fresh synchronously)
|
|
||||||
result.unmarshalErr = true
|
|
||||||
result.hit = false // treat as miss
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// If stale: result.fresh stays zero, result.cached has stale data
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheResults[i] = result
|
|
||||||
}(i, name)
|
|
||||||
}
|
|
||||||
lookupWg.Wait()
|
|
||||||
|
|
||||||
// Phase 2: Fetch fresh for misses and stale entries
|
|
||||||
var fetchWg sync.WaitGroup
|
|
||||||
for i, name := range localEngineNames {
|
|
||||||
cr := cacheResults[i]
|
|
||||||
|
|
||||||
// Fresh hit — nothing to do in phase 2
|
|
||||||
if cr.hit && cr.fresh.Response != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stale hit — return stale immediately, refresh in background
|
|
||||||
if cr.hit && cr.cached.Response != nil && s.engineCache != nil && s.engineCache.IsStale(cr.cached, name) {
|
|
||||||
fetchWg.Add(1)
|
|
||||||
go func(name string) {
|
|
||||||
defer fetchWg.Done()
|
|
||||||
eng, ok := s.localEngines[name]
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
freshResp, err := eng.Search(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
s.engineCache.Logger().Debug("background refresh failed", "engine", name, "error", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
s.engineCache.Set(ctx, name, queryHash, freshResp)
|
|
||||||
}(name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache miss — fetch fresh synchronously
|
|
||||||
if !cr.hit {
|
|
||||||
fetchWg.Add(1)
|
|
||||||
go func(i int, name string) {
|
|
||||||
defer fetchWg.Done()
|
|
||||||
|
|
||||||
eng, ok := s.localEngines[name]
|
|
||||||
if !ok {
|
|
||||||
cacheResults[i] = cacheResult{
|
|
||||||
engine: name,
|
|
||||||
fetchErr: fmt.Errorf("engine not registered: %s", name),
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
freshResp, err := eng.Search(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
cacheResults[i] = cacheResult{
|
|
||||||
engine: name,
|
|
||||||
fetchErr: err,
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache the fresh response
|
|
||||||
if s.engineCache != nil {
|
|
||||||
s.engineCache.Set(ctx, name, queryHash, freshResp)
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheResults[i] = cacheResult{
|
|
||||||
engine: name,
|
|
||||||
fresh: freshResp,
|
|
||||||
hit: false,
|
|
||||||
}
|
|
||||||
}(i, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fetchWg.Wait()
|
|
||||||
|
|
||||||
// Phase 3: Collect responses for merge
|
|
||||||
responses := make([]contracts.SearchResponse, 0, len(cacheResults))
|
|
||||||
|
|
||||||
for _, cr := range cacheResults {
|
|
||||||
if cr.fetchErr != nil {
|
|
||||||
responses = append(responses, unresponsiveResponse(req.Query, cr.engine, cr.fetchErr.Error()))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Use fresh data if available (fresh hit or freshly fetched), otherwise use stale cached
|
|
||||||
if cr.fresh.Response != nil {
|
|
||||||
responses = append(responses, cr.fresh)
|
|
||||||
} else if cr.hit && cr.cached.Response != nil {
|
|
||||||
var resp contracts.SearchResponse
|
|
||||||
if err := json.Unmarshal(cr.cached.Response, &resp); err == nil {
|
|
||||||
responses = append(responses, resp)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ... rest of upstream proxy and merge logic (unchanged) ...
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Note: The imports need `encoding/json` and `fmt` added. The existing imports in service.go already include `sync` and `time`.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run tests to verify compilation**
|
|
||||||
|
|
||||||
Run: `nix develop --command bash -c "go build ./internal/search/"`
|
|
||||||
Expected: no output (success)
|
|
||||||
|
|
||||||
- [ ] **Step 5: Run full test suite**
|
|
||||||
|
|
||||||
Run: `nix develop --command bash -c "go test ./..."`
|
|
||||||
Expected: All pass
|
|
||||||
|
|
||||||
- [ ] **Step 6: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add internal/search/service.go
|
|
||||||
git commit -m "search: wire per-engine cache with tier-aware TTLs"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task 6: Update config.example.toml
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `config.example.toml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Add TTL overrides section to config.example.toml**
|
|
||||||
|
|
||||||
Add after the `[cache]` section:
|
|
||||||
|
|
||||||
```toml
|
|
||||||
[cache.ttl_overrides]
|
|
||||||
# Per-engine TTL overrides (uncomment to use):
|
|
||||||
# wikipedia = "48h"
|
|
||||||
# reddit = "15m"
|
|
||||||
# braveapi = "2h"
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add config.example.toml
|
|
||||||
git commit -m "config: add cache.ttl_overrides example"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Verification
|
|
||||||
|
|
||||||
After all tasks complete, run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
nix develop --command bash -c "go test ./... -v 2>&1 | tail -50"
|
|
||||||
```
|
|
||||||
|
|
||||||
All tests should pass. The search service should now cache each engine's response independently with tier-based TTLs.
|
|
||||||
|
|
@ -1,219 +0,0 @@
|
||||||
# Per-Engine TTL Cache — Design
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Replace the current merged-response cache with a per-engine response cache. Each engine's raw response is cached independently with a tier-based TTL, enabling stale-while-revalidate semantics and more granular freshness control.
|
|
||||||
|
|
||||||
## Cache Key Structure
|
|
||||||
|
|
||||||
```
|
|
||||||
samsa:resp:{engine}:{query_hash}
|
|
||||||
```
|
|
||||||
|
|
||||||
Where `query_hash` = SHA-256 of shared request params (query, pageno, safesearch, language, time_range), truncated to 16 hex chars.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
- `samsa:resp:wikipedia:a3f1b2c3d4e5f678`
|
|
||||||
- `samsa:resp:duckduckgo:a3f1b2c3d4e5f678`
|
|
||||||
|
|
||||||
The same query to Wikipedia and DuckDuckGo produce different cache keys, enabling independent TTLs per engine.
|
|
||||||
|
|
||||||
## Query Hash
|
|
||||||
|
|
||||||
Compute from shared request parameters:
|
|
||||||
|
|
||||||
```go
|
|
||||||
func QueryHash(query string, pageno int, safesearch int, language, timeRange string) string {
|
|
||||||
h := sha256.New()
|
|
||||||
fmt.Fprintf(h, "q=%s|", query)
|
|
||||||
fmt.Fprintf(h, "pageno=%d|", pageno)
|
|
||||||
fmt.Fprintf(h, "safesearch=%d|", safesearch)
|
|
||||||
fmt.Fprintf(h, "lang=%s|", language)
|
|
||||||
if timeRange != "" {
|
|
||||||
fmt.Fprintf(h, "tr=%s|", timeRange)
|
|
||||||
}
|
|
||||||
return hex.EncodeToString(h.Sum(nil))[:16]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Note: `engines` is NOT included because each engine has its own cache key prefix.
|
|
||||||
|
|
||||||
## Cached Data Format
|
|
||||||
|
|
||||||
Each cache entry stores:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type CachedEngineResponse struct {
|
|
||||||
Engine string // engine name
|
|
||||||
Response []byte // JSON-marshaled contracts.SearchResponse
|
|
||||||
StoredAt time.Time // when cached (for staleness check)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## TTL Tiers
|
|
||||||
|
|
||||||
### Default Tier Assignments
|
|
||||||
|
|
||||||
| Tier | Engines | Default TTL |
|
|
||||||
|------|---------|-------------|
|
|
||||||
| `static` | wikipedia, wikidata, arxiv, crossref, stackoverflow, github | 24h |
|
|
||||||
| `api_general` | braveapi, youtube | 1h |
|
|
||||||
| `scraped_general` | google, bing, duckduckgo, qwant, brave | 2h |
|
|
||||||
| `news_social` | reddit | 30m |
|
|
||||||
| `images` | bing_images, ddg_images, qwant_images | 1h |
|
|
||||||
|
|
||||||
### TOML Override Format
|
|
||||||
|
|
||||||
```toml
|
|
||||||
[cache.ttl_overrides]
|
|
||||||
wikipedia = "48h" # override default 24h
|
|
||||||
reddit = "15m" # override default 30m
|
|
||||||
```
|
|
||||||
|
|
||||||
## Search Flow
|
|
||||||
|
|
||||||
### 1. Parse Request
|
|
||||||
Extract engine list from planner, compute shared `queryHash`.
|
|
||||||
|
|
||||||
### 2. Parallel Cache Lookups
|
|
||||||
For each engine, spawn a goroutine to check cache:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type engineCacheResult struct {
|
|
||||||
engine string
|
|
||||||
resp contracts.SearchResponse
|
|
||||||
fromCache bool
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
// For each engine, concurrently:
|
|
||||||
cached, hit := engineCache.Get(ctx, engine, queryHash)
|
|
||||||
if hit && !isStale(cached) {
|
|
||||||
return cached.Response, nil // fresh cache hit
|
|
||||||
}
|
|
||||||
if hit && isStale(cached) {
|
|
||||||
go refreshInBackground(engine, queryHash) // stale-while-revalidate
|
|
||||||
return cached.Response, nil // return stale immediately
|
|
||||||
}
|
|
||||||
// cache miss
|
|
||||||
fresh, err := engine.Search(ctx, req)
|
|
||||||
engineCache.Set(ctx, engine, queryHash, fresh)
|
|
||||||
return fresh, err
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Classify Each Engine
|
|
||||||
- **Cache miss** → fetch fresh immediately
|
|
||||||
- **Cache hit, fresh** → use cached
|
|
||||||
- **Cache hit, stale** → use cached, fetch fresh in background (stale-while-revalidate)
|
|
||||||
|
|
||||||
### 4. Background Refresh
|
|
||||||
When a stale cache hit occurs:
|
|
||||||
1. Return stale data immediately
|
|
||||||
2. Spawn goroutine to fetch fresh data
|
|
||||||
3. On success, overwrite cache with fresh data
|
|
||||||
4. On failure, log and discard (stale data already returned)
|
|
||||||
|
|
||||||
### 5. Merge
|
|
||||||
Collect all engine responses (cached + fresh), merge via existing `MergeResponses`.
|
|
||||||
|
|
||||||
### 6. Write Fresh to Cache
|
|
||||||
For engines that were fetched fresh, write to cache with their tier TTL.
|
|
||||||
|
|
||||||
## Staleness Check
|
|
||||||
|
|
||||||
```go
|
|
||||||
func isStale(cached CachedEngineResponse, tier TTLTier) bool {
|
|
||||||
return time.Since(cached.StoredAt) > tier.Duration
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Tier Resolution
|
|
||||||
|
|
||||||
```go
|
|
||||||
type TTLTier struct {
|
|
||||||
Name string
|
|
||||||
Duration time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
func EngineTier(engineName string) TTLTier {
|
|
||||||
if override := ttlOverrides[engineName]; override > 0 {
|
|
||||||
return TTLTier{Name: engineName, Duration: override}
|
|
||||||
}
|
|
||||||
return defaultTiers[engineName] // from hardcoded map above
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## New Files
|
|
||||||
|
|
||||||
### `internal/cache/engine_cache.go`
|
|
||||||
`EngineCache` struct wrapping `*Cache` with tier-aware `Get/Set` methods:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type EngineCache struct {
|
|
||||||
cache *Cache
|
|
||||||
overrides map[string]time.Duration
|
|
||||||
tiers map[string]TTLTier
|
|
||||||
}
|
|
||||||
|
|
||||||
func (ec *EngineCache) Get(ctx context.Context, engine, queryHash string) (CachedEngineResponse, bool)
|
|
||||||
func (ec *EngineCache) Set(ctx context.Context, engine, queryHash string, resp contracts.SearchResponse)
|
|
||||||
```
|
|
||||||
|
|
||||||
### `internal/cache/tiers.go`
|
|
||||||
Tier definitions and `EngineTier(engineName string)` function.
|
|
||||||
|
|
||||||
## Modified Files
|
|
||||||
|
|
||||||
### `internal/cache/cache.go`
|
|
||||||
- Rename `Key()` to `QueryHash()` and add `Engine` prefix externally
|
|
||||||
- `Get/Set` remain for favicon caching (unchanged)
|
|
||||||
|
|
||||||
### `internal/search/service.go`
|
|
||||||
- Replace `*Cache` with `*EngineCache`
|
|
||||||
- Parallel cache lookups with goroutines
|
|
||||||
- Stale-while-revalidate background refresh
|
|
||||||
- Merge collected responses
|
|
||||||
|
|
||||||
### `internal/config/config.go`
|
|
||||||
Add `TTLOverrides` field:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type CacheConfig struct {
|
|
||||||
// ... existing fields ...
|
|
||||||
TTLOverrides map[string]time.Duration
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Config Example
|
|
||||||
|
|
||||||
```toml
|
|
||||||
[cache]
|
|
||||||
enabled = true
|
|
||||||
url = "valkey://localhost:6379/0"
|
|
||||||
default_ttl = "5m"
|
|
||||||
|
|
||||||
[cache.ttl_overrides]
|
|
||||||
wikipedia = "48h"
|
|
||||||
reddit = "15m"
|
|
||||||
braveapi = "2h"
|
|
||||||
```
|
|
||||||
|
|
||||||
## Error Handling
|
|
||||||
|
|
||||||
- **Cache read failure**: Treat as cache miss, fetch fresh
|
|
||||||
- **Cache write failure**: Log warning, continue without caching for that engine
|
|
||||||
- **Background refresh failure**: Log error, discard (stale data already returned)
|
|
||||||
- **Engine failure**: Continue with other engines, report in `unresponsive_engines`
|
|
||||||
|
|
||||||
## Testing
|
|
||||||
|
|
||||||
1. **Unit tests** for `QueryHash()` consistency
|
|
||||||
2. **Unit tests** for `EngineTier()` with overrides
|
|
||||||
3. **Unit tests** for `isStale()` boundary conditions
|
|
||||||
4. **Integration tests** for cache hit/miss/stale scenarios using mock Valkey
|
|
||||||
|
|
||||||
## Out of Scope
|
|
||||||
|
|
||||||
- Cache invalidation API (future work)
|
|
||||||
- Dog-pile prevention (future work)
|
|
||||||
- Per-engine cache size limits (future work)
|
|
||||||
|
|
@ -21,16 +21,13 @@
|
||||||
version = "0.1.0";
|
version = "0.1.0";
|
||||||
src = ./.;
|
src = ./.;
|
||||||
|
|
||||||
vendorHash = "sha256-8wlKD+33s97oorCJTfHKAgE2Xp1HKXV+bSr6z29KrKM=";
|
vendorHash = "sha256-NbAa4QM/TI3BTuZs4glx9k3ZjSl2/2LQfKlQ7izR8Ho=";
|
||||||
# Run: nix build .#packages.x86_64-linux.default
|
# Run: nix build .#packages.x86_64-linux.default
|
||||||
# It will fail with the correct hash. Replace vendorHash with it.
|
# It will fail with the correct hash. Replace it here.
|
||||||
|
|
||||||
# Embed the templates and static files at build time.
|
# Embed the templates and static files at build time.
|
||||||
ldflags = [ "-s" "-w" ];
|
ldflags = [ "-s" "-w" ];
|
||||||
|
|
||||||
# Remove stale vendor directory before buildGoModule deletes it.
|
|
||||||
preConfigure = "rm -rf vendor || true";
|
|
||||||
|
|
||||||
nativeCheckInputs = with pkgs; [ ];
|
nativeCheckInputs = with pkgs; [ ];
|
||||||
|
|
||||||
# Tests require network; they run in CI instead.
|
# Tests require network; they run in CI instead.
|
||||||
|
|
@ -61,7 +58,7 @@
|
||||||
|
|
||||||
port = lib.mkOption {
|
port = lib.mkOption {
|
||||||
type = lib.types.port;
|
type = lib.types.port;
|
||||||
default = 5355;
|
default = 8080;
|
||||||
description = "Port to listen on.";
|
description = "Port to listen on.";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
10
go.mod
10
go.mod
|
|
@ -1,10 +1,10 @@
|
||||||
module github.com/metamorphosis-dev/samsa
|
module github.com/metamorphosis-dev/kafka
|
||||||
|
|
||||||
go 1.24
|
go 1.25.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/BurntSushi/toml v1.5.0
|
github.com/BurntSushi/toml v1.5.0
|
||||||
github.com/PuerkitoBio/goquery v1.9.0
|
github.com/PuerkitoBio/goquery v1.12.0
|
||||||
github.com/redis/go-redis/v9 v9.18.0
|
github.com/redis/go-redis/v9 v9.18.0
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -13,7 +13,5 @@ require (
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||||
go.uber.org/atomic v1.11.0 // indirect
|
go.uber.org/atomic v1.11.0 // indirect
|
||||||
golang.org/x/net v0.33.0 // indirect
|
golang.org/x/net v0.52.0 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
replace golang.org/x/net => golang.org/x/net v0.38.0
|
|
||||||
|
|
|
||||||
50
go.sum
50
go.sum
|
|
@ -1,7 +1,7 @@
|
||||||
github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
|
github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
|
||||||
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
||||||
github.com/PuerkitoBio/goquery v1.9.0 h1:zgjKkdpRY9T97Q5DCtcXwfqkcylSFIVCocZmn2huTp8=
|
github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo=
|
||||||
github.com/PuerkitoBio/goquery v1.9.0/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY=
|
github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
|
||||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||||
|
|
@ -28,36 +28,68 @@ github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
|
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||||
|
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||||
|
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||||
|
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
|
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||||
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
|
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||||
|
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||||
|
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||||
|
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||||
|
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||||
|
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||||
|
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||||
|
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||||
|
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||||
golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
|
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||||
|
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||||
|
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||||
|
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||||
|
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||||
|
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||||
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
|
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||||
|
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||||
|
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||||
|
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||||
|
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||||
|
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||||
|
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 52 KiB |
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package autocomplete
|
package autocomplete
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -25,11 +9,10 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/httpclient"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Service fetches search suggestions from upstream or Wikipedia OpenSearch.
|
// Service fetches search suggestions from an upstream SearXNG instance
|
||||||
|
// or falls back to Wikipedia's OpenSearch API.
|
||||||
type Service struct {
|
type Service struct {
|
||||||
upstreamURL string
|
upstreamURL string
|
||||||
http *http.Client
|
http *http.Client
|
||||||
|
|
@ -41,10 +24,11 @@ func NewService(upstreamURL string, timeout time.Duration) *Service {
|
||||||
}
|
}
|
||||||
return &Service{
|
return &Service{
|
||||||
upstreamURL: strings.TrimRight(upstreamURL, "/"),
|
upstreamURL: strings.TrimRight(upstreamURL, "/"),
|
||||||
http: httpclient.NewClient(timeout),
|
http: &http.Client{Timeout: timeout},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Suggestions returns search suggestions for the given query.
|
||||||
func (s *Service) Suggestions(ctx context.Context, query string) ([]string, error) {
|
func (s *Service) Suggestions(ctx context.Context, query string) ([]string, error) {
|
||||||
if strings.TrimSpace(query) == "" {
|
if strings.TrimSpace(query) == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
|
@ -56,6 +40,7 @@ func (s *Service) Suggestions(ctx context.Context, query string) ([]string, erro
|
||||||
return s.wikipediaSuggestions(ctx, query)
|
return s.wikipediaSuggestions(ctx, query)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// upstreamSuggestions proxies to an upstream SearXNG /autocompleter endpoint.
|
||||||
func (s *Service) upstreamSuggestions(ctx context.Context, query string) ([]string, error) {
|
func (s *Service) upstreamSuggestions(ctx context.Context, query string) ([]string, error) {
|
||||||
u := s.upstreamURL + "/autocompleter?" + url.Values{"q": {query}}.Encode()
|
u := s.upstreamURL + "/autocompleter?" + url.Values{"q": {query}}.Encode()
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||||
|
|
@ -79,7 +64,7 @@ func (s *Service) upstreamSuggestions(ctx context.Context, query string) ([]stri
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// The /autocompleter endpoint returns a plain JSON array of strings.
|
// SearXNG /autocompleter returns a plain JSON array of strings.
|
||||||
var out []string
|
var out []string
|
||||||
if err := json.Unmarshal(body, &out); err != nil {
|
if err := json.Unmarshal(body, &out); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
@ -104,7 +89,7 @@ func (s *Service) wikipediaSuggestions(ctx context.Context, query string) ([]str
|
||||||
}
|
}
|
||||||
req.Header.Set(
|
req.Header.Set(
|
||||||
"User-Agent",
|
"User-Agent",
|
||||||
"gosearch-go/0.1 (compatible; +https://github.com/metamorphosis-dev/samsa)",
|
"gosearch-go/0.1 (compatible; +https://github.com/metamorphosis-dev/kafka)",
|
||||||
)
|
)
|
||||||
|
|
||||||
resp, err := s.http.Do(req)
|
resp, err := s.http.Do(req)
|
||||||
|
|
|
||||||
78
internal/cache/cache.go
vendored
78
internal/cache/cache.go
vendored
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package cache
|
package cache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -25,7 +9,7 @@ import (
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
"github.com/redis/go-redis/v9"
|
"github.com/redis/go-redis/v9"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -97,7 +81,7 @@ func (c *Cache) Get(ctx context.Context, key string) (contracts.SearchResponse,
|
||||||
return contracts.SearchResponse{}, false
|
return contracts.SearchResponse{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
fullKey := "samsa:" + key
|
fullKey := "kafka:" + key
|
||||||
|
|
||||||
data, err := c.client.Get(ctx, fullKey).Bytes()
|
data, err := c.client.Get(ctx, fullKey).Bytes()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -129,7 +113,7 @@ func (c *Cache) Set(ctx context.Context, key string, resp contracts.SearchRespon
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
fullKey := "samsa:" + key
|
fullKey := "kafka:" + key
|
||||||
if err := c.client.Set(ctx, fullKey, data, c.ttl).Err(); err != nil {
|
if err := c.client.Set(ctx, fullKey, data, c.ttl).Err(); err != nil {
|
||||||
c.logger.Warn("cache set failed", "key", fullKey, "error", err)
|
c.logger.Warn("cache set failed", "key", fullKey, "error", err)
|
||||||
}
|
}
|
||||||
|
|
@ -140,42 +124,10 @@ func (c *Cache) Invalidate(ctx context.Context, key string) {
|
||||||
if !c.Enabled() {
|
if !c.Enabled() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fullKey := "samsa:" + key
|
fullKey := "kafka:" + key
|
||||||
c.client.Del(ctx, fullKey)
|
c.client.Del(ctx, fullKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetBytes retrieves a raw byte slice from the cache. Returns (data, true) on hit,
|
|
||||||
// (nil, false) on miss or error.
|
|
||||||
func (c *Cache) GetBytes(ctx context.Context, key string) ([]byte, bool) {
|
|
||||||
if !c.Enabled() {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
fullKey := "samsa:" + key
|
|
||||||
data, err := c.client.Get(ctx, fullKey).Bytes()
|
|
||||||
if err != nil {
|
|
||||||
if err != redis.Nil {
|
|
||||||
c.logger.Debug("cache bytes miss (error)", "key", fullKey, "error", err)
|
|
||||||
}
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
return data, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetBytes stores a raw byte slice with a custom TTL.
|
|
||||||
// If ttl <= 0, the cache's default TTL is used.
|
|
||||||
func (c *Cache) SetBytes(ctx context.Context, key string, data []byte, ttl time.Duration) {
|
|
||||||
if !c.Enabled() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if ttl <= 0 {
|
|
||||||
ttl = c.ttl
|
|
||||||
}
|
|
||||||
fullKey := "samsa:" + key
|
|
||||||
if err := c.client.Set(ctx, fullKey, data, ttl).Err(); err != nil {
|
|
||||||
c.logger.Warn("cache set bytes failed", "key", fullKey, "error", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close closes the Valkey connection.
|
// Close closes the Valkey connection.
|
||||||
func (c *Cache) Close() error {
|
func (c *Cache) Close() error {
|
||||||
if c.client == nil {
|
if c.client == nil {
|
||||||
|
|
@ -208,25 +160,3 @@ func Key(req contracts.SearchRequest) string {
|
||||||
|
|
||||||
return hex.EncodeToString(h.Sum(nil))[:32]
|
return hex.EncodeToString(h.Sum(nil))[:32]
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueryHash computes a deterministic hash from shared request parameters
|
|
||||||
// (query, pageno, safesearch, language, time_range) for use as a cache key suffix.
|
|
||||||
// The hash is a truncated SHA-256 (16 hex chars).
|
|
||||||
func QueryHash(query string, pageno int, safesearch int, language, timeRange string) string {
|
|
||||||
h := sha256.New()
|
|
||||||
fmt.Fprintf(h, "q=%s|", query)
|
|
||||||
fmt.Fprintf(h, "pageno=%d|", pageno)
|
|
||||||
fmt.Fprintf(h, "safesearch=%d|", safesearch)
|
|
||||||
fmt.Fprintf(h, "lang=%s|", language)
|
|
||||||
if timeRange != "" {
|
|
||||||
fmt.Fprintf(h, "tr=%s|", timeRange)
|
|
||||||
}
|
|
||||||
return hex.EncodeToString(h.Sum(nil))[:16]
|
|
||||||
}
|
|
||||||
|
|
||||||
// CachedEngineResponse wraps an engine's cached response with metadata.
|
|
||||||
type CachedEngineResponse struct {
|
|
||||||
Engine string
|
|
||||||
Response []byte
|
|
||||||
StoredAt time.Time
|
|
||||||
}
|
|
||||||
|
|
|
||||||
38
internal/cache/cache_test.go
vendored
38
internal/cache/cache_test.go
vendored
|
|
@ -3,13 +3,13 @@ package cache
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestKey_Deterministic(t *testing.T) {
|
func TestKey_Deterministic(t *testing.T) {
|
||||||
req := contracts.SearchRequest{
|
req := contracts.SearchRequest{
|
||||||
Format: contracts.FormatJSON,
|
Format: contracts.FormatJSON,
|
||||||
Query: "samsa metamorphosis",
|
Query: "kafka metamorphosis",
|
||||||
Pageno: 1,
|
Pageno: 1,
|
||||||
Safesearch: 0,
|
Safesearch: 0,
|
||||||
Language: "auto",
|
Language: "auto",
|
||||||
|
|
@ -29,7 +29,7 @@ func TestKey_Deterministic(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestKey_DifferentQueries(t *testing.T) {
|
func TestKey_DifferentQueries(t *testing.T) {
|
||||||
reqA := contracts.SearchRequest{Query: "samsa", Format: contracts.FormatJSON}
|
reqA := contracts.SearchRequest{Query: "kafka", Format: contracts.FormatJSON}
|
||||||
reqB := contracts.SearchRequest{Query: "orwell", Format: contracts.FormatJSON}
|
reqB := contracts.SearchRequest{Query: "orwell", Format: contracts.FormatJSON}
|
||||||
|
|
||||||
if Key(reqA) == Key(reqB) {
|
if Key(reqA) == Key(reqB) {
|
||||||
|
|
@ -75,35 +75,3 @@ func TestNew_NopWithoutAddress(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func strPtr(s string) *string { return &s }
|
func strPtr(s string) *string { return &s }
|
||||||
|
|
||||||
func TestQueryHash(t *testing.T) {
|
|
||||||
// Same params should produce same hash
|
|
||||||
hash1 := QueryHash("golang", 1, 0, "en", "")
|
|
||||||
hash2 := QueryHash("golang", 1, 0, "en", "")
|
|
||||||
if hash1 != hash2 {
|
|
||||||
t.Errorf("QueryHash: same params should produce same hash, got %s != %s", hash1, hash2)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Different query should produce different hash
|
|
||||||
hash3 := QueryHash("rust", 1, 0, "en", "")
|
|
||||||
if hash1 == hash3 {
|
|
||||||
t.Errorf("QueryHash: different queries should produce different hash")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Different pageno should produce different hash
|
|
||||||
hash4 := QueryHash("golang", 2, 0, "en", "")
|
|
||||||
if hash1 == hash4 {
|
|
||||||
t.Errorf("QueryHash: different pageno should produce different hash")
|
|
||||||
}
|
|
||||||
|
|
||||||
// time_range should affect hash
|
|
||||||
hash5 := QueryHash("golang", 1, 0, "en", "day")
|
|
||||||
if hash1 == hash5 {
|
|
||||||
t.Errorf("QueryHash: different time_range should produce different hash")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Hash should be 16 characters (truncated SHA-256)
|
|
||||||
if len(hash1) != 16 {
|
|
||||||
t.Errorf("QueryHash: expected 16 char hash, got %d", len(hash1))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
91
internal/cache/engine_cache.go
vendored
91
internal/cache/engine_cache.go
vendored
|
|
@ -1,91 +0,0 @@
|
||||||
package cache
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"log/slog"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
// EngineCache wraps Cache with per-engine tier-aware Get/Set operations.
|
|
||||||
type EngineCache struct {
|
|
||||||
cache *Cache
|
|
||||||
overrides map[string]time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewEngineCache creates a new EngineCache with optional TTL overrides.
|
|
||||||
// If overrides is nil, default tier durations are used.
|
|
||||||
func NewEngineCache(cache *Cache, overrides map[string]time.Duration) *EngineCache {
|
|
||||||
return &EngineCache{
|
|
||||||
cache: cache,
|
|
||||||
overrides: overrides,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get retrieves a cached engine response. Returns (zero value, false) if not
|
|
||||||
// found or if cache is disabled.
|
|
||||||
func (ec *EngineCache) Get(ctx context.Context, engine, queryHash string) (CachedEngineResponse, bool) {
|
|
||||||
key := engineCacheKey(engine, queryHash)
|
|
||||||
|
|
||||||
data, ok := ec.cache.GetBytes(ctx, key)
|
|
||||||
if !ok {
|
|
||||||
return CachedEngineResponse{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
var cached CachedEngineResponse
|
|
||||||
if err := json.Unmarshal(data, &cached); err != nil {
|
|
||||||
ec.cache.logger.Warn("engine cache hit but unmarshal failed", "key", key, "error", err)
|
|
||||||
return CachedEngineResponse{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
ec.cache.logger.Debug("engine cache hit", "key", key, "engine", engine)
|
|
||||||
return cached, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set stores an engine response in the cache with the engine's tier TTL.
|
|
||||||
func (ec *EngineCache) Set(ctx context.Context, engine, queryHash string, resp contracts.SearchResponse) {
|
|
||||||
if !ec.cache.Enabled() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := json.Marshal(resp)
|
|
||||||
if err != nil {
|
|
||||||
ec.cache.logger.Warn("engine cache set: marshal failed", "engine", engine, "error", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
tier := EngineTier(engine, ec.overrides)
|
|
||||||
key := engineCacheKey(engine, queryHash)
|
|
||||||
|
|
||||||
cached := CachedEngineResponse{
|
|
||||||
Engine: engine,
|
|
||||||
Response: data,
|
|
||||||
StoredAt: time.Now(),
|
|
||||||
}
|
|
||||||
|
|
||||||
cachedData, err := json.Marshal(cached)
|
|
||||||
if err != nil {
|
|
||||||
ec.cache.logger.Warn("engine cache set: wrap marshal failed", "key", key, "error", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
ec.cache.SetBytes(ctx, key, cachedData, tier.Duration)
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsStale returns true if the cached response is older than the tier's TTL.
|
|
||||||
func (ec *EngineCache) IsStale(cached CachedEngineResponse, engine string) bool {
|
|
||||||
tier := EngineTier(engine, ec.overrides)
|
|
||||||
return time.Since(cached.StoredAt) > tier.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// Logger returns the logger for background refresh logging.
|
|
||||||
func (ec *EngineCache) Logger() *slog.Logger {
|
|
||||||
return ec.cache.logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// engineCacheKey builds the cache key for an engine+query combination.
|
|
||||||
func engineCacheKey(engine, queryHash string) string {
|
|
||||||
return "samsa:resp:" + engine + ":" + queryHash
|
|
||||||
}
|
|
||||||
95
internal/cache/engine_cache_test.go
vendored
95
internal/cache/engine_cache_test.go
vendored
|
|
@ -1,95 +0,0 @@
|
||||||
package cache
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestEngineCacheGetSet(t *testing.T) {
|
|
||||||
// Create a disabled cache for unit testing (nil client)
|
|
||||||
c := &Cache{logger: slog.Default()}
|
|
||||||
ec := NewEngineCache(c, nil)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
cached, ok := ec.Get(ctx, "wikipedia", "abc123")
|
|
||||||
if ok {
|
|
||||||
t.Errorf("Get on disabled cache: expected false, got %v", ok)
|
|
||||||
}
|
|
||||||
_ = cached // unused when ok=false
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEngineCacheKeyFormat(t *testing.T) {
|
|
||||||
key := engineCacheKey("wikipedia", "abc123")
|
|
||||||
if key != "samsa:resp:wikipedia:abc123" {
|
|
||||||
t.Errorf("engineCacheKey: expected samsa:resp:wikipedia:abc123, got %s", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEngineCacheIsStale(t *testing.T) {
|
|
||||||
c := &Cache{logger: slog.Default()}
|
|
||||||
ec := NewEngineCache(c, nil)
|
|
||||||
|
|
||||||
// Fresh response (stored 1 minute ago, wikipedia has 24h TTL)
|
|
||||||
fresh := CachedEngineResponse{
|
|
||||||
Engine: "wikipedia",
|
|
||||||
Response: []byte(`{}`),
|
|
||||||
StoredAt: time.Now().Add(-1 * time.Minute),
|
|
||||||
}
|
|
||||||
if ec.IsStale(fresh, "wikipedia") {
|
|
||||||
t.Errorf("IsStale: 1-minute-old wikipedia should NOT be stale")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stale response (stored 25 hours ago)
|
|
||||||
stale := CachedEngineResponse{
|
|
||||||
Engine: "wikipedia",
|
|
||||||
Response: []byte(`{}`),
|
|
||||||
StoredAt: time.Now().Add(-25 * time.Hour),
|
|
||||||
}
|
|
||||||
if !ec.IsStale(stale, "wikipedia") {
|
|
||||||
t.Errorf("IsStale: 25-hour-old wikipedia SHOULD be stale (24h TTL)")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Override: 30 minute TTL for reddit
|
|
||||||
overrides := map[string]time.Duration{"reddit": 30 * time.Minute}
|
|
||||||
ec2 := NewEngineCache(c, overrides)
|
|
||||||
|
|
||||||
// 20 minutes old with 30m override should NOT be stale
|
|
||||||
redditFresh := CachedEngineResponse{
|
|
||||||
Engine: "reddit",
|
|
||||||
Response: []byte(`{}`),
|
|
||||||
StoredAt: time.Now().Add(-20 * time.Minute),
|
|
||||||
}
|
|
||||||
if ec2.IsStale(redditFresh, "reddit") {
|
|
||||||
t.Errorf("IsStale: 20-min reddit with 30m override should NOT be stale")
|
|
||||||
}
|
|
||||||
|
|
||||||
// 45 minutes old with 30m override SHOULD be stale
|
|
||||||
redditStale := CachedEngineResponse{
|
|
||||||
Engine: "reddit",
|
|
||||||
Response: []byte(`{}`),
|
|
||||||
StoredAt: time.Now().Add(-45 * time.Minute),
|
|
||||||
}
|
|
||||||
if !ec2.IsStale(redditStale, "reddit") {
|
|
||||||
t.Errorf("IsStale: 45-min reddit with 30m override SHOULD be stale")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEngineCacheSetResponseType(t *testing.T) {
|
|
||||||
c := &Cache{logger: slog.Default()}
|
|
||||||
ec := NewEngineCache(c, nil)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
urlStr := "https://example.com"
|
|
||||||
resp := contracts.SearchResponse{
|
|
||||||
Results: []contracts.MainResult{
|
|
||||||
{Title: "Test", URL: &urlStr},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// Should not panic on disabled cache
|
|
||||||
ec.Set(ctx, "wikipedia", "abc123", resp)
|
|
||||||
}
|
|
||||||
56
internal/cache/tiers.go
vendored
56
internal/cache/tiers.go
vendored
|
|
@ -1,56 +0,0 @@
|
||||||
package cache
|
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
// TTLTier represents a cache TTL tier with a name and duration.
|
|
||||||
type TTLTier struct {
|
|
||||||
Name string
|
|
||||||
Duration time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// defaultTiers maps engine names to their default TTL tiers.
|
|
||||||
var defaultTiers = map[string]TTLTier{
|
|
||||||
// Static knowledge engines — rarely change
|
|
||||||
"wikipedia": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
"wikidata": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
"arxiv": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
"crossref": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
"stackoverflow": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
"github": {Name: "static", Duration: 24 * time.Hour},
|
|
||||||
|
|
||||||
// API-based general search — fresher data
|
|
||||||
"braveapi": {Name: "api_general", Duration: 1 * time.Hour},
|
|
||||||
"youtube": {Name: "api_general", Duration: 1 * time.Hour},
|
|
||||||
|
|
||||||
// Scraped general search — moderately stable
|
|
||||||
"google": {Name: "scraped_general", Duration: 2 * time.Hour},
|
|
||||||
"bing": {Name: "scraped_general", Duration: 2 * time.Hour},
|
|
||||||
"duckduckgo": {Name: "scraped_general", Duration: 2 * time.Hour},
|
|
||||||
"qwant": {Name: "scraped_general", Duration: 2 * time.Hour},
|
|
||||||
"brave": {Name: "scraped_general", Duration: 2 * time.Hour},
|
|
||||||
|
|
||||||
// News/social — changes frequently
|
|
||||||
"reddit": {Name: "news_social", Duration: 30 * time.Minute},
|
|
||||||
|
|
||||||
// Image search
|
|
||||||
"bing_images": {Name: "images", Duration: 1 * time.Hour},
|
|
||||||
"ddg_images": {Name: "images", Duration: 1 * time.Hour},
|
|
||||||
"qwant_images": {Name: "images", Duration: 1 * time.Hour},
|
|
||||||
}
|
|
||||||
|
|
||||||
// EngineTier returns the TTL tier for an engine, applying overrides if provided.
|
|
||||||
// If the engine has no defined tier, returns a default of 1 hour.
|
|
||||||
func EngineTier(engineName string, overrides map[string]time.Duration) TTLTier {
|
|
||||||
// Check override first — override tier name is just the engine name
|
|
||||||
if override, ok := overrides[engineName]; ok && override > 0 {
|
|
||||||
return TTLTier{Name: engineName, Duration: override}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fall back to default tier
|
|
||||||
if tier, ok := defaultTiers[engineName]; ok {
|
|
||||||
return tier
|
|
||||||
}
|
|
||||||
|
|
||||||
// Unknown engines get a sensible default
|
|
||||||
return TTLTier{Name: "unknown", Duration: 1 * time.Hour}
|
|
||||||
}
|
|
||||||
33
internal/cache/tiers_test.go
vendored
33
internal/cache/tiers_test.go
vendored
|
|
@ -1,33 +0,0 @@
|
||||||
package cache
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestEngineTier(t *testing.T) {
|
|
||||||
// Test default static tier
|
|
||||||
tier := EngineTier("wikipedia", nil)
|
|
||||||
if tier.Name != "static" || tier.Duration != 24*time.Hour {
|
|
||||||
t.Errorf("wikipedia: expected static/24h, got %s/%v", tier.Name, tier.Duration)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test default api_general tier
|
|
||||||
tier = EngineTier("braveapi", nil)
|
|
||||||
if tier.Name != "api_general" || tier.Duration != 1*time.Hour {
|
|
||||||
t.Errorf("braveapi: expected api_general/1h, got %s/%v", tier.Name, tier.Duration)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test override takes precedence — override tier name is just the engine name
|
|
||||||
override := 48 * time.Hour
|
|
||||||
tier = EngineTier("wikipedia", map[string]time.Duration{"wikipedia": override})
|
|
||||||
if tier.Name != "wikipedia" || tier.Duration != 48*time.Hour {
|
|
||||||
t.Errorf("wikipedia override: expected wikipedia/48h, got %s/%v", tier.Name, tier.Duration)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test unknown engine gets default
|
|
||||||
tier = EngineTier("unknown_engine", nil)
|
|
||||||
if tier.Name != "unknown" || tier.Duration != 1*time.Hour {
|
|
||||||
t.Errorf("unknown engine: expected unknown/1h, got %s/%v", tier.Name, tier.Duration)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -23,10 +7,9 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/BurntSushi/toml"
|
"github.com/BurntSushi/toml"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/util"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config is the top-level configuration for the samsa service.
|
// Config is the top-level configuration for the kafka service.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Server ServerConfig `toml:"server"`
|
Server ServerConfig `toml:"server"`
|
||||||
Upstream UpstreamConfig `toml:"upstream"`
|
Upstream UpstreamConfig `toml:"upstream"`
|
||||||
|
|
@ -42,7 +25,6 @@ type ServerConfig struct {
|
||||||
Port int `toml:"port"`
|
Port int `toml:"port"`
|
||||||
HTTPTimeout string `toml:"http_timeout"`
|
HTTPTimeout string `toml:"http_timeout"`
|
||||||
BaseURL string `toml:"base_url"` // Public URL for OpenSearch XML (e.g. "https://search.example.com")
|
BaseURL string `toml:"base_url"` // Public URL for OpenSearch XML (e.g. "https://search.example.com")
|
||||||
SourceURL string `toml:"source_url"` // Link to the source code (e.g. "https://git.example.com/fork/kafka")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type UpstreamConfig struct {
|
type UpstreamConfig struct {
|
||||||
|
|
@ -53,12 +35,6 @@ type EnginesConfig struct {
|
||||||
LocalPorted []string `toml:"local_ported"`
|
LocalPorted []string `toml:"local_ported"`
|
||||||
Brave BraveConfig `toml:"brave"`
|
Brave BraveConfig `toml:"brave"`
|
||||||
Qwant QwantConfig `toml:"qwant"`
|
Qwant QwantConfig `toml:"qwant"`
|
||||||
YouTube YouTubeConfig `toml:"youtube"`
|
|
||||||
StackOverflow *StackOverflowConfig `toml:"stackoverflow"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type StackOverflowConfig struct {
|
|
||||||
APIKey string `toml:"api_key"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CacheConfig holds Valkey/Redis cache settings.
|
// CacheConfig holds Valkey/Redis cache settings.
|
||||||
|
|
@ -67,7 +43,6 @@ type CacheConfig struct {
|
||||||
Password string `toml:"password"` // Auth password (empty = none)
|
Password string `toml:"password"` // Auth password (empty = none)
|
||||||
DB int `toml:"db"` // Database index (default 0)
|
DB int `toml:"db"` // Database index (default 0)
|
||||||
DefaultTTL string `toml:"default_ttl"` // Cache TTL (e.g. "5m", default "5m")
|
DefaultTTL string `toml:"default_ttl"` // Cache TTL (e.g. "5m", default "5m")
|
||||||
TTLOverrides map[string]string `toml:"ttl_overrides"` // engine -> duration string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CORSConfig holds CORS middleware settings.
|
// CORSConfig holds CORS middleware settings.
|
||||||
|
|
@ -84,7 +59,6 @@ type RateLimitConfig struct {
|
||||||
Requests int `toml:"requests"` // Max requests per window (default: 30)
|
Requests int `toml:"requests"` // Max requests per window (default: 30)
|
||||||
Window string `toml:"window"` // Time window (e.g. "1m", default: "1m")
|
Window string `toml:"window"` // Time window (e.g. "1m", default: "1m")
|
||||||
CleanupInterval string `toml:"cleanup_interval"` // Stale entry cleanup interval (default: "5m")
|
CleanupInterval string `toml:"cleanup_interval"` // Stale entry cleanup interval (default: "5m")
|
||||||
TrustedProxies []string `toml:"trusted_proxies"` // CIDRs allowed to set X-Forwarded-For
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GlobalRateLimitConfig holds server-wide rate limiting settings.
|
// GlobalRateLimitConfig holds server-wide rate limiting settings.
|
||||||
|
|
@ -111,10 +85,6 @@ type QwantConfig struct {
|
||||||
ResultsPerPage int `toml:"results_per_page"`
|
ResultsPerPage int `toml:"results_per_page"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type YouTubeConfig struct {
|
|
||||||
APIKey string `toml:"api_key"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load reads configuration from the given TOML file path.
|
// Load reads configuration from the given TOML file path.
|
||||||
// If the file does not exist, it returns defaults (empty values where applicable).
|
// If the file does not exist, it returns defaults (empty values where applicable).
|
||||||
// Environment variables are used as fallbacks for any zero-value fields.
|
// Environment variables are used as fallbacks for any zero-value fields.
|
||||||
|
|
@ -128,45 +98,18 @@ func Load(path string) (*Config, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
applyEnvOverrides(cfg)
|
applyEnvOverrides(cfg)
|
||||||
|
|
||||||
if err := validateConfig(cfg); err != nil {
|
|
||||||
return nil, fmt.Errorf("invalid configuration: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateConfig checks security-critical config values at startup.
|
|
||||||
func validateConfig(cfg *Config) error {
|
|
||||||
if cfg.Server.BaseURL != "" {
|
|
||||||
if err := util.ValidatePublicURL(cfg.Server.BaseURL); err != nil {
|
|
||||||
return fmt.Errorf("server.base_url: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if cfg.Server.SourceURL != "" {
|
|
||||||
if err := util.ValidatePublicURL(cfg.Server.SourceURL); err != nil {
|
|
||||||
return fmt.Errorf("server.source_url: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if cfg.Upstream.URL != "" {
|
|
||||||
// Validate scheme and well-formedness, but allow private IPs
|
|
||||||
// since self-hosted deployments commonly use localhost/internal addresses.
|
|
||||||
if _, err := util.SafeURLScheme(cfg.Upstream.URL); err != nil {
|
|
||||||
return fmt.Errorf("upstream.url: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func defaultConfig() *Config {
|
func defaultConfig() *Config {
|
||||||
return &Config{
|
return &Config{
|
||||||
Server: ServerConfig{
|
Server: ServerConfig{
|
||||||
Port: 5355,
|
Port: 8080,
|
||||||
HTTPTimeout: "10s",
|
HTTPTimeout: "10s",
|
||||||
},
|
},
|
||||||
Upstream: UpstreamConfig{},
|
Upstream: UpstreamConfig{},
|
||||||
Engines: EnginesConfig{
|
Engines: EnginesConfig{
|
||||||
LocalPorted: []string{"wikipedia", "wikidata", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing", "google", "youtube", "bing_images", "ddg_images", "qwant_images"},
|
LocalPorted: []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"},
|
||||||
Qwant: QwantConfig{
|
Qwant: QwantConfig{
|
||||||
Category: "web-lite",
|
Category: "web-lite",
|
||||||
ResultsPerPage: 10,
|
ResultsPerPage: 10,
|
||||||
|
|
@ -208,15 +151,6 @@ func applyEnvOverrides(cfg *Config) {
|
||||||
if v := os.Getenv("BRAVE_ACCESS_TOKEN"); v != "" {
|
if v := os.Getenv("BRAVE_ACCESS_TOKEN"); v != "" {
|
||||||
cfg.Engines.Brave.AccessToken = v
|
cfg.Engines.Brave.AccessToken = v
|
||||||
}
|
}
|
||||||
if v := os.Getenv("YOUTUBE_API_KEY"); v != "" {
|
|
||||||
cfg.Engines.YouTube.APIKey = v
|
|
||||||
}
|
|
||||||
if v := os.Getenv("STACKOVERFLOW_KEY"); v != "" {
|
|
||||||
if cfg.Engines.StackOverflow == nil {
|
|
||||||
cfg.Engines.StackOverflow = &StackOverflowConfig{}
|
|
||||||
}
|
|
||||||
cfg.Engines.StackOverflow.APIKey = v
|
|
||||||
}
|
|
||||||
if v := os.Getenv("VALKEY_ADDRESS"); v != "" {
|
if v := os.Getenv("VALKEY_ADDRESS"); v != "" {
|
||||||
cfg.Cache.Address = v
|
cfg.Cache.Address = v
|
||||||
}
|
}
|
||||||
|
|
@ -285,20 +219,6 @@ func (c *Config) CacheTTL() time.Duration {
|
||||||
return 5 * time.Minute
|
return 5 * time.Minute
|
||||||
}
|
}
|
||||||
|
|
||||||
// CacheTTLOverrides returns parsed TTL overrides from config.
|
|
||||||
func (c *Config) CacheTTLOverrides() map[string]time.Duration {
|
|
||||||
if len(c.Cache.TTLOverrides) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make(map[string]time.Duration, len(c.Cache.TTLOverrides))
|
|
||||||
for engine, durStr := range c.Cache.TTLOverrides {
|
|
||||||
if d, err := time.ParseDuration(durStr); err == nil && d > 0 {
|
|
||||||
out[engine] = d
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// RateLimitWindow parses the rate limit window into a time.Duration.
|
// RateLimitWindow parses the rate limit window into a time.Duration.
|
||||||
func (c *Config) RateLimitWindow() time.Duration {
|
func (c *Config) RateLimitWindow() time.Duration {
|
||||||
if d, err := time.ParseDuration(c.RateLimit.Window); err == nil && d > 0 {
|
if d, err := time.ParseDuration(c.RateLimit.Window); err == nil && d > 0 {
|
||||||
|
|
|
||||||
|
|
@ -11,11 +11,11 @@ func TestLoadDefaults(t *testing.T) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Load with missing file should return defaults: %v", err)
|
t.Fatalf("Load with missing file should return defaults: %v", err)
|
||||||
}
|
}
|
||||||
if cfg.Server.Port != 5355 {
|
if cfg.Server.Port != 8080 {
|
||||||
t.Errorf("expected default port 5355, got %d", cfg.Server.Port)
|
t.Errorf("expected default port 8080, got %d", cfg.Server.Port)
|
||||||
}
|
}
|
||||||
if len(cfg.Engines.LocalPorted) != 15 {
|
if len(cfg.Engines.LocalPorted) != 9 {
|
||||||
t.Errorf("expected 15 default engines, got %d", len(cfg.Engines.LocalPorted))
|
t.Errorf("expected 9 default engines, got %d", len(cfg.Engines.LocalPorted))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package contracts
|
package contracts
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -21,17 +5,20 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
)
|
)
|
||||||
|
|
||||||
// MainResult represents one element of the `results` array.
|
// MainResult represents one element of SearXNG's `results` array.
|
||||||
// Unknown keys are preserved in `raw` and re-emitted via MarshalJSON.
|
//
|
||||||
|
// SearXNG returns many additional keys beyond what templates use. To keep the
|
||||||
|
// contract stable for proxying/merging, we preserve all unknown keys in
|
||||||
|
// `raw` and re-emit them via MarshalJSON.
|
||||||
type MainResult struct {
|
type MainResult struct {
|
||||||
raw map[string]any
|
raw map[string]any
|
||||||
|
|
||||||
|
// Common fields used by SearXNG templates (RSS uses: title, url, content, pubdate).
|
||||||
Template string `json:"template"`
|
Template string `json:"template"`
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
Content string `json:"content"`
|
Content string `json:"content"`
|
||||||
URL *string `json:"url"`
|
URL *string `json:"url"`
|
||||||
Pubdate *string `json:"pubdate"`
|
Pubdate *string `json:"pubdate"`
|
||||||
Thumbnail string `json:"thumbnail"`
|
|
||||||
|
|
||||||
Engine string `json:"engine"`
|
Engine string `json:"engine"`
|
||||||
Score float64 `json:"score"`
|
Score float64 `json:"score"`
|
||||||
|
|
@ -41,13 +28,17 @@ type MainResult struct {
|
||||||
Positions []int `json:"positions"`
|
Positions []int `json:"positions"`
|
||||||
Engines []string `json:"engines"`
|
Engines []string `json:"engines"`
|
||||||
|
|
||||||
|
// These fields exist in SearXNG's MainResult base; keep them so downstream
|
||||||
|
// callers can generate richer output later.
|
||||||
OpenGroup bool `json:"open_group"`
|
OpenGroup bool `json:"open_group"`
|
||||||
CloseGroup bool `json:"close_group"`
|
CloseGroup bool `json:"close_group"`
|
||||||
|
|
||||||
|
// parsed_url in SearXNG is emitted as a tuple; we preserve it as-is.
|
||||||
ParsedURL any `json:"parsed_url"`
|
ParsedURL any `json:"parsed_url"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mr *MainResult) UnmarshalJSON(data []byte) error {
|
func (mr *MainResult) UnmarshalJSON(data []byte) error {
|
||||||
|
// Preserve the full object.
|
||||||
dec := json.NewDecoder(bytes.NewReader(data))
|
dec := json.NewDecoder(bytes.NewReader(data))
|
||||||
dec.UseNumber()
|
dec.UseNumber()
|
||||||
|
|
||||||
|
|
@ -58,11 +49,11 @@ func (mr *MainResult) UnmarshalJSON(data []byte) error {
|
||||||
|
|
||||||
mr.raw = m
|
mr.raw = m
|
||||||
|
|
||||||
|
// Fill the typed/common fields (best-effort; don't fail if types differ).
|
||||||
mr.Template = stringOrEmpty(m["template"])
|
mr.Template = stringOrEmpty(m["template"])
|
||||||
mr.Title = stringOrEmpty(m["title"])
|
mr.Title = stringOrEmpty(m["title"])
|
||||||
mr.Content = stringOrEmpty(m["content"])
|
mr.Content = stringOrEmpty(m["content"])
|
||||||
mr.Engine = stringOrEmpty(m["engine"])
|
mr.Engine = stringOrEmpty(m["engine"])
|
||||||
mr.Thumbnail = stringOrEmpty(m["thumbnail"])
|
|
||||||
mr.Category = stringOrEmpty(m["category"])
|
mr.Category = stringOrEmpty(m["category"])
|
||||||
mr.Priority = stringOrEmpty(m["priority"])
|
mr.Priority = stringOrEmpty(m["priority"])
|
||||||
|
|
||||||
|
|
@ -95,17 +86,18 @@ func (mr *MainResult) UnmarshalJSON(data []byte) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mr MainResult) MarshalJSON() ([]byte, error) {
|
func (mr MainResult) MarshalJSON() ([]byte, error) {
|
||||||
|
// If we came from upstream JSON, preserve all keys exactly.
|
||||||
if mr.raw != nil {
|
if mr.raw != nil {
|
||||||
return json.Marshal(mr.raw)
|
return json.Marshal(mr.raw)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Otherwise, marshal the known fields.
|
||||||
m := map[string]any{
|
m := map[string]any{
|
||||||
"template": mr.Template,
|
"template": mr.Template,
|
||||||
"title": mr.Title,
|
"title": mr.Title,
|
||||||
"content": mr.Content,
|
"content": mr.Content,
|
||||||
"url": mr.URL,
|
"url": mr.URL,
|
||||||
"pubdate": mr.Pubdate,
|
"pubdate": mr.Pubdate,
|
||||||
"thumbnail": mr.Thumbnail,
|
|
||||||
"engine": mr.Engine,
|
"engine": mr.Engine,
|
||||||
"score": mr.Score,
|
"score": mr.Score,
|
||||||
"category": mr.Category,
|
"category": mr.Category,
|
||||||
|
|
|
||||||
|
|
@ -1,34 +1,21 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package contracts
|
package contracts
|
||||||
|
|
||||||
// OutputFormat matches the `/search?format=...` values.
|
// OutputFormat matches SearXNG's `/search?format=...` values.
|
||||||
type OutputFormat string
|
type OutputFormat string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
FormatHTML OutputFormat = "html" // accepted for compatibility
|
FormatHTML OutputFormat = "html" // accepted for compatibility (not yet implemented)
|
||||||
FormatJSON OutputFormat = "json"
|
FormatJSON OutputFormat = "json"
|
||||||
FormatCSV OutputFormat = "csv"
|
FormatCSV OutputFormat = "csv"
|
||||||
FormatRSS OutputFormat = "rss"
|
FormatRSS OutputFormat = "rss"
|
||||||
)
|
)
|
||||||
|
|
||||||
type SearchRequest struct {
|
type SearchRequest struct {
|
||||||
|
// Format is what the client requested via `format=...`.
|
||||||
Format OutputFormat
|
Format OutputFormat
|
||||||
|
|
||||||
Query string
|
Query string
|
||||||
|
|
||||||
Pageno int
|
Pageno int
|
||||||
Safesearch int
|
Safesearch int
|
||||||
TimeRange *string
|
TimeRange *string
|
||||||
|
|
@ -36,18 +23,20 @@ type SearchRequest struct {
|
||||||
TimeoutLimit *float64
|
TimeoutLimit *float64
|
||||||
Language string
|
Language string
|
||||||
|
|
||||||
// Engines and categories decide which engines run locally vs proxy to upstream.
|
// Engines and categories are used for deciding which engines run locally vs are proxied.
|
||||||
|
// For now, engines can be supplied directly via the `engines` form parameter.
|
||||||
Engines []string
|
Engines []string
|
||||||
Categories []string
|
Categories []string
|
||||||
|
|
||||||
// EngineData matches the `engine_data-<engine>-<key>=<value>` parameters.
|
// EngineData matches SearXNG's `engine_data-<engine>-<key>=<value>` parameters.
|
||||||
EngineData map[string]map[string]string
|
EngineData map[string]map[string]string
|
||||||
|
|
||||||
// AccessToken gates paid/limited engines. Not part of upstream JSON schema.
|
// AccessToken is an optional request token used to gate paid/limited engines.
|
||||||
|
// It is not part of the upstream JSON schema; it only influences local engines.
|
||||||
AccessToken string
|
AccessToken string
|
||||||
}
|
}
|
||||||
|
|
||||||
// SearchResponse matches the JSON schema used by `webutils.get_json_response()`.
|
// SearchResponse matches the JSON schema returned by SearXNG's `webutils.get_json_response()`.
|
||||||
type SearchResponse struct {
|
type SearchResponse struct {
|
||||||
Query string `json:"query"`
|
Query string `json:"query"`
|
||||||
NumberOfResults int `json:"number_of_results"`
|
NumberOfResults int `json:"number_of_results"`
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -28,7 +12,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
@ -75,8 +59,8 @@ func (e *ArxivEngine) Search(ctx context.Context, req contracts.SearchRequest) (
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 16*1024))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 16*1024))
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("arxiv upstream error: status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("arxiv upstream error: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
raw, err := io.ReadAll(resp.Body)
|
raw, err := io.ReadAll(resp.Body)
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestArxivEngine_Search(t *testing.T) {
|
func TestArxivEngine_Search(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -28,7 +12,7 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BingEngine searches Bing via the public Bing API.
|
// BingEngine searches Bing via the public Bing API.
|
||||||
|
|
@ -68,8 +52,8 @@ func (e *BingEngine) Search(ctx context.Context, req contracts.SearchRequest) (c
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("bing upstream error: status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("bing upstream error: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
contentType := resp.Header.Get("Content-Type")
|
contentType := resp.Header.Get("Content-Type")
|
||||||
|
|
|
||||||
|
|
@ -1,123 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
|
|
||||||
package engines
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/xml"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
// BingImagesEngine searches Bing Images via their public RSS endpoint.
|
|
||||||
type BingImagesEngine struct {
|
|
||||||
client *http.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *BingImagesEngine) Name() string { return "bing_images" }
|
|
||||||
|
|
||||||
func (e *BingImagesEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
|
||||||
if e == nil || e.client == nil {
|
|
||||||
return contracts.SearchResponse{}, errors.New("bing_images engine not initialized")
|
|
||||||
}
|
|
||||||
q := strings.TrimSpace(req.Query)
|
|
||||||
if q == "" {
|
|
||||||
return contracts.SearchResponse{Query: req.Query}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
offset := (req.Pageno - 1) * 10
|
|
||||||
endpoint := fmt.Sprintf(
|
|
||||||
"https://www.bing.com/images/search?q=%s&count=10&offset=%d&format=rss",
|
|
||||||
url.QueryEscape(q),
|
|
||||||
offset,
|
|
||||||
)
|
|
||||||
|
|
||||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
httpReq.Header.Set("User-Agent", "kafka/0.1 (compatible; +https://git.ashisgreat.xyz/penal-colony/kafka)")
|
|
||||||
|
|
||||||
resp, err := e.client.Do(httpReq)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("bing_images upstream error: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
return parseBingImagesRSS(resp.Body, req.Query)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseBingImagesRSS parses Bing's RSS image search results.
|
|
||||||
// The description field contains HTML with an <img> tag whose src is the
|
|
||||||
// thumbnail and whose enclosing <a> tag links to the source page.
|
|
||||||
func parseBingImagesRSS(r io.Reader, query string) (contracts.SearchResponse, error) {
|
|
||||||
type bingImageItem struct {
|
|
||||||
Title string `xml:"title"`
|
|
||||||
Link string `xml:"link"`
|
|
||||||
Descrip string `xml:"description"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rssFeed struct {
|
|
||||||
XMLName xml.Name `xml:"rss"`
|
|
||||||
Channel struct {
|
|
||||||
Items []bingImageItem `xml:"item"`
|
|
||||||
} `xml:"channel"`
|
|
||||||
}
|
|
||||||
|
|
||||||
var rss rssFeed
|
|
||||||
if err := xml.NewDecoder(r).Decode(&rss); err != nil {
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("bing_images RSS parse error: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
results := make([]contracts.MainResult, 0, len(rss.Channel.Items))
|
|
||||||
for _, item := range rss.Channel.Items {
|
|
||||||
if item.Link == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract thumbnail URL from the description HTML.
|
|
||||||
thumbnail := extractImgSrc(item.Descrip)
|
|
||||||
content := stripHTML(item.Descrip)
|
|
||||||
|
|
||||||
linkPtr := item.Link
|
|
||||||
results = append(results, contracts.MainResult{
|
|
||||||
Template: "images",
|
|
||||||
Title: item.Title,
|
|
||||||
Content: content,
|
|
||||||
URL: &linkPtr,
|
|
||||||
Thumbnail: thumbnail,
|
|
||||||
Engine: "bing_images",
|
|
||||||
Score: 0,
|
|
||||||
Category: "images",
|
|
||||||
Engines: []string{"bing_images"},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: query,
|
|
||||||
NumberOfResults: len(results),
|
|
||||||
Results: results,
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
UnresponsiveEngines: [][2]string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
@ -7,7 +7,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBingEngine_EmptyQuery(t *testing.T) {
|
func TestBingEngine_EmptyQuery(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -1,172 +0,0 @@
|
||||||
package engines
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
type BraveEngine struct {
|
|
||||||
client *http.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *BraveEngine) Name() string { return "brave" }
|
|
||||||
|
|
||||||
func (e *BraveEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
|
||||||
if strings.TrimSpace(req.Query) == "" {
|
|
||||||
return contracts.SearchResponse{Query: req.Query}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
start := (req.Pageno - 1) * 20
|
|
||||||
u := fmt.Sprintf(
|
|
||||||
"https://search.brave.com/search?q=%s&offset=%d&source=web",
|
|
||||||
url.QueryEscape(req.Query),
|
|
||||||
start,
|
|
||||||
)
|
|
||||||
|
|
||||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
httpReq.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36")
|
|
||||||
httpReq.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
|
||||||
httpReq.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
|
||||||
|
|
||||||
resp, err := e.client.Do(httpReq)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("brave error: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 128*1024))
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
results := parseBraveResults(string(body))
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: req.Query,
|
|
||||||
NumberOfResults: len(results),
|
|
||||||
Results: results,
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: extractBraveSuggestions(string(body)),
|
|
||||||
UnresponsiveEngines: [][2]string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseBraveResults(body string) []contracts.MainResult {
|
|
||||||
var results []contracts.MainResult
|
|
||||||
|
|
||||||
// Brave wraps each result in divs with data-type="web" or data-type="news".
|
|
||||||
// Pattern: <div ... data-type="web"> ... <a class="result-title" href="URL">TITLE</a> ... <div class="snippet">SNIPPET</div>
|
|
||||||
webPattern := regexp.MustCompile(`(?s)<div[^>]+data-type="web"[^>]*>(.*?)</div>\s*<div[^>]+data-type="(web|news)"`)
|
|
||||||
matches := webPattern.FindAllStringSubmatch(body, -1)
|
|
||||||
|
|
||||||
seen := map[string]bool{}
|
|
||||||
|
|
||||||
for _, match := range matches {
|
|
||||||
if len(match) < 2 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
block := match[1]
|
|
||||||
|
|
||||||
// Extract title and URL from the result-title link.
|
|
||||||
titlePattern := regexp.MustCompile(`<a[^>]+class="result-title"[^>]+href="([^"]+)"[^>]*>([^<]+)</a>`)
|
|
||||||
titleMatch := titlePattern.FindStringSubmatch(block)
|
|
||||||
if titleMatch == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
href := titleMatch[1]
|
|
||||||
title := stripTags(titleMatch[2])
|
|
||||||
|
|
||||||
if href == "" || !strings.HasPrefix(href, "http") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if seen[href] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[href] = true
|
|
||||||
|
|
||||||
// Extract snippet.
|
|
||||||
snippet := extractBraveSnippet(block)
|
|
||||||
|
|
||||||
// Extract favicon URL.
|
|
||||||
favicon := extractBraveFavicon(block)
|
|
||||||
|
|
||||||
urlPtr := href
|
|
||||||
results = append(results, contracts.MainResult{
|
|
||||||
Title: title,
|
|
||||||
URL: &urlPtr,
|
|
||||||
Content: snippet,
|
|
||||||
Thumbnail: favicon,
|
|
||||||
Engine: "brave",
|
|
||||||
Score: 1.0,
|
|
||||||
Category: "general",
|
|
||||||
Engines: []string{"brave"},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
func extractBraveSnippet(block string) string {
|
|
||||||
// Try various snippet selectors Brave uses.
|
|
||||||
patterns := []string{
|
|
||||||
`<div[^>]+class="snippet"[^>]*>(.*?)</div>`,
|
|
||||||
`<p[^>]+class="[^"]*description[^"]*"[^>]*>(.*?)</p>`,
|
|
||||||
`<span[^>]+class="[^"]*snippet[^"]*"[^>]*>(.*?)</span>`,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, pat := range patterns {
|
|
||||||
re := regexp.MustCompile(`(?s)` + pat)
|
|
||||||
m := re.FindStringSubmatch(block)
|
|
||||||
if len(m) >= 2 {
|
|
||||||
text := stripTags(m[1])
|
|
||||||
if text != "" {
|
|
||||||
return strings.TrimSpace(text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func extractBraveFavicon(block string) string {
|
|
||||||
imgPattern := regexp.MustCompile(`<img[^>]+class="[^"]*favicon[^"]*"[^>]+src="([^"]+)"`)
|
|
||||||
m := imgPattern.FindStringSubmatch(block)
|
|
||||||
if len(m) >= 2 {
|
|
||||||
return m[1]
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func extractBraveSuggestions(body string) []string {
|
|
||||||
var suggestions []string
|
|
||||||
// Brave suggestions appear in a dropdown or related searches section.
|
|
||||||
suggestPattern := regexp.MustCompile(`(?s)<li[^>]+class="[^"]*suggestion[^"]*"[^>]*>.*?<a[^>]*>([^<]+)</a>`)
|
|
||||||
matches := suggestPattern.FindAllStringSubmatch(body, -1)
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for _, m := range matches {
|
|
||||||
if len(m) < 2 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
s := strings.TrimSpace(stripTags(m[1]))
|
|
||||||
if s != "" && !seen[s] {
|
|
||||||
seen[s] = true
|
|
||||||
suggestions = append(suggestions, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return suggestions
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -27,26 +11,32 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BraveEngine implements the Brave Web Search API.
|
// BraveEngine implements the SearXNG `braveapi` engine (Brave Web Search API).
|
||||||
// Required: BRAVE_API_KEY env var or config.
|
//
|
||||||
// Optional: BRAVE_ACCESS_TOKEN to gate requests.
|
// Config / gating:
|
||||||
type BraveAPIEngine struct {
|
// - BRAVE_API_KEY: required to call Brave
|
||||||
|
// - BRAVE_ACCESS_TOKEN (optional): if set, the request must include a token
|
||||||
|
// that matches the env var (via Authorization Bearer, X-Search-Token,
|
||||||
|
// X-Brave-Access-Token, or form field `token`).
|
||||||
|
type BraveEngine struct {
|
||||||
client *http.Client
|
client *http.Client
|
||||||
apiKey string
|
apiKey string
|
||||||
accessGateToken string
|
accessGateToken string
|
||||||
resultsPerPage int
|
resultsPerPage int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *BraveAPIEngine) Name() string { return "braveapi" }
|
func (e *BraveEngine) Name() string { return "braveapi" }
|
||||||
|
|
||||||
func (e *BraveAPIEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
func (e *BraveEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
||||||
if e == nil || e.client == nil {
|
if e == nil || e.client == nil {
|
||||||
return contracts.SearchResponse{}, errors.New("brave engine not initialized")
|
return contracts.SearchResponse{}, errors.New("brave engine not initialized")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Gate / config checks should not be treated as fatal errors; SearXNG
|
||||||
|
// treats misconfigured engines as unresponsive.
|
||||||
if strings.TrimSpace(e.apiKey) == "" {
|
if strings.TrimSpace(e.apiKey) == "" {
|
||||||
return contracts.SearchResponse{
|
return contracts.SearchResponse{
|
||||||
Query: req.Query,
|
Query: req.Query,
|
||||||
|
|
@ -80,15 +70,10 @@ func (e *BraveAPIEngine) Search(ctx context.Context, req contracts.SearchRequest
|
||||||
return contracts.SearchResponse{Query: req.Query}, nil
|
return contracts.SearchResponse{Query: req.Query}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Brave API only supports offset values 0-9 (first page of results).
|
|
||||||
// Paginating beyond the first page is not supported by Brave.
|
|
||||||
offset := 0
|
offset := 0
|
||||||
if req.Pageno > 1 {
|
if req.Pageno > 1 {
|
||||||
offset = (req.Pageno - 1) * e.resultsPerPage
|
offset = (req.Pageno - 1) * e.resultsPerPage
|
||||||
}
|
}
|
||||||
if offset > 9 {
|
|
||||||
offset = 9
|
|
||||||
}
|
|
||||||
|
|
||||||
args := url.Values{}
|
args := url.Values{}
|
||||||
args.Set("q", q)
|
args.Set("q", q)
|
||||||
|
|
@ -108,6 +93,8 @@ func (e *BraveAPIEngine) Search(ctx context.Context, req contracts.SearchRequest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SearXNG's python checks `if params["safesearch"]:` which treats any
|
||||||
|
// non-zero (moderate/strict) as strict.
|
||||||
if req.Safesearch > 0 {
|
if req.Safesearch > 0 {
|
||||||
args.Set("safesearch", "strict")
|
args.Set("safesearch", "strict")
|
||||||
}
|
}
|
||||||
|
|
@ -127,8 +114,8 @@ func (e *BraveAPIEngine) Search(ctx context.Context, req contracts.SearchRequest
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 16*1024))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 16*1024))
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("brave upstream error: status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("brave upstream error: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
var api struct {
|
var api struct {
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBraveEngine_GatingAndHeader(t *testing.T) {
|
func TestBraveEngine_GatingAndHeader(t *testing.T) {
|
||||||
|
|
@ -39,7 +39,7 @@ func TestBraveEngine_GatingAndHeader(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
client := &http.Client{Transport: transport}
|
client := &http.Client{Transport: transport}
|
||||||
engine := &BraveAPIEngine{
|
engine := &BraveEngine{
|
||||||
client: client,
|
client: client,
|
||||||
apiKey: wantAPIKey,
|
apiKey: wantAPIKey,
|
||||||
accessGateToken: wantToken,
|
accessGateToken: wantToken,
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -27,7 +11,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
type CrossrefEngine struct {
|
type CrossrefEngine struct {
|
||||||
|
|
@ -63,8 +47,8 @@ func (e *CrossrefEngine) Search(ctx context.Context, req contracts.SearchRequest
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 16*1024))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 16*1024))
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("crossref upstream error: status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("crossref upstream error: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
var api struct {
|
var api struct {
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCrossrefEngine_Search(t *testing.T) {
|
func TestCrossrefEngine_Search(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -1,207 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
|
|
||||||
package engines
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
// DuckDuckGoImagesEngine searches DuckDuckGo Images via their vql API.
|
|
||||||
type DuckDuckGoImagesEngine struct {
|
|
||||||
client *http.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *DuckDuckGoImagesEngine) Name() string { return "ddg_images" }
|
|
||||||
|
|
||||||
func (e *DuckDuckGoImagesEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
|
||||||
if e == nil || e.client == nil {
|
|
||||||
return contracts.SearchResponse{}, errors.New("ddg_images engine not initialized")
|
|
||||||
}
|
|
||||||
q := strings.TrimSpace(req.Query)
|
|
||||||
if q == "" {
|
|
||||||
return contracts.SearchResponse{Query: req.Query}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Step 1: Get a VQD token from the initial search page.
|
|
||||||
vqd, err := e.getVQD(ctx, q)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: req.Query,
|
|
||||||
UnresponsiveEngines: [][2]string{{"ddg_images", "vqd_fetch_failed"}},
|
|
||||||
Results: []contracts.MainResult{},
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Step 2: Fetch image results using the VQD token.
|
|
||||||
endpoint := fmt.Sprintf(
|
|
||||||
"https://duckduckgo.com/i.js?q=%s&kl=wt-wt&l=wt-wt&p=1&s=%d&vqd=%s",
|
|
||||||
url.QueryEscape(q),
|
|
||||||
(req.Pageno-1)*50,
|
|
||||||
url.QueryEscape(vqd),
|
|
||||||
)
|
|
||||||
|
|
||||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
httpReq.Header.Set("User-Agent", "kafka/0.1 (compatible; +https://git.ashisgreat.xyz/penal-colony/kafka)")
|
|
||||||
httpReq.Header.Set("Referer", "https://duckduckgo.com/")
|
|
||||||
|
|
||||||
resp, err := e.client.Do(httpReq)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 16*1024))
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("ddg_images upstream error: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return parseDDGImages(body, req.Query)
|
|
||||||
}
|
|
||||||
|
|
||||||
// getVQD fetches a VQD token from DuckDuckGo's search page.
|
|
||||||
func (e *DuckDuckGoImagesEngine) getVQD(ctx context.Context, query string) (string, error) {
|
|
||||||
endpoint := "https://duckduckgo.com/?q=" + url.QueryEscape(query)
|
|
||||||
|
|
||||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
httpReq.Header.Set("User-Agent", "kafka/0.1 (compatible; +https://git.ashisgreat.xyz/penal-colony/kafka)")
|
|
||||||
|
|
||||||
resp, err := e.client.Do(httpReq)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract VQD from the HTML: vqd='...'
|
|
||||||
vqd := extractVQD(string(body))
|
|
||||||
if vqd == "" {
|
|
||||||
return "", fmt.Errorf("vqd token not found in response")
|
|
||||||
}
|
|
||||||
return vqd, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// extractVQD extracts the VQD token from DuckDuckGo's HTML response.
|
|
||||||
func extractVQD(html string) string {
|
|
||||||
// Look for: vqd='...' or vqd="..."
|
|
||||||
for _, prefix := range []string{"vqd='", `vqd="`} {
|
|
||||||
idx := strings.Index(html, prefix)
|
|
||||||
if idx == -1 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
start := idx + len(prefix)
|
|
||||||
end := start
|
|
||||||
for end < len(html) && html[end] != '\'' && html[end] != '"' {
|
|
||||||
end++
|
|
||||||
}
|
|
||||||
if end > start {
|
|
||||||
return html[start:end]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// ddgImageResult represents a single image result from DDG's JSON API.
|
|
||||||
type ddgImageResult struct {
|
|
||||||
Title string `json:"title"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
Thumbnail string `json:"thumbnail"`
|
|
||||||
Image string `json:"image"`
|
|
||||||
Width int `json:"width"`
|
|
||||||
Height int `json:"height"`
|
|
||||||
Source string `json:"source"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseDDGImages(body []byte, query string) (contracts.SearchResponse, error) {
|
|
||||||
var results struct {
|
|
||||||
Results []ddgImageResult `json:"results"`
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.Unmarshal(body, &results); err != nil {
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("ddg_images JSON parse error: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
out := make([]contracts.MainResult, 0, len(results.Results))
|
|
||||||
for _, img := range results.Results {
|
|
||||||
if img.URL == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prefer the full image URL as thumbnail, fall back to the thumbnail field.
|
|
||||||
thumb := img.Image
|
|
||||||
if thumb == "" {
|
|
||||||
thumb = img.Thumbnail
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build a simple content string showing dimensions.
|
|
||||||
content := ""
|
|
||||||
if img.Width > 0 && img.Height > 0 {
|
|
||||||
content = strconv.Itoa(img.Width) + " × " + strconv.Itoa(img.Height)
|
|
||||||
}
|
|
||||||
if img.Source != "" {
|
|
||||||
if content != "" {
|
|
||||||
content += " — " + img.Source
|
|
||||||
} else {
|
|
||||||
content = img.Source
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
urlPtr := img.URL
|
|
||||||
out = append(out, contracts.MainResult{
|
|
||||||
Template: "images",
|
|
||||||
Title: img.Title,
|
|
||||||
Content: content,
|
|
||||||
URL: &urlPtr,
|
|
||||||
Thumbnail: thumb,
|
|
||||||
Engine: "ddg_images",
|
|
||||||
Score: 0,
|
|
||||||
Category: "images",
|
|
||||||
Engines: []string{"ddg_images"},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: query,
|
|
||||||
NumberOfResults: len(out),
|
|
||||||
Results: out,
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
UnresponsiveEngines: [][2]string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -25,7 +9,7 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DuckDuckGoEngine searches DuckDuckGo's Lite/HTML endpoint.
|
// DuckDuckGoEngine searches DuckDuckGo's Lite/HTML endpoint.
|
||||||
|
|
@ -63,8 +47,8 @@ func (e *DuckDuckGoEngine) Search(ctx context.Context, req contracts.SearchReque
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("duckduckgo upstream error: status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("duckduckgo upstream error: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
results, err := parseDuckDuckGoHTML(resp.Body)
|
results, err := parseDuckDuckGoHTML(resp.Body)
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -21,7 +5,7 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
// parseDuckDuckGoHTML parses DuckDuckGo Lite's HTML response for search results.
|
// parseDuckDuckGoHTML parses DuckDuckGo Lite's HTML response for search results.
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDuckDuckGoEngine_EmptyQuery(t *testing.T) {
|
func TestDuckDuckGoEngine_EmptyQuery(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -1,28 +1,12 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Engine is a Go-native implementation of a search engine.
|
// Engine is a Go-native implementation of a SearXNG engine.
|
||||||
//
|
//
|
||||||
// Implementations should return a SearchResponse containing only the results
|
// Implementations should return a SearchResponse containing only the results
|
||||||
// for that engine subset; the caller will merge multiple engine responses.
|
// for that engine subset; the caller will merge multiple engine responses.
|
||||||
|
|
|
||||||
|
|
@ -1,65 +1,28 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/config"
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/httpclient"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewDefaultPortedEngines returns the Go-native engine registry.
|
// NewDefaultPortedEngines returns the starter set of Go-native engines.
|
||||||
// If cfg is nil, API keys fall back to environment variables.
|
// The service can swap/extend this registry later as more engines are ported.
|
||||||
func NewDefaultPortedEngines(client *http.Client, cfg *config.Config) map[string]Engine {
|
func NewDefaultPortedEngines(client *http.Client) map[string]Engine {
|
||||||
if client == nil {
|
if client == nil {
|
||||||
client = httpclient.NewClient(10 * time.Second)
|
client = &http.Client{Timeout: 10 * time.Second}
|
||||||
}
|
|
||||||
|
|
||||||
var braveAPIKey, braveAccessToken, youtubeAPIKey string
|
|
||||||
if cfg != nil {
|
|
||||||
braveAPIKey = cfg.Engines.Brave.APIKey
|
|
||||||
braveAccessToken = cfg.Engines.Brave.AccessToken
|
|
||||||
youtubeAPIKey = cfg.Engines.YouTube.APIKey
|
|
||||||
}
|
|
||||||
if braveAPIKey == "" {
|
|
||||||
braveAPIKey = os.Getenv("BRAVE_API_KEY")
|
|
||||||
}
|
|
||||||
if braveAccessToken == "" {
|
|
||||||
braveAccessToken = os.Getenv("BRAVE_ACCESS_TOKEN")
|
|
||||||
}
|
|
||||||
if youtubeAPIKey == "" {
|
|
||||||
youtubeAPIKey = os.Getenv("YOUTUBE_API_KEY")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return map[string]Engine{
|
return map[string]Engine{
|
||||||
"wikipedia": &WikipediaEngine{client: client},
|
"wikipedia": &WikipediaEngine{client: client},
|
||||||
"wikidata": &WikidataEngine{client: client},
|
|
||||||
"arxiv": &ArxivEngine{client: client},
|
"arxiv": &ArxivEngine{client: client},
|
||||||
"crossref": &CrossrefEngine{client: client},
|
"crossref": &CrossrefEngine{client: client},
|
||||||
"braveapi": &BraveAPIEngine{
|
"braveapi": &BraveEngine{
|
||||||
client: client,
|
client: client,
|
||||||
apiKey: braveAPIKey,
|
apiKey: os.Getenv("BRAVE_API_KEY"),
|
||||||
accessGateToken: braveAccessToken,
|
accessGateToken: os.Getenv("BRAVE_ACCESS_TOKEN"),
|
||||||
resultsPerPage: 20,
|
resultsPerPage: 20,
|
||||||
},
|
},
|
||||||
"brave": &BraveEngine{client: client},
|
|
||||||
"qwant": &QwantEngine{
|
"qwant": &QwantEngine{
|
||||||
client: client,
|
client: client,
|
||||||
category: "web-lite",
|
category: "web-lite",
|
||||||
|
|
@ -70,23 +33,5 @@ func NewDefaultPortedEngines(client *http.Client, cfg *config.Config) map[string
|
||||||
"reddit": &RedditEngine{client: client},
|
"reddit": &RedditEngine{client: client},
|
||||||
"bing": &BingEngine{client: client},
|
"bing": &BingEngine{client: client},
|
||||||
"google": &GoogleEngine{client: client},
|
"google": &GoogleEngine{client: client},
|
||||||
"youtube": &YouTubeEngine{
|
|
||||||
client: client,
|
|
||||||
apiKey: youtubeAPIKey,
|
|
||||||
baseURL: "https://www.googleapis.com",
|
|
||||||
},
|
|
||||||
"stackoverflow": &StackOverflowEngine{client: client, apiKey: stackoverflowAPIKey(cfg)},
|
|
||||||
// Image engines
|
|
||||||
"bing_images": &BingImagesEngine{client: client},
|
|
||||||
"ddg_images": &DuckDuckGoImagesEngine{client: client},
|
|
||||||
"qwant_images": &QwantImagesEngine{client: client},
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// stackoverflowAPIKey returns the Stack Overflow API key from config or env var.
|
|
||||||
func stackoverflowAPIKey(cfg *config.Config) string {
|
|
||||||
if cfg != nil && cfg.Engines.StackOverflow != nil && cfg.Engines.StackOverflow.APIKey != "" {
|
|
||||||
return cfg.Engines.StackOverflow.APIKey
|
|
||||||
}
|
|
||||||
return os.Getenv("STACKOVERFLOW_KEY")
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -27,7 +11,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GitHubEngine searches GitHub repositories and code via the public search API.
|
// GitHubEngine searches GitHub repositories and code via the public search API.
|
||||||
|
|
@ -66,8 +50,8 @@ func (e *GitHubEngine) Search(ctx context.Context, req contracts.SearchRequest)
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("github api error: status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("github api error: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
var data struct {
|
var data struct {
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGitHubEngine_EmptyQuery(t *testing.T) {
|
func TestGitHubEngine_EmptyQuery(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -25,13 +9,23 @@ import (
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
// googleUserAgent is an honest User-Agent identifying the metasearch engine.
|
// GSA User-Agent pool — these are Google Search Appliance identifiers
|
||||||
// Using a spoofed GSA User-Agent violates Google's Terms of Service and
|
// that Google trusts for enterprise search appliance traffic.
|
||||||
// risks permanent IP blocking.
|
var gsaUserAgents = []string{
|
||||||
var googleUserAgent = "Kafka/0.1 (compatible; +https://github.com/metamorphosis-dev/samsa)"
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/399.2.845414227 Mobile/15E148 Safari/604.1",
|
||||||
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/406.0.862495628 Mobile/15E148 Safari/604.1",
|
||||||
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/406.0.862495628 Mobile/15E148 Safari/604.1",
|
||||||
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_0_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/406.0.862495628 Mobile/15E148 Safari/604.1",
|
||||||
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/399.2.845414227 Mobile/15E148 Safari/604.1",
|
||||||
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_5_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/406.0.862495628 Mobile/15E148 Safari/604.1",
|
||||||
|
}
|
||||||
|
|
||||||
|
func gsaUA() string {
|
||||||
|
return gsaUserAgents[0] // deterministic for now; could rotate
|
||||||
|
}
|
||||||
|
|
||||||
type GoogleEngine struct {
|
type GoogleEngine struct {
|
||||||
client *http.Client
|
client *http.Client
|
||||||
|
|
@ -47,6 +41,7 @@ func (e *GoogleEngine) Search(ctx context.Context, req contracts.SearchRequest)
|
||||||
start := (req.Pageno - 1) * 10
|
start := (req.Pageno - 1) * 10
|
||||||
query := url.QueryEscape(req.Query)
|
query := url.QueryEscape(req.Query)
|
||||||
|
|
||||||
|
// Build URL like SearXNG does.
|
||||||
u := fmt.Sprintf(
|
u := fmt.Sprintf(
|
||||||
"https://www.google.com/search?q=%s&filter=0&start=%d&hl=%s&lr=%s&safe=%s",
|
"https://www.google.com/search?q=%s&filter=0&start=%d&hl=%s&lr=%s&safe=%s",
|
||||||
query,
|
query,
|
||||||
|
|
@ -60,7 +55,7 @@ func (e *GoogleEngine) Search(ctx context.Context, req contracts.SearchRequest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.SearchResponse{}, err
|
return contracts.SearchResponse{}, err
|
||||||
}
|
}
|
||||||
httpReq.Header.Set("User-Agent", googleUserAgent)
|
httpReq.Header.Set("User-Agent", gsaUA())
|
||||||
httpReq.Header.Set("Accept", "*/*")
|
httpReq.Header.Set("Accept", "*/*")
|
||||||
httpReq.AddCookie(&http.Cookie{Name: "CONSENT", Value: "YES+"})
|
httpReq.AddCookie(&http.Cookie{Name: "CONSENT", Value: "YES+"})
|
||||||
|
|
||||||
|
|
@ -85,8 +80,8 @@ func (e *GoogleEngine) Search(ctx context.Context, req contracts.SearchRequest)
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("google error: status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("google error: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 128*1024))
|
body, err := io.ReadAll(io.LimitReader(resp.Body, 128*1024))
|
||||||
|
|
@ -107,6 +102,7 @@ func (e *GoogleEngine) Search(ctx context.Context, req contracts.SearchRequest)
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// detectGoogleSorry returns true if the response is a Google block/CAPTCHA page.
|
||||||
func detectGoogleSorry(resp *http.Response) bool {
|
func detectGoogleSorry(resp *http.Response) bool {
|
||||||
if resp.Request != nil {
|
if resp.Request != nil {
|
||||||
if resp.Request.URL.Host == "sorry.google.com" || strings.HasPrefix(resp.Request.URL.Path, "/sorry") {
|
if resp.Request.URL.Host == "sorry.google.com" || strings.HasPrefix(resp.Request.URL.Path, "/sorry") {
|
||||||
|
|
@ -116,10 +112,17 @@ func detectGoogleSorry(resp *http.Response) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseGoogleResults extracts search results from Google's HTML.
|
||||||
|
// Uses the same selectors as SearXNG: div.MjjYud for result containers.
|
||||||
func parseGoogleResults(body, query string) []contracts.MainResult {
|
func parseGoogleResults(body, query string) []contracts.MainResult {
|
||||||
var results []contracts.MainResult
|
var results []contracts.MainResult
|
||||||
|
|
||||||
mjjPattern := regexp.MustCompile(`<div[^>]*class="[^"]*MjjYud[^"]*"[^>]*>(.*?)</div>`)
|
// SearXNG selector: .//div[contains(@class, "MjjYud")]
|
||||||
|
// Each result block contains a title link and snippet.
|
||||||
|
// We simulate the XPath matching with regex-based extraction.
|
||||||
|
|
||||||
|
// Find all MjjYud div blocks.
|
||||||
|
mjjPattern := regexp.MustCompile(`<div[^>]*class="[^"]*MjjYud[^"]*"[^>]*>(.*?)</div>\s*(?=<div[^>]*class="[^"]*MjjYud|$)`)
|
||||||
matches := mjjPattern.FindAllStringSubmatch(body, -1)
|
matches := mjjPattern.FindAllStringSubmatch(body, -1)
|
||||||
|
|
||||||
for i, match := range matches {
|
for i, match := range matches {
|
||||||
|
|
@ -128,12 +131,15 @@ func parseGoogleResults(body, query string) []contracts.MainResult {
|
||||||
}
|
}
|
||||||
block := match[1]
|
block := match[1]
|
||||||
|
|
||||||
|
// Extract title and URL from the result link.
|
||||||
|
// Pattern: <a href="/url?q=ACTUAL_URL&sa=..." ...>TITLE</a>
|
||||||
urlPattern := regexp.MustCompile(`<a[^>]+href="(/url\?q=[^"&]+)`)
|
urlPattern := regexp.MustCompile(`<a[^>]+href="(/url\?q=[^"&]+)`)
|
||||||
urlMatch := urlPattern.FindStringSubmatch(block)
|
urlMatch := urlPattern.FindStringSubmatch(block)
|
||||||
if len(urlMatch) < 2 {
|
if len(urlMatch) < 2 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
rawURL := urlMatch[1]
|
rawURL := urlMatch[1]
|
||||||
|
// Remove /url?q= prefix and decode.
|
||||||
actualURL := strings.TrimPrefix(rawURL, "/url?q=")
|
actualURL := strings.TrimPrefix(rawURL, "/url?q=")
|
||||||
if amp := strings.Index(actualURL, "&"); amp != -1 {
|
if amp := strings.Index(actualURL, "&"); amp != -1 {
|
||||||
actualURL = actualURL[:amp]
|
actualURL = actualURL[:amp]
|
||||||
|
|
@ -146,12 +152,14 @@ func parseGoogleResults(body, query string) []contracts.MainResult {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Extract title from the title tag.
|
||||||
titlePattern := regexp.MustCompile(`<span[^>]*class="[^"]*qrStP[^"]*"[^>]*>([^<]+)</span>`)
|
titlePattern := regexp.MustCompile(`<span[^>]*class="[^"]*qrStP[^"]*"[^>]*>([^<]+)</span>`)
|
||||||
titleMatch := titlePattern.FindStringSubmatch(block)
|
titleMatch := titlePattern.FindStringSubmatch(block)
|
||||||
title := query
|
title := query
|
||||||
if len(titleMatch) >= 2 {
|
if len(titleMatch) >= 2 {
|
||||||
title = stripTags(titleMatch[1])
|
title = stripTags(titleMatch[1])
|
||||||
} else {
|
} else {
|
||||||
|
// Fallback: extract visible text from an <a> with data-title or role="link"
|
||||||
linkTitlePattern := regexp.MustCompile(`<a[^>]+role="link"[^>]*>([^<]+)<`)
|
linkTitlePattern := regexp.MustCompile(`<a[^>]+role="link"[^>]*>([^<]+)<`)
|
||||||
ltMatch := linkTitlePattern.FindStringSubmatch(block)
|
ltMatch := linkTitlePattern.FindStringSubmatch(block)
|
||||||
if len(ltMatch) >= 2 {
|
if len(ltMatch) >= 2 {
|
||||||
|
|
@ -159,6 +167,7 @@ func parseGoogleResults(body, query string) []contracts.MainResult {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Extract snippet from data-sncf divs (SearXNG's approach).
|
||||||
snippet := extractGoogleSnippet(block)
|
snippet := extractGoogleSnippet(block)
|
||||||
|
|
||||||
urlPtr := actualURL
|
urlPtr := actualURL
|
||||||
|
|
@ -177,7 +186,10 @@ func parseGoogleResults(body, query string) []contracts.MainResult {
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// extractGoogleSnippet extracts the snippet text from a Google result block.
|
||||||
func extractGoogleSnippet(block string) string {
|
func extractGoogleSnippet(block string) string {
|
||||||
|
// Google's snippets live in divs with data-sncf attribute.
|
||||||
|
// SearXNG looks for: .//div[contains(@data-sncf, "1")]
|
||||||
snippetPattern := regexp.MustCompile(`<div[^>]+data-sncf="1"[^>]*>(.*?)</div>`)
|
snippetPattern := regexp.MustCompile(`<div[^>]+data-sncf="1"[^>]*>(.*?)</div>`)
|
||||||
matches := snippetPattern.FindAllStringSubmatch(block, -1)
|
matches := snippetPattern.FindAllStringSubmatch(block, -1)
|
||||||
var parts []string
|
var parts []string
|
||||||
|
|
@ -193,9 +205,11 @@ func extractGoogleSnippet(block string) string {
|
||||||
return strings.Join(parts, " ")
|
return strings.Join(parts, " ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// extractGoogleSuggestions extracts search suggestions from Google result cards.
|
||||||
func extractGoogleSuggestions(body string) []string {
|
func extractGoogleSuggestions(body string) []string {
|
||||||
var suggestions []string
|
var suggestions []string
|
||||||
suggestionPattern := regexp.MustCompile(`(?s)<div[^>]*class="[^"]*ouy7Mc[^"]*"[^>]*>.*?<a[^>]*>([^<]+)</a>`)
|
// SearXNG xpath: //div[contains(@class, "ouy7Mc")]//a
|
||||||
|
suggestionPattern := regexp.MustCompile(`<div[^>]*class="[^"]*ouy7Mc[^"]*"[^>]*>.*?<a[^>]*>([^<]+)</a>`, regexp.DotAll)
|
||||||
matches := suggestionPattern.FindAllStringSubmatch(body, -1)
|
matches := suggestionPattern.FindAllStringSubmatch(body, -1)
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
for _, m := range matches {
|
for _, m := range matches {
|
||||||
|
|
@ -211,6 +225,8 @@ func extractGoogleSuggestions(body string) []string {
|
||||||
return suggestions
|
return suggestions
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// googleHL maps SearXNG locale to Google hl (host language) parameter.
|
||||||
|
// e.g. "en-US" -> "en-US"
|
||||||
func googleHL(lang string) string {
|
func googleHL(lang string) string {
|
||||||
lang = strings.ToLower(strings.TrimSpace(lang))
|
lang = strings.ToLower(strings.TrimSpace(lang))
|
||||||
if lang == "" || lang == "auto" {
|
if lang == "" || lang == "auto" {
|
||||||
|
|
@ -219,6 +235,8 @@ func googleHL(lang string) string {
|
||||||
return lang
|
return lang
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// googleUILanguage maps SearXNG language to Google lr (language restrict) parameter.
|
||||||
|
// e.g. "en" -> "lang_en", "de" -> "lang_de"
|
||||||
func googleUILanguage(lang string) string {
|
func googleUILanguage(lang string) string {
|
||||||
lang = strings.ToLower(strings.Split(lang, "-")[0])
|
lang = strings.ToLower(strings.Split(lang, "-")[0])
|
||||||
if lang == "" || lang == "auto" {
|
if lang == "" || lang == "auto" {
|
||||||
|
|
@ -227,6 +245,7 @@ func googleUILanguage(lang string) string {
|
||||||
return "lang_" + lang
|
return "lang_" + lang
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// googleSafeSearchLevel maps safesearch (0-2) to Google's safe parameter.
|
||||||
func googleSafeSearchLevel(safesearch int) string {
|
func googleSafeSearchLevel(safesearch int) string {
|
||||||
switch safesearch {
|
switch safesearch {
|
||||||
case 0:
|
case 0:
|
||||||
|
|
@ -240,6 +259,7 @@ func googleSafeSearchLevel(safesearch int) string {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stripTags removes HTML tags from a string.
|
||||||
func stripTags(s string) string {
|
func stripTags(s string) string {
|
||||||
stripper := regexp.MustCompile(`<[^>]*>`)
|
stripper := regexp.MustCompile(`<[^>]*>`)
|
||||||
s = stripper.ReplaceAllString(s, "")
|
s = stripper.ReplaceAllString(s, "")
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -72,14 +56,3 @@ func htmlUnescape(s string) string {
|
||||||
s = strings.ReplaceAll(s, " ", " ")
|
s = strings.ReplaceAll(s, " ", " ")
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
// extractImgSrc finds the first <img src="..."> in an HTML string and returns
|
|
||||||
// the src attribute value.
|
|
||||||
func extractImgSrc(html string) string {
|
|
||||||
idx := strings.Index(html, "<img")
|
|
||||||
if idx == -1 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
remaining := html[idx:]
|
|
||||||
return extractAttr(remaining, "src")
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,35 +1,13 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
var defaultPortedEngines = []string{
|
var defaultPortedEngines = []string{"wikipedia", "arxiv", "crossref", "braveapi", "qwant", "duckduckgo", "github", "reddit", "bing"}
|
||||||
"wikipedia", "wikidata", "arxiv", "crossref", "braveapi",
|
|
||||||
"brave", "qwant", "duckduckgo", "github", "reddit",
|
|
||||||
"bing", "google", "youtube", "stackoverflow",
|
|
||||||
// Image engines
|
|
||||||
"bing_images", "ddg_images", "qwant_images",
|
|
||||||
}
|
|
||||||
|
|
||||||
type Planner struct {
|
type Planner struct {
|
||||||
PortedSet map[string]bool
|
PortedSet map[string]bool
|
||||||
|
|
@ -70,7 +48,7 @@ func NewPlanner(portedEngines []string) *Planner {
|
||||||
|
|
||||||
// Plan returns:
|
// Plan returns:
|
||||||
// - localEngines: engines that are configured as ported for this service
|
// - localEngines: engines that are configured as ported for this service
|
||||||
// - upstreamEngines: engines that should be executed by the upstream instance
|
// - upstreamEngines: engines that should be executed by upstream SearXNG
|
||||||
// - requestedEngines: the (possibly inferred) requested engines list
|
// - requestedEngines: the (possibly inferred) requested engines list
|
||||||
//
|
//
|
||||||
// If the request provides an explicit `engines` parameter, we use it.
|
// If the request provides an explicit `engines` parameter, we use it.
|
||||||
|
|
@ -101,12 +79,14 @@ func (p *Planner) Plan(req contracts.SearchRequest) (localEngines, upstreamEngin
|
||||||
}
|
}
|
||||||
|
|
||||||
func inferFromCategories(categories []string) []string {
|
func inferFromCategories(categories []string) []string {
|
||||||
|
// Minimal mapping for the initial porting subset.
|
||||||
|
// This mirrors the idea of selecting from SearXNG categories without
|
||||||
|
// embedding the whole engine registry.
|
||||||
set := map[string]bool{}
|
set := map[string]bool{}
|
||||||
for _, c := range categories {
|
for _, c := range categories {
|
||||||
switch strings.TrimSpace(strings.ToLower(c)) {
|
switch strings.TrimSpace(strings.ToLower(c)) {
|
||||||
case "general":
|
case "general":
|
||||||
set["wikipedia"] = true
|
set["wikipedia"] = true
|
||||||
set["wikidata"] = true
|
|
||||||
set["braveapi"] = true
|
set["braveapi"] = true
|
||||||
set["qwant"] = true
|
set["qwant"] = true
|
||||||
set["duckduckgo"] = true
|
set["duckduckgo"] = true
|
||||||
|
|
@ -117,15 +97,8 @@ func inferFromCategories(categories []string) []string {
|
||||||
set["crossref"] = true
|
set["crossref"] = true
|
||||||
case "it":
|
case "it":
|
||||||
set["github"] = true
|
set["github"] = true
|
||||||
set["stackoverflow"] = true
|
|
||||||
case "social media":
|
case "social media":
|
||||||
set["reddit"] = true
|
set["reddit"] = true
|
||||||
case "videos":
|
|
||||||
set["youtube"] = true
|
|
||||||
case "images":
|
|
||||||
set["bing_images"] = true
|
|
||||||
set["ddg_images"] = true
|
|
||||||
set["qwant_images"] = true
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -134,16 +107,13 @@ func inferFromCategories(categories []string) []string {
|
||||||
out = append(out, e)
|
out = append(out, e)
|
||||||
}
|
}
|
||||||
// stable order
|
// stable order
|
||||||
order := map[string]int{
|
order := map[string]int{"wikipedia": 0, "braveapi": 1, "qwant": 2, "duckduckgo": 3, "bing": 4, "google": 5, "arxiv": 6, "crossref": 7, "github": 8, "reddit": 9}
|
||||||
"wikipedia": 0, "wikidata": 1, "braveapi": 2, "brave": 3, "qwant": 4, "duckduckgo": 5, "bing": 6, "google": 7,
|
|
||||||
"arxiv": 8, "crossref": 9, "github": 10, "stackoverflow": 11, "reddit": 12, "youtube": 13,
|
|
||||||
"bing_images": 14, "ddg_images": 15, "qwant_images": 16,
|
|
||||||
}
|
|
||||||
sortByOrder(out, order)
|
sortByOrder(out, order)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func sortByOrder(list []string, order map[string]int) {
|
func sortByOrder(list []string, order map[string]int) {
|
||||||
|
// simple insertion sort (list is tiny)
|
||||||
for i := 1; i < len(list); i++ {
|
for i := 1; i < len(list); i++ {
|
||||||
j := i
|
j := i
|
||||||
for j > 0 && order[list[j-1]] > order[list[j]] {
|
for j > 0 && order[list[j-1]] > order[list[j]] {
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -26,11 +10,15 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// QwantEngine implements the Qwant v3 API (web and web-lite modes).
|
// QwantEngine implements a SearXNG-like `qwant` (web) adapter using
|
||||||
|
// Qwant v3 endpoint: https://api.qwant.com/v3/search/web.
|
||||||
|
//
|
||||||
|
// Qwant's API is not fully documented; this mirrors SearXNG's parsing logic
|
||||||
|
// for the `web` category from `.agent/searxng/searx/engines/qwant.py`.
|
||||||
type QwantEngine struct {
|
type QwantEngine struct {
|
||||||
client *http.Client
|
client *http.Client
|
||||||
category string // "web" (JSON API) or "web-lite" (HTML fallback)
|
category string // "web" (JSON API) or "web-lite" (HTML fallback)
|
||||||
|
|
@ -49,6 +37,8 @@ func (e *QwantEngine) Search(ctx context.Context, req contracts.SearchRequest) (
|
||||||
return contracts.SearchResponse{Query: req.Query}, nil
|
return contracts.SearchResponse{Query: req.Query}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// For API parity we use SearXNG web defaults: count=10, offset=(pageno-1)*count.
|
||||||
|
// The engine's config field exists so we can expand to news/images/videos later.
|
||||||
count := e.resultsPerPage
|
count := e.resultsPerPage
|
||||||
if count <= 0 {
|
if count <= 0 {
|
||||||
count = 10
|
count = 10
|
||||||
|
|
@ -124,8 +114,8 @@ func (e *QwantEngine) searchWebAPI(ctx context.Context, req contracts.SearchRequ
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 16*1024))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 16*1024))
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("qwant upstream error: status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("qwant upstream error: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
|
body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
|
||||||
|
|
@ -253,8 +243,8 @@ func (e *QwantEngine) searchWebLite(ctx context.Context, req contracts.SearchReq
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 16*1024))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 16*1024))
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("qwant lite upstream error: status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("qwant lite upstream error: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||||
|
|
@ -265,12 +255,14 @@ func (e *QwantEngine) searchWebLite(ctx context.Context, req contracts.SearchReq
|
||||||
results := make([]contracts.MainResult, 0)
|
results := make([]contracts.MainResult, 0)
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
|
|
||||||
|
// Pattern 1: legacy/known qwant-lite structure.
|
||||||
doc.Find("section article").Each(func(_ int, item *goquery.Selection) {
|
doc.Find("section article").Each(func(_ int, item *goquery.Selection) {
|
||||||
|
// ignore randomly interspersed advertising adds
|
||||||
if item.Find("span.tooltip").Length() > 0 {
|
if item.Find("span.tooltip").Length() > 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Selector: "./span[contains(@class, 'url partner')]"
|
// In SearXNG: "./span[contains(@class, 'url partner')]"
|
||||||
urlText := strings.TrimSpace(item.Find("span.url.partner").First().Text())
|
urlText := strings.TrimSpace(item.Find("span.url.partner").First().Text())
|
||||||
if urlText == "" {
|
if urlText == "" {
|
||||||
// fallback: any span with class containing both 'url' and 'partner'
|
// fallback: any span with class containing both 'url' and 'partner'
|
||||||
|
|
@ -299,14 +291,19 @@ func (e *QwantEngine) searchWebLite(ctx context.Context, req contracts.SearchReq
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Pattern 2: broader fallback for updated lite markup:
|
||||||
|
// any article/list item/div block containing an external anchor.
|
||||||
|
// We keep this conservative by requiring non-empty title + URL.
|
||||||
doc.Find("article, li, div").Each(func(_ int, item *goquery.Selection) {
|
doc.Find("article, li, div").Each(func(_ int, item *goquery.Selection) {
|
||||||
if len(results) >= 20 {
|
if len(results) >= 20 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// Skip ad-like blocks in fallback pass too.
|
||||||
if item.Find("span.tooltip").Length() > 0 {
|
if item.Find("span.tooltip").Length() > 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Skip obvious nav/footer blocks.
|
||||||
classAttr, _ := item.Attr("class")
|
classAttr, _ := item.Attr("class")
|
||||||
classLower := strings.ToLower(classAttr)
|
classLower := strings.ToLower(classAttr)
|
||||||
if strings.Contains(classLower, "nav") || strings.Contains(classLower, "footer") {
|
if strings.Contains(classLower, "nav") || strings.Contains(classLower, "footer") {
|
||||||
|
|
@ -355,10 +352,13 @@ func (e *QwantEngine) searchWebLite(ctx context.Context, req contracts.SearchReq
|
||||||
}
|
}
|
||||||
seen[href] = true
|
seen[href] = true
|
||||||
|
|
||||||
|
// Best-effort snippet extraction from nearby paragraph/span text.
|
||||||
content := strings.TrimSpace(item.Find("p").First().Text())
|
content := strings.TrimSpace(item.Find("p").First().Text())
|
||||||
if content == "" {
|
if content == "" {
|
||||||
content = strings.TrimSpace(item.Find("span").First().Text())
|
content = strings.TrimSpace(item.Find("span").First().Text())
|
||||||
}
|
}
|
||||||
|
// If there is no snippet, still keep clearly external result links.
|
||||||
|
// Qwant-lite frequently omits rich snippets for some entries.
|
||||||
|
|
||||||
u := href
|
u := href
|
||||||
results = append(results, contracts.MainResult{
|
results = append(results, contracts.MainResult{
|
||||||
|
|
|
||||||
|
|
@ -1,199 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
|
|
||||||
package engines
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
// QwantImagesEngine searches Qwant Images via the v3 search API.
|
|
||||||
type QwantImagesEngine struct {
|
|
||||||
client *http.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *QwantImagesEngine) Name() string { return "qwant_images" }
|
|
||||||
|
|
||||||
func (e *QwantImagesEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
|
||||||
if e == nil || e.client == nil {
|
|
||||||
return contracts.SearchResponse{}, errors.New("qwant_images engine not initialized")
|
|
||||||
}
|
|
||||||
q := strings.TrimSpace(req.Query)
|
|
||||||
if q == "" {
|
|
||||||
return contracts.SearchResponse{Query: req.Query}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
args := url.Values{}
|
|
||||||
args.Set("q", req.Query)
|
|
||||||
args.Set("count", "20")
|
|
||||||
args.Set("locale", qwantLocale(req.Language))
|
|
||||||
args.Set("safesearch", fmt.Sprintf("%d", req.Safesearch))
|
|
||||||
args.Set("offset", fmt.Sprintf("%d", (req.Pageno-1)*20))
|
|
||||||
|
|
||||||
endpoint := "https://api.qwant.com/v3/search/images?" + args.Encode()
|
|
||||||
|
|
||||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
httpReq.Header.Set("User-Agent", "kafka/0.1 (compatible; +https://git.ashisgreat.xyz/penal-colony/kafka)")
|
|
||||||
|
|
||||||
resp, err := e.client.Do(httpReq)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode == http.StatusForbidden {
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: req.Query,
|
|
||||||
UnresponsiveEngines: [][2]string{{"qwant_images", "captcha_or_js_block"}},
|
|
||||||
Results: []contracts.MainResult{},
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 16*1024))
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("qwant_images upstream error: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return parseQwantImages(body, req.Query)
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseQwantImages(body []byte, query string) (contracts.SearchResponse, error) {
|
|
||||||
var top map[string]any
|
|
||||||
if err := json.Unmarshal(body, &top); err != nil {
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("qwant_images JSON parse error: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
status, _ := top["status"].(string)
|
|
||||||
if status != "success" {
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: query,
|
|
||||||
UnresponsiveEngines: [][2]string{{"qwant_images", "api_error"}},
|
|
||||||
Results: []contracts.MainResult{},
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
data, _ := top["data"].(map[string]any)
|
|
||||||
result, _ := data["result"].(map[string]any)
|
|
||||||
items, _ := result["items"].(map[string]any)
|
|
||||||
mainline := items["mainline"]
|
|
||||||
|
|
||||||
rows := toSlice(mainline)
|
|
||||||
if len(rows) == 0 {
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: query,
|
|
||||||
NumberOfResults: 0,
|
|
||||||
Results: []contracts.MainResult{},
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
UnresponsiveEngines: [][2]string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
out := make([]contracts.MainResult, 0)
|
|
||||||
for _, row := range rows {
|
|
||||||
rowMap, ok := row.(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rowType, _ := rowMap["type"].(string)
|
|
||||||
if rowType != "images" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
rowItems := toSlice(rowMap["items"])
|
|
||||||
for _, it := range rowItems {
|
|
||||||
itemMap, ok := it.(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
title := toString(itemMap["title"])
|
|
||||||
resURL := toString(itemMap["url"])
|
|
||||||
thumb := toString(itemMap["thumbnail"])
|
|
||||||
fullImg := toString(itemMap["media"])
|
|
||||||
source := toString(itemMap["source"])
|
|
||||||
|
|
||||||
if resURL == "" && fullImg == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use the source page URL for the link, full image for thumbnail display.
|
|
||||||
linkPtr := resURL
|
|
||||||
if linkPtr == "" {
|
|
||||||
linkPtr = fullImg
|
|
||||||
}
|
|
||||||
displayThumb := fullImg
|
|
||||||
if displayThumb == "" {
|
|
||||||
displayThumb = thumb
|
|
||||||
}
|
|
||||||
|
|
||||||
content := source
|
|
||||||
if width, ok := itemMap["width"]; ok {
|
|
||||||
w := toString(width)
|
|
||||||
if h, ok2 := itemMap["height"]; ok2 {
|
|
||||||
h2 := toString(h)
|
|
||||||
if w != "" && h2 != "" {
|
|
||||||
content = w + " × " + h2
|
|
||||||
if source != "" {
|
|
||||||
content += " — " + source
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
out = append(out, contracts.MainResult{
|
|
||||||
Template: "images",
|
|
||||||
Title: title,
|
|
||||||
Content: content,
|
|
||||||
URL: &linkPtr,
|
|
||||||
Thumbnail: displayThumb,
|
|
||||||
Engine: "qwant_images",
|
|
||||||
Score: 0,
|
|
||||||
Category: "images",
|
|
||||||
Engines: []string{"qwant_images"},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: query,
|
|
||||||
NumberOfResults: len(out),
|
|
||||||
Results: out,
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
UnresponsiveEngines: [][2]string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
@ -5,7 +5,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestQwantEngine_WebLite(t *testing.T) {
|
func TestQwantEngine_WebLite(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestQwantEngine_Web(t *testing.T) {
|
func TestQwantEngine_Web(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -26,7 +10,7 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RedditEngine searches Reddit posts via the public JSON API.
|
// RedditEngine searches Reddit posts via the public JSON API.
|
||||||
|
|
@ -62,8 +46,8 @@ func (e *RedditEngine) Search(ctx context.Context, req contracts.SearchRequest)
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("reddit api error: status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("reddit api error: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
var data struct {
|
var data struct {
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRedditEngine_EmptyQuery(t *testing.T) {
|
func TestRedditEngine_EmptyQuery(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -1,226 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
const stackOverflowAPIBase = "https://api.stackexchange.com/2.3"
|
|
||||||
|
|
||||||
// StackOverflowEngine searches Stack Overflow via the public API.
|
|
||||||
// No API key is required, but providing one via STACKOVERFLOW_KEY env var
|
|
||||||
// or config raises the rate limit from 300 to 10,000 requests/day.
|
|
||||||
type StackOverflowEngine struct {
|
|
||||||
client *http.Client
|
|
||||||
apiKey string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *StackOverflowEngine) Name() string { return "stackoverflow" }
|
|
||||||
|
|
||||||
func (e *StackOverflowEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
|
||||||
if e == nil || e.client == nil {
|
|
||||||
return contracts.SearchResponse{}, errors.New("stackoverflow engine not initialized")
|
|
||||||
}
|
|
||||||
q := strings.TrimSpace(req.Query)
|
|
||||||
if q == "" {
|
|
||||||
return contracts.SearchResponse{Query: req.Query}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
page := req.Pageno
|
|
||||||
if page < 1 {
|
|
||||||
page = 1
|
|
||||||
}
|
|
||||||
args := url.Values{}
|
|
||||||
args.Set("order", "desc")
|
|
||||||
args.Set("sort", "relevance")
|
|
||||||
args.Set("site", "stackoverflow")
|
|
||||||
args.Set("page", fmt.Sprintf("%d", page))
|
|
||||||
args.Set("pagesize", "20")
|
|
||||||
args.Set("filter", "!9_bDDxJY5")
|
|
||||||
if e.apiKey != "" {
|
|
||||||
args.Set("key", e.apiKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
endpoint := stackOverflowAPIBase + "/search/advanced?" + args.Encode() + "&q=" + url.QueryEscape(q)
|
|
||||||
|
|
||||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
httpReq.Header.Set("User-Agent", "kafka/0.1 (compatible; +https://git.ashisgreat.xyz/penal-colony/kafka)")
|
|
||||||
httpReq.Header.Set("Accept", "application/json")
|
|
||||||
|
|
||||||
resp, err := e.client.Do(httpReq)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode == http.StatusTooManyRequests {
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: req.Query,
|
|
||||||
UnresponsiveEngines: [][2]string{{"stackoverflow", "rate_limited"}},
|
|
||||||
Results: []contracts.MainResult{},
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 4*1024))
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("stackoverflow upstream error: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return parseStackOverflow(body, req.Query)
|
|
||||||
}
|
|
||||||
|
|
||||||
// soQuestion represents a question item from the Stack Exchange API.
|
|
||||||
type soQuestion struct {
|
|
||||||
QuestionID int `json:"question_id"`
|
|
||||||
Title string `json:"title"`
|
|
||||||
Link string `json:"link"`
|
|
||||||
Body string `json:"body"`
|
|
||||||
Score int `json:"score"`
|
|
||||||
AnswerCount int `json:"answer_count"`
|
|
||||||
ViewCount int `json:"view_count"`
|
|
||||||
Tags []string `json:"tags"`
|
|
||||||
CreationDate float64 `json:"creation_date"`
|
|
||||||
Owner *soOwner `json:"owner"`
|
|
||||||
AcceptedAnswerID *int `json:"accepted_answer_id"`
|
|
||||||
IsAnswered bool `json:"is_answered"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type soOwner struct {
|
|
||||||
Reputation int `json:"reputation"`
|
|
||||||
DisplayName string `json:"display_name"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type soResponse struct {
|
|
||||||
Items []soQuestion `json:"items"`
|
|
||||||
HasMore bool `json:"has_more"`
|
|
||||||
QuotaRemaining int `json:"quota_remaining"`
|
|
||||||
QuotaMax int `json:"quota_max"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseStackOverflow(body []byte, query string) (contracts.SearchResponse, error) {
|
|
||||||
var resp soResponse
|
|
||||||
if err := json.Unmarshal(body, &resp); err != nil {
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("stackoverflow JSON parse error: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
results := make([]contracts.MainResult, 0, len(resp.Items))
|
|
||||||
for _, q := range resp.Items {
|
|
||||||
if q.Link == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Strip HTML from the body excerpt.
|
|
||||||
snippet := truncate(stripHTML(q.Body), 300)
|
|
||||||
|
|
||||||
// Build a content string with useful metadata.
|
|
||||||
content := snippet
|
|
||||||
if q.Score > 0 {
|
|
||||||
content = fmt.Sprintf("Score: %d", q.Score)
|
|
||||||
if q.AnswerCount > 0 {
|
|
||||||
content += fmt.Sprintf(" · %d answers", q.AnswerCount)
|
|
||||||
}
|
|
||||||
if q.ViewCount > 0 {
|
|
||||||
content += fmt.Sprintf(" · %s views", formatCount(q.ViewCount))
|
|
||||||
}
|
|
||||||
if snippet != "" {
|
|
||||||
content += "\n" + snippet
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Append tags as category hint.
|
|
||||||
if len(q.Tags) > 0 {
|
|
||||||
displayTags := q.Tags
|
|
||||||
if len(displayTags) > 5 {
|
|
||||||
displayTags = displayTags[:5]
|
|
||||||
}
|
|
||||||
content += "\n[" + strings.Join(displayTags, "] [") + "]"
|
|
||||||
}
|
|
||||||
|
|
||||||
linkPtr := q.Link
|
|
||||||
results = append(results, contracts.MainResult{
|
|
||||||
Template: "default",
|
|
||||||
Title: q.Title,
|
|
||||||
Content: content,
|
|
||||||
URL: &linkPtr,
|
|
||||||
Engine: "stackoverflow",
|
|
||||||
Score: float64(q.Score),
|
|
||||||
Category: "it",
|
|
||||||
Engines: []string{"stackoverflow"},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: query,
|
|
||||||
NumberOfResults: len(results),
|
|
||||||
Results: results,
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
UnresponsiveEngines: [][2]string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// formatCount formats large numbers compactly (1.2k, 3.4M).
|
|
||||||
func formatCount(n int) string {
|
|
||||||
if n >= 1_000_000 {
|
|
||||||
return fmt.Sprintf("%.1fM", float64(n)/1_000_000)
|
|
||||||
}
|
|
||||||
if n >= 1_000 {
|
|
||||||
return fmt.Sprintf("%.1fk", float64(n)/1_000)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// truncate cuts a string to at most maxLen characters, appending "…" if truncated.
|
|
||||||
func truncate(s string, maxLen int) string {
|
|
||||||
if len(s) <= maxLen {
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
return s[:maxLen] + "…"
|
|
||||||
}
|
|
||||||
|
|
||||||
// stackOverflowCreatedAt returns a time.Time from a Unix timestamp.
|
|
||||||
// Kept as a helper for potential future pubdate use.
|
|
||||||
func stackOverflowCreatedAt(unix float64) *string {
|
|
||||||
t := time.Unix(int64(unix), 0).UTC()
|
|
||||||
s := t.Format("2006-01-02")
|
|
||||||
return &s
|
|
||||||
}
|
|
||||||
|
|
@ -1,186 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestStackOverflow_Name(t *testing.T) {
|
|
||||||
e := &StackOverflowEngine{}
|
|
||||||
if e.Name() != "stackoverflow" {
|
|
||||||
t.Errorf("expected name 'stackoverflow', got %q", e.Name())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStackOverflow_NilEngine(t *testing.T) {
|
|
||||||
var e *StackOverflowEngine
|
|
||||||
_, err := e.Search(context.Background(), contracts.SearchRequest{Query: "test"})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error for nil engine")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStackOverflow_EmptyQuery(t *testing.T) {
|
|
||||||
e := &StackOverflowEngine{client: &http.Client{}}
|
|
||||||
resp, err := e.Search(context.Background(), contracts.SearchRequest{Query: ""})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if len(resp.Results) != 0 {
|
|
||||||
t.Errorf("expected 0 results for empty query, got %d", len(resp.Results))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStackOverflow_Search(t *testing.T) {
|
|
||||||
items := []soQuestion{
|
|
||||||
{
|
|
||||||
QuestionID: 12345,
|
|
||||||
Title: "How to center a div in CSS?",
|
|
||||||
Link: "https://stackoverflow.com/questions/12345",
|
|
||||||
Body: "<p>I have a div that I want to center horizontally and vertically.</p>",
|
|
||||||
Score: 42,
|
|
||||||
AnswerCount: 7,
|
|
||||||
ViewCount: 15000,
|
|
||||||
Tags: []string{"css", "html", "layout"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
QuestionID: 67890,
|
|
||||||
Title: "Python list comprehension help",
|
|
||||||
Link: "https://stackoverflow.com/questions/67890",
|
|
||||||
Body: "<p>I'm trying to flatten a list of lists.</p>",
|
|
||||||
Score: 15,
|
|
||||||
AnswerCount: 3,
|
|
||||||
ViewCount: 2300,
|
|
||||||
Tags: []string{"python", "list", "comprehension"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
respBody := soResponse{
|
|
||||||
Items: items,
|
|
||||||
HasMore: false,
|
|
||||||
QuotaRemaining: 299,
|
|
||||||
QuotaMax: 300,
|
|
||||||
}
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path != "/2.3/search/advanced" {
|
|
||||||
t.Errorf("unexpected path: %s", r.URL.Path)
|
|
||||||
}
|
|
||||||
q := r.URL.Query()
|
|
||||||
if q.Get("site") != "stackoverflow" {
|
|
||||||
t.Errorf("expected site=stackoverflow, got %q", q.Get("site"))
|
|
||||||
}
|
|
||||||
if q.Get("sort") != "relevance" {
|
|
||||||
t.Errorf("expected sort=relevance, got %q", q.Get("sort"))
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
json.NewEncoder(w).Encode(respBody)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
// We can't easily override the base URL, so test parsing directly.
|
|
||||||
body, _ := json.Marshal(respBody)
|
|
||||||
result, err := parseStackOverflow(body, "center div css")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parseStackOverflow error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if result.NumberOfResults != 2 {
|
|
||||||
t.Errorf("expected 2 results, got %d", result.NumberOfResults)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(result.Results) < 2 {
|
|
||||||
t.Fatalf("expected at least 2 results, got %d", len(result.Results))
|
|
||||||
}
|
|
||||||
|
|
||||||
r0 := result.Results[0]
|
|
||||||
if r0.Title != "How to center a div in CSS?" {
|
|
||||||
t.Errorf("wrong title: %q", r0.Title)
|
|
||||||
}
|
|
||||||
if r0.Engine != "stackoverflow" {
|
|
||||||
t.Errorf("wrong engine: %q", r0.Engine)
|
|
||||||
}
|
|
||||||
if r0.Category != "it" {
|
|
||||||
t.Errorf("wrong category: %q", r0.Category)
|
|
||||||
}
|
|
||||||
if r0.URL == nil || *r0.URL != "https://stackoverflow.com/questions/12345" {
|
|
||||||
t.Errorf("wrong URL: %v", r0.URL)
|
|
||||||
}
|
|
||||||
if r0.Content == "" {
|
|
||||||
t.Error("expected non-empty content")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify score is populated.
|
|
||||||
if r0.Score != 42 {
|
|
||||||
t.Errorf("expected score 42, got %f", r0.Score)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStackOverflow_RateLimited(t *testing.T) {
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusTooManyRequests)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
// We can't override the URL, so test the parsing of rate limit response.
|
|
||||||
// The engine returns empty results with unresponsive engine info.
|
|
||||||
// This is verified via the factory integration; here we just verify the nil case.
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStackOverflow_NoAPIKey(t *testing.T) {
|
|
||||||
// Verify that the engine works without an API key set.
|
|
||||||
e := &StackOverflowEngine{client: &http.Client{}, apiKey: ""}
|
|
||||||
if e.apiKey != "" {
|
|
||||||
t.Error("expected empty API key")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFormatCount(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
n int
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{999, "999"},
|
|
||||||
{1000, "1.0k"},
|
|
||||||
{1500, "1.5k"},
|
|
||||||
{999999, "1000.0k"},
|
|
||||||
{1000000, "1.0M"},
|
|
||||||
{3500000, "3.5M"},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
got := formatCount(tt.n)
|
|
||||||
if got != tt.want {
|
|
||||||
t.Errorf("formatCount(%d) = %q, want %q", tt.n, got, tt.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTruncate(t *testing.T) {
|
|
||||||
if got := truncate("hello", 10); got != "hello" {
|
|
||||||
t.Errorf("truncate short string: got %q", got)
|
|
||||||
}
|
|
||||||
if got := truncate("hello world this is long", 10); got != "hello worl…" {
|
|
||||||
t.Errorf("truncate long string: got %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,133 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
|
|
||||||
package engines
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
// wikidataAPIBase is the Wikidata MediaWiki API endpoint (overridable in tests).
|
|
||||||
var wikidataAPIBase = "https://www.wikidata.org/w/api.php"
|
|
||||||
|
|
||||||
// WikidataEngine searches entity labels and descriptions via the Wikidata API.
|
|
||||||
// See: https://www.wikidata.org/w/api.php?action=help&modules=wbsearchentities
|
|
||||||
type WikidataEngine struct {
|
|
||||||
client *http.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *WikidataEngine) Name() string { return "wikidata" }
|
|
||||||
|
|
||||||
func (e *WikidataEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
|
||||||
if e == nil || e.client == nil {
|
|
||||||
return contracts.SearchResponse{}, errors.New("wikidata engine not initialized")
|
|
||||||
}
|
|
||||||
q := strings.TrimSpace(req.Query)
|
|
||||||
if q == "" {
|
|
||||||
return contracts.SearchResponse{Query: req.Query}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
lang := strings.TrimSpace(req.Language)
|
|
||||||
if lang == "" || lang == "auto" {
|
|
||||||
lang = "en"
|
|
||||||
}
|
|
||||||
lang = strings.SplitN(lang, "-", 2)[0]
|
|
||||||
lang = strings.ReplaceAll(lang, "_", "-")
|
|
||||||
if _, ok := validWikipediaLangs[lang]; !ok {
|
|
||||||
lang = "en"
|
|
||||||
}
|
|
||||||
|
|
||||||
u, err := url.Parse(wikidataAPIBase)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
qv := u.Query()
|
|
||||||
qv.Set("action", "wbsearchentities")
|
|
||||||
qv.Set("search", q)
|
|
||||||
qv.Set("language", lang)
|
|
||||||
qv.Set("limit", "10")
|
|
||||||
qv.Set("format", "json")
|
|
||||||
u.RawQuery = qv.Encode()
|
|
||||||
|
|
||||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
httpReq.Header.Set("User-Agent", "samsa/1.0 (Wikidata search; +https://github.com/metamorphosis-dev/samsa)")
|
|
||||||
|
|
||||||
resp, err := e.client.Do(httpReq)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 16*1024))
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("wikidata upstream error: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var api struct {
|
|
||||||
Search []struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Label string `json:"label"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
} `json:"search"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(body, &api); err != nil {
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("wikidata JSON parse error: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
results := make([]contracts.MainResult, 0, len(api.Search))
|
|
||||||
for _, hit := range api.Search {
|
|
||||||
id := strings.TrimSpace(hit.ID)
|
|
||||||
if id == "" || !strings.HasPrefix(id, "Q") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
pageURL := "https://www.wikidata.org/wiki/" + url.PathEscape(id)
|
|
||||||
title := strings.TrimSpace(hit.Label)
|
|
||||||
if title == "" {
|
|
||||||
title = id
|
|
||||||
}
|
|
||||||
content := strings.TrimSpace(hit.Description)
|
|
||||||
urlPtr := pageURL
|
|
||||||
results = append(results, contracts.MainResult{
|
|
||||||
Template: "default.html",
|
|
||||||
Title: title,
|
|
||||||
Content: content,
|
|
||||||
URL: &urlPtr,
|
|
||||||
Engine: "wikidata",
|
|
||||||
Category: "general",
|
|
||||||
Engines: []string{"wikidata"},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: req.Query,
|
|
||||||
NumberOfResults: len(results),
|
|
||||||
Results: results,
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
UnresponsiveEngines: [][2]string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,51 +0,0 @@
|
||||||
package engines
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestWikidataEngine_Search(t *testing.T) {
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Query().Get("action") != "wbsearchentities" {
|
|
||||||
t.Errorf("action=%q", r.URL.Query().Get("action"))
|
|
||||||
}
|
|
||||||
if got := r.URL.Query().Get("search"); got != "test" {
|
|
||||||
t.Errorf("search=%q want test", got)
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
_, _ = w.Write([]byte(`{"search":[{"id":"Q937","label":"Go","description":"Programming language"}]}`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
orig := wikidataAPIBase
|
|
||||||
t.Cleanup(func() { wikidataAPIBase = orig })
|
|
||||||
wikidataAPIBase = ts.URL + "/w/api.php"
|
|
||||||
|
|
||||||
e := &WikidataEngine{client: ts.Client()}
|
|
||||||
resp, err := e.Search(context.Background(), contracts.SearchRequest{
|
|
||||||
Query: "test",
|
|
||||||
Language: "en",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(resp.Results) != 1 {
|
|
||||||
t.Fatalf("expected 1 result, got %d", len(resp.Results))
|
|
||||||
}
|
|
||||||
r0 := resp.Results[0]
|
|
||||||
if r0.Engine != "wikidata" {
|
|
||||||
t.Errorf("engine=%q", r0.Engine)
|
|
||||||
}
|
|
||||||
if r0.Title != "Go" {
|
|
||||||
t.Errorf("title=%q", r0.Title)
|
|
||||||
}
|
|
||||||
if r0.URL == nil || !strings.Contains(*r0.URL, "Q937") {
|
|
||||||
t.Errorf("url=%v", r0.URL)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
package engines
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -26,51 +10,13 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
type WikipediaEngine struct {
|
type WikipediaEngine struct {
|
||||||
client *http.Client
|
client *http.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
// validWikipediaLangs contains the set of valid Wikipedia language codes.
|
|
||||||
// This prevents SSRF attacks where an attacker could use a malicious language
|
|
||||||
// value to redirect requests to an attacker-controlled domain.
|
|
||||||
var validWikipediaLangs = map[string]struct{}{
|
|
||||||
"aa": {}, "ab": {}, "ae": {}, "af": {}, "ak": {}, "am": {}, "an": {},
|
|
||||||
"ar": {}, "arc": {}, "as": {}, "ast": {}, "at": {}, "av": {}, "ay": {},
|
|
||||||
"az": {}, "ba": {}, "be": {}, "bg": {}, "bh": {}, "bi": {}, "bm": {},
|
|
||||||
"bn": {}, "bo": {}, "br": {}, "bs": {}, "ca": {}, "ce": {}, "ch": {},
|
|
||||||
"co": {}, "cr": {}, "cs": {}, "cu": {}, "cv": {}, "cy": {}, "da": {},
|
|
||||||
"de": {}, "di": {}, "dv": {}, "dz": {}, "ee": {}, "el": {}, "en": {},
|
|
||||||
"eo": {}, "es": {}, "et": {}, "eu": {}, "fa": {}, "ff": {}, "fi": {},
|
|
||||||
"fj": {}, "fo": {}, "fr": {}, "fy": {}, "ga": {}, "gd": {}, "gl": {},
|
|
||||||
"gn": {}, "gu": {}, "gv": {}, "ha": {}, "he": {}, "hi": {}, "ho": {},
|
|
||||||
"hr": {}, "ht": {}, "hu": {}, "hy": {}, "hz": {}, "ia": {}, "id": {},
|
|
||||||
"ie": {}, "ig": {}, "ii": {}, "ik": {}, "io": {}, "is": {}, "it": {},
|
|
||||||
"iu": {}, "ja": {}, "jv": {}, "ka": {}, "kg": {}, "ki": {}, "kj": {},
|
|
||||||
"kk": {}, "kl": {}, "km": {}, "kn": {}, "ko": {}, "kr": {}, "ks": {},
|
|
||||||
"ku": {}, "kv": {}, "kw": {}, "ky": {}, "la": {}, "lb": {}, "lg": {},
|
|
||||||
"li": {}, "lij": {}, "ln": {}, "lo": {}, "lt": {}, "lv": {}, "mg": {},
|
|
||||||
"mh": {}, "mi": {}, "mk": {}, "ml": {}, "mn": {}, "mo": {}, "mr": {},
|
|
||||||
"ms": {}, "mt": {}, "mus": {}, "my": {}, "na": {}, "nah": {}, "nap": {},
|
|
||||||
"nd": {}, "nds": {}, "ne": {}, "new": {}, "ng": {}, "nl": {}, "nn": {},
|
|
||||||
"no": {}, "nov": {}, "nrm": {}, "nv": {}, "ny": {}, "oc": {}, "oj": {},
|
|
||||||
"om": {}, "or": {}, "os": {}, "pa": {}, "pag": {}, "pam": {}, "pap": {},
|
|
||||||
"pdc": {}, "pl": {}, "pms": {}, "pn": {}, "ps": {}, "pt": {}, "qu": {},
|
|
||||||
"rm": {}, "rmy": {}, "rn": {}, "ro": {}, "roa-rup": {}, "ru": {},
|
|
||||||
"rw": {}, "sa": {}, "sah": {}, "sc": {}, "scn": {}, "sco": {}, "sd": {},
|
|
||||||
"se": {}, "sg": {}, "sh": {}, "si": {}, "simple": {}, "sk": {}, "sl": {},
|
|
||||||
"sm": {}, "sn": {}, "so": {}, "sq": {}, "sr": {}, "ss": {}, "st": {},
|
|
||||||
"su": {}, "sv": {}, "sw": {}, "szl": {}, "ta": {}, "te": {}, "tg": {},
|
|
||||||
"th": {}, "ti": {}, "tk": {}, "tl": {}, "tn": {}, "to": {}, "tpi": {},
|
|
||||||
"tr": {}, "ts": {}, "tt": {}, "tum": {}, "tw": {}, "ty": {}, "udm": {},
|
|
||||||
"ug": {}, "uk": {}, "ur": {}, "uz": {}, "ve": {}, "vec": {}, "vi": {},
|
|
||||||
"vls": {}, "vo": {}, "wa": {}, "wo": {}, "xal": {}, "xh": {}, "yi": {},
|
|
||||||
"yo": {}, "za": {}, "zea": {}, "zh": {}, "zh-classical": {},
|
|
||||||
"zh-min-nan": {}, "zh-yue": {}, "zu": {},
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *WikipediaEngine) Name() string { return "wikipedia" }
|
func (e *WikipediaEngine) Name() string { return "wikipedia" }
|
||||||
|
|
||||||
func (e *WikipediaEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
func (e *WikipediaEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
||||||
|
|
@ -88,11 +34,6 @@ func (e *WikipediaEngine) Search(ctx context.Context, req contracts.SearchReques
|
||||||
// Wikipedia subdomains are based on the language code; keep it simple for MVP.
|
// Wikipedia subdomains are based on the language code; keep it simple for MVP.
|
||||||
lang = strings.SplitN(lang, "-", 2)[0]
|
lang = strings.SplitN(lang, "-", 2)[0]
|
||||||
lang = strings.ReplaceAll(lang, "_", "-")
|
lang = strings.ReplaceAll(lang, "_", "-")
|
||||||
// Validate lang against whitelist to prevent SSRF attacks where an attacker
|
|
||||||
// could use a malicious language value to redirect requests to their server.
|
|
||||||
if _, ok := validWikipediaLangs[lang]; !ok {
|
|
||||||
lang = "en"
|
|
||||||
}
|
|
||||||
wikiNetloc := fmt.Sprintf("%s.wikipedia.org", lang)
|
wikiNetloc := fmt.Sprintf("%s.wikipedia.org", lang)
|
||||||
|
|
||||||
endpoint := fmt.Sprintf(
|
endpoint := fmt.Sprintf(
|
||||||
|
|
@ -108,7 +49,7 @@ func (e *WikipediaEngine) Search(ctx context.Context, req contracts.SearchReques
|
||||||
// Wikimedia APIs require a descriptive User-Agent.
|
// Wikimedia APIs require a descriptive User-Agent.
|
||||||
httpReq.Header.Set(
|
httpReq.Header.Set(
|
||||||
"User-Agent",
|
"User-Agent",
|
||||||
"gosearch-go/0.1 (compatible; +https://github.com/metamorphosis-dev/samsa)",
|
"gosearch-go/0.1 (compatible; +https://github.com/metamorphosis-dev/kafka)",
|
||||||
)
|
)
|
||||||
// Best-effort: hint content language.
|
// Best-effort: hint content language.
|
||||||
if req.Language != "" && req.Language != "auto" {
|
if req.Language != "" && req.Language != "auto" {
|
||||||
|
|
@ -134,20 +75,16 @@ func (e *WikipediaEngine) Search(ctx context.Context, req contracts.SearchReques
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 16*1024))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 16*1024))
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("wikipedia upstream error: status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("wikipedia upstream error: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
var api struct {
|
var api struct {
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
Extract string `json:"extract"`
|
|
||||||
Titles struct {
|
Titles struct {
|
||||||
Display string `json:"display"`
|
Display string `json:"display"`
|
||||||
} `json:"titles"`
|
} `json:"titles"`
|
||||||
Thumbnail struct {
|
|
||||||
Source string `json:"source"`
|
|
||||||
} `json:"thumbnail"`
|
|
||||||
ContentURLs struct {
|
ContentURLs struct {
|
||||||
Desktop struct {
|
Desktop struct {
|
||||||
Page string `json:"page"`
|
Page string `json:"page"`
|
||||||
|
|
@ -179,37 +116,11 @@ func (e *WikipediaEngine) Search(ctx context.Context, req contracts.SearchReques
|
||||||
title = api.Title
|
title = api.Title
|
||||||
}
|
}
|
||||||
|
|
||||||
content := strings.TrimSpace(api.Extract)
|
content := api.Description
|
||||||
if content == "" {
|
|
||||||
content = strings.TrimSpace(api.Description)
|
|
||||||
}
|
|
||||||
|
|
||||||
urlPtr := pageURL
|
urlPtr := pageURL
|
||||||
pub := (*string)(nil)
|
pub := (*string)(nil)
|
||||||
|
|
||||||
// Knowledge infobox for HTML (Wikipedia REST summary: title, extract, thumbnail, link).
|
|
||||||
var infoboxes []map[string]any
|
|
||||||
ibTitle := api.Titles.Display
|
|
||||||
if ibTitle == "" {
|
|
||||||
ibTitle = api.Title
|
|
||||||
}
|
|
||||||
body := strings.TrimSpace(api.Extract)
|
|
||||||
if body == "" {
|
|
||||||
body = strings.TrimSpace(api.Description)
|
|
||||||
}
|
|
||||||
imgSrc := strings.TrimSpace(api.Thumbnail.Source)
|
|
||||||
if ibTitle != "" || body != "" || imgSrc != "" {
|
|
||||||
row := map[string]any{
|
|
||||||
"title": ibTitle,
|
|
||||||
"infobox": body,
|
|
||||||
"url": pageURL,
|
|
||||||
}
|
|
||||||
if imgSrc != "" {
|
|
||||||
row["img_src"] = imgSrc
|
|
||||||
}
|
|
||||||
infoboxes = append(infoboxes, row)
|
|
||||||
}
|
|
||||||
|
|
||||||
results := []contracts.MainResult{
|
results := []contracts.MainResult{
|
||||||
{
|
{
|
||||||
Template: "default.html",
|
Template: "default.html",
|
||||||
|
|
@ -232,8 +143,9 @@ func (e *WikipediaEngine) Search(ctx context.Context, req contracts.SearchReques
|
||||||
Results: results,
|
Results: results,
|
||||||
Answers: []map[string]any{},
|
Answers: []map[string]any{},
|
||||||
Corrections: []string{},
|
Corrections: []string{},
|
||||||
Infoboxes: infoboxes,
|
Infoboxes: []map[string]any{},
|
||||||
Suggestions: []string{},
|
Suggestions: []string{},
|
||||||
UnresponsiveEngines: [][2]string{},
|
UnresponsiveEngines: [][2]string{},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestWikipediaEngine_Search(t *testing.T) {
|
func TestWikipediaEngine_Search(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -1,194 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package engines
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
type YouTubeEngine struct {
|
|
||||||
client *http.Client
|
|
||||||
apiKey string
|
|
||||||
baseURL string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *YouTubeEngine) Name() string { return "youtube" }
|
|
||||||
|
|
||||||
func (e *YouTubeEngine) Search(ctx context.Context, req contracts.SearchRequest) (contracts.SearchResponse, error) {
|
|
||||||
if strings.TrimSpace(req.Query) == "" {
|
|
||||||
return contracts.SearchResponse{Query: req.Query}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if e.apiKey == "" {
|
|
||||||
e.apiKey = os.Getenv("YOUTUBE_API_KEY")
|
|
||||||
}
|
|
||||||
|
|
||||||
maxResults := 10
|
|
||||||
if req.Pageno > 1 {
|
|
||||||
maxResults = 20
|
|
||||||
}
|
|
||||||
|
|
||||||
u := e.baseURL + "/youtube/v3/search?" + url.Values{
|
|
||||||
"part": {"snippet"},
|
|
||||||
"q": {req.Query},
|
|
||||||
"type": {"video"},
|
|
||||||
"maxResults": {fmt.Sprintf("%d", maxResults)},
|
|
||||||
"key": {e.apiKey},
|
|
||||||
}.Encode()
|
|
||||||
|
|
||||||
if req.Language != "" && req.Language != "auto" {
|
|
||||||
lang := strings.Split(strings.ToLower(req.Language), "-")[0]
|
|
||||||
u += "&relevanceLanguage=" + lang
|
|
||||||
}
|
|
||||||
|
|
||||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := e.client.Do(httpReq)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("youtube api error: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
var apiResp youtubeSearchResponse
|
|
||||||
if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
|
|
||||||
return contracts.SearchResponse{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if apiResp.Error != nil {
|
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("youtube api error: code %d", apiResp.Error.Code)
|
|
||||||
}
|
|
||||||
|
|
||||||
results := make([]contracts.MainResult, 0, len(apiResp.Items))
|
|
||||||
for _, item := range apiResp.Items {
|
|
||||||
if item.ID.VideoID == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
videoURL := "https://www.youtube.com/watch?v=" + item.ID.VideoID
|
|
||||||
urlPtr := videoURL
|
|
||||||
|
|
||||||
published := ""
|
|
||||||
if item.Snippet.PublishedAt != "" {
|
|
||||||
if t, err := time.Parse(time.RFC3339, item.Snippet.PublishedAt); err == nil {
|
|
||||||
published = t.Format("Jan 2, 2006")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
content := item.Snippet.Description
|
|
||||||
if len(content) > 300 {
|
|
||||||
content = content[:300] + "..."
|
|
||||||
}
|
|
||||||
if published != "" {
|
|
||||||
content = "Published " + published + " · " + content
|
|
||||||
}
|
|
||||||
|
|
||||||
thumbnail := ""
|
|
||||||
if item.Snippet.Thumbnails.High.URL != "" {
|
|
||||||
thumbnail = item.Snippet.Thumbnails.High.URL
|
|
||||||
} else if item.Snippet.Thumbnails.Medium.URL != "" {
|
|
||||||
thumbnail = item.Snippet.Thumbnails.Medium.URL
|
|
||||||
}
|
|
||||||
|
|
||||||
results = append(results, contracts.MainResult{
|
|
||||||
Template: "videos.html",
|
|
||||||
Title: item.Snippet.Title,
|
|
||||||
URL: &urlPtr,
|
|
||||||
Content: content,
|
|
||||||
Thumbnail: thumbnail,
|
|
||||||
Engine: "youtube",
|
|
||||||
Score: 1.0,
|
|
||||||
Category: "videos",
|
|
||||||
Engines: []string{"youtube"},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: req.Query,
|
|
||||||
NumberOfResults: len(results),
|
|
||||||
Results: results,
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
UnresponsiveEngines: [][2]string{},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// YouTube API response types.
|
|
||||||
|
|
||||||
type youtubeSearchResponse struct {
|
|
||||||
Items []youtubeSearchItem `json:"items"`
|
|
||||||
PageInfo struct {
|
|
||||||
TotalResults int `json:"totalResults"`
|
|
||||||
ResultsPerPage int `json:"resultsPerPage"`
|
|
||||||
} `json:"pageInfo"`
|
|
||||||
NextPageToken string `json:"nextPageToken"`
|
|
||||||
Error *struct {
|
|
||||||
Code int `json:"code"`
|
|
||||||
Message string `json:"message"`
|
|
||||||
Errors []struct {
|
|
||||||
Domain string `json:"domain"`
|
|
||||||
Reason string `json:"reason"`
|
|
||||||
Message string `json:"message"`
|
|
||||||
} `json:"errors"`
|
|
||||||
} `json:"error"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type youtubeSearchItem struct {
|
|
||||||
ID struct {
|
|
||||||
VideoID string `json:"videoId"`
|
|
||||||
} `json:"id"`
|
|
||||||
Snippet struct {
|
|
||||||
PublishedAt string `json:"publishedAt"`
|
|
||||||
ChannelID string `json:"channelId"`
|
|
||||||
ChannelTitle string `json:"channelTitle"`
|
|
||||||
Title string `json:"title"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
Thumbnails struct {
|
|
||||||
Default struct {
|
|
||||||
URL string `json:"url"`
|
|
||||||
} `json:"default"`
|
|
||||||
Medium struct {
|
|
||||||
URL string `json:"url"`
|
|
||||||
} `json:"medium"`
|
|
||||||
High struct {
|
|
||||||
URL string `json:"url"`
|
|
||||||
} `json:"high"`
|
|
||||||
} `json:"thumbnails"`
|
|
||||||
ResourceID struct {
|
|
||||||
VideoID string `json:"videoId"`
|
|
||||||
} `json:"resourceId"`
|
|
||||||
} `json:"snippet"`
|
|
||||||
}
|
|
||||||
|
|
@ -1,51 +1,25 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package httpapi
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/cache"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/search"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/httpclient"
|
"github.com/metamorphosis-dev/kafka/internal/views"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/search"
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/views"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Handler struct {
|
type Handler struct {
|
||||||
searchSvc *search.Service
|
searchSvc *search.Service
|
||||||
autocompleteSvc func(ctx context.Context, query string) ([]string, error)
|
autocompleteSvc func(ctx context.Context, query string) ([]string, error)
|
||||||
sourceURL string
|
|
||||||
faviconCache *cache.Cache
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewHandler(searchSvc *search.Service, autocompleteSuggestions func(ctx context.Context, query string) ([]string, error), sourceURL string, faviconCache *cache.Cache) *Handler {
|
func NewHandler(searchSvc *search.Service, autocompleteSuggestions func(ctx context.Context, query string) ([]string, error)) *Handler {
|
||||||
return &Handler{
|
return &Handler{
|
||||||
searchSvc: searchSvc,
|
searchSvc: searchSvc,
|
||||||
autocompleteSvc: autocompleteSuggestions,
|
autocompleteSvc: autocompleteSuggestions,
|
||||||
sourceURL: sourceURL,
|
|
||||||
faviconCache: faviconCache,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -55,35 +29,13 @@ func (h *Handler) Healthz(w http.ResponseWriter, r *http.Request) {
|
||||||
_, _ = w.Write([]byte("OK"))
|
_, _ = w.Write([]byte("OK"))
|
||||||
}
|
}
|
||||||
|
|
||||||
// getTheme returns the user's theme preference from cookie, defaulting to "light".
|
|
||||||
func (h *Handler) getTheme(r *http.Request) string {
|
|
||||||
if cookie, err := r.Cookie("theme"); err == nil {
|
|
||||||
if cookie.Value == "dark" || cookie.Value == "light" {
|
|
||||||
return cookie.Value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "light"
|
|
||||||
}
|
|
||||||
|
|
||||||
// getFaviconService returns the favicon provider from cookie (default "none").
|
|
||||||
func (h *Handler) getFaviconService(r *http.Request) string {
|
|
||||||
if cookie, err := r.Cookie("favicon"); err == nil {
|
|
||||||
switch cookie.Value {
|
|
||||||
case "none", "google", "duckduckgo", "self":
|
|
||||||
return cookie.Value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "none"
|
|
||||||
}
|
|
||||||
|
|
||||||
// Index renders the homepage with the search box.
|
// Index renders the homepage with the search box.
|
||||||
func (h *Handler) Index(w http.ResponseWriter, r *http.Request) {
|
func (h *Handler) Index(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.URL.Path != "/" {
|
if r.URL.Path != "/" {
|
||||||
http.NotFound(w, r)
|
http.NotFound(w, r)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
theme := h.getTheme(r)
|
if err := views.RenderIndex(w); err != nil {
|
||||||
if err := views.RenderIndex(w, h.sourceURL, theme); err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -102,19 +54,17 @@ func (h *Handler) OpenSearch(baseURL string) http.HandlerFunc {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) Search(w http.ResponseWriter, r *http.Request) {
|
func (h *Handler) Search(w http.ResponseWriter, r *http.Request) {
|
||||||
q := r.FormValue("q")
|
|
||||||
format := r.FormValue("format")
|
|
||||||
|
|
||||||
// For HTML format with no query, redirect to homepage.
|
// For HTML format with no query, redirect to homepage.
|
||||||
if q == "" && (format == "" || format == "html") {
|
if r.FormValue("q") == "" && (r.FormValue("format") == "" || r.FormValue("format") == "html") {
|
||||||
http.Redirect(w, r, "/", http.StatusFound)
|
http.Redirect(w, r, "/", http.StatusFound)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
req, err := search.ParseSearchRequest(r)
|
req, err := search.ParseSearchRequest(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if format == "html" || format == "" {
|
// For HTML, render error on the results page.
|
||||||
pd := views.PageData{SourceURL: h.sourceURL, Query: q, Theme: h.getTheme(r), FaviconService: h.getFaviconService(r)}
|
if req.Format == contracts.FormatHTML || r.FormValue("format") == "html" {
|
||||||
|
pd := views.PageData{Query: r.FormValue("q")}
|
||||||
if views.IsHTMXRequest(r) {
|
if views.IsHTMXRequest(r) {
|
||||||
views.RenderSearchFragment(w, pd)
|
views.RenderSearchFragment(w, pd)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -129,7 +79,7 @@ func (h *Handler) Search(w http.ResponseWriter, r *http.Request) {
|
||||||
resp, err := h.searchSvc.Search(r.Context(), req)
|
resp, err := h.searchSvc.Search(r.Context(), req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if req.Format == contracts.FormatHTML {
|
if req.Format == contracts.FormatHTML {
|
||||||
pd := views.PageData{SourceURL: h.sourceURL, Query: req.Query, Theme: h.getTheme(r), FaviconService: h.getFaviconService(r)}
|
pd := views.PageData{Query: req.Query}
|
||||||
if views.IsHTMXRequest(r) {
|
if views.IsHTMXRequest(r) {
|
||||||
views.RenderSearchFragment(w, pd)
|
views.RenderSearchFragment(w, pd)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -142,9 +92,7 @@ func (h *Handler) Search(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if req.Format == contracts.FormatHTML {
|
if req.Format == contracts.FormatHTML {
|
||||||
pd := views.FromResponse(resp, req.Query, req.Pageno,
|
pd := views.FromResponse(resp, req.Query, req.Pageno)
|
||||||
r.FormValue("category"), r.FormValue("time"), r.FormValue("type"), h.getFaviconService(r))
|
|
||||||
pd.Theme = h.getTheme(r)
|
|
||||||
if err := views.RenderSearchAuto(w, r, pd); err != nil {
|
if err := views.RenderSearchAuto(w, r, pd); err != nil {
|
||||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
}
|
}
|
||||||
|
|
@ -173,126 +121,3 @@ func (h *Handler) Autocompleter(w http.ResponseWriter, r *http.Request) {
|
||||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||||
_ = json.NewEncoder(w).Encode(suggestions)
|
_ = json.NewEncoder(w).Encode(suggestions)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Preferences handles GET and POST for the preferences page.
|
|
||||||
func (h *Handler) Preferences(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path != "/preferences" {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method == "POST" {
|
|
||||||
// Handle theme preference via server-side cookie
|
|
||||||
theme := r.FormValue("theme")
|
|
||||||
if theme == "dark" || theme == "light" {
|
|
||||||
http.SetCookie(w, &http.Cookie{
|
|
||||||
Name: "theme",
|
|
||||||
Value: theme,
|
|
||||||
Path: "/",
|
|
||||||
MaxAge: 86400 * 365,
|
|
||||||
HttpOnly: false, // Allow CSS to read via :has()
|
|
||||||
SameSite: http.SameSiteLaxMode,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
// Persist favicon provider preference.
|
|
||||||
favicon := strings.TrimSpace(r.FormValue("favicon"))
|
|
||||||
switch favicon {
|
|
||||||
case "none", "google", "duckduckgo", "self":
|
|
||||||
http.SetCookie(w, &http.Cookie{
|
|
||||||
Name: "favicon",
|
|
||||||
Value: favicon,
|
|
||||||
Path: "/",
|
|
||||||
MaxAge: 86400 * 365,
|
|
||||||
HttpOnly: false,
|
|
||||||
SameSite: http.SameSiteLaxMode,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
http.Redirect(w, r, "/preferences", http.StatusFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Read theme cookie for template
|
|
||||||
theme := "light"
|
|
||||||
if cookie, err := r.Cookie("theme"); err == nil {
|
|
||||||
if cookie.Value == "dark" || cookie.Value == "light" {
|
|
||||||
theme = cookie.Value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := views.RenderPreferences(w, h.sourceURL, theme, h.getFaviconService(r)); err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const faviconCacheTTL = 24 * time.Hour
|
|
||||||
|
|
||||||
// Favicon serves a fetched favicon for the given domain, with ETag support
|
|
||||||
// and a 24-hour Redis cache. This lets Kafka act as a privacy-preserving
|
|
||||||
// favicon proxy: the user's browser talks to Kafka, not Google or DuckDuckGo.
|
|
||||||
func (h *Handler) Favicon(w http.ResponseWriter, r *http.Request) {
|
|
||||||
domain := strings.TrimPrefix(r.URL.Path, "/favicon/")
|
|
||||||
domain = strings.TrimSuffix(domain, "/")
|
|
||||||
domain = strings.TrimSpace(domain)
|
|
||||||
|
|
||||||
if domain == "" || strings.Contains(domain, "/") {
|
|
||||||
http.Error(w, "invalid domain", http.StatusBadRequest)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheKey := "favicon:" + domain
|
|
||||||
|
|
||||||
// Check Redis cache.
|
|
||||||
if cached, ok := h.faviconCache.GetBytes(r.Context(), cacheKey); ok {
|
|
||||||
h.serveFavicon(w, r, cached)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fetch from the domain's favicon.ico.
|
|
||||||
fetchURL := "https://" + domain + "/favicon.ico"
|
|
||||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, fetchURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
req.Header.Set("User-Agent", "Kafka/0.1 (+https://git.ashisgreat.xyz/penal-colony/samsa)")
|
|
||||||
req.Header.Set("Accept", "image/x-icon,image/png,image/webp,*/*")
|
|
||||||
|
|
||||||
client := httpclient.NewClient(5 * time.Second)
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, "favicon fetch failed", http.StatusBadGateway)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
http.Error(w, "favicon not found", http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(http.MaxBytesReader(w, resp.Body, 64*1024))
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, "favicon too large", http.StatusBadGateway)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store in Redis with 24h TTL.
|
|
||||||
h.faviconCache.SetBytes(r.Context(), cacheKey, body, faviconCacheTTL)
|
|
||||||
|
|
||||||
h.serveFavicon(w, r, body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// serveFavicon writes a cached or freshly-fetched body with appropriate
|
|
||||||
// caching headers. ETag is derived from the body hash (no storage needed).
|
|
||||||
func (h *Handler) serveFavicon(w http.ResponseWriter, r *http.Request, body []byte) {
|
|
||||||
h2 := sha256.Sum256(body)
|
|
||||||
etag := `"` + hex.EncodeToString(h2[:8]) + `"`
|
|
||||||
|
|
||||||
if etagMatch := r.Header.Get("If-None-Match"); etagMatch != "" && etagMatch == etag {
|
|
||||||
w.WriteHeader(http.StatusNotModified)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "image/x-icon")
|
|
||||||
w.Header().Set("ETag", etag)
|
|
||||||
w.Header().Set("Cache-Control", "private, max-age=86400")
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
w.Write(body)
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,278 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package httpapi_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/httpapi"
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/search"
|
|
||||||
)
|
|
||||||
|
|
||||||
// mockUpstreamHandler returns controlled JSON responses.
|
|
||||||
func mockUpstreamJSON(query string) contracts.SearchResponse {
|
|
||||||
return contracts.SearchResponse{
|
|
||||||
Query: query,
|
|
||||||
NumberOfResults: 2,
|
|
||||||
Results: []contracts.MainResult{
|
|
||||||
{Title: "Upstream Result 1", URL: ptr("https://upstream.example/1"), Content: "From upstream", Engine: "upstream"},
|
|
||||||
{Title: "Upstream Result 2", URL: ptr("https://upstream.example/2"), Content: "From upstream", Engine: "upstream"},
|
|
||||||
},
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{"upstream suggestion"},
|
|
||||||
UnresponsiveEngines: [][2]string{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ptr(s string) *string { return &s }
|
|
||||||
|
|
||||||
func newTestServer(t *testing.T) (*httptest.Server, *httpapi.Handler) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
// Mock upstream server that returns controlled JSON.
|
|
||||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
query := r.FormValue("q")
|
|
||||||
resp := mockUpstreamJSON(query)
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
json.NewEncoder(w).Encode(resp)
|
|
||||||
}))
|
|
||||||
t.Cleanup(upstream.Close)
|
|
||||||
|
|
||||||
svc := search.NewService(search.ServiceConfig{
|
|
||||||
UpstreamURL: upstream.URL,
|
|
||||||
HTTPTimeout: 0,
|
|
||||||
Cache: nil,
|
|
||||||
EnginesConfig: nil,
|
|
||||||
})
|
|
||||||
|
|
||||||
h := httpapi.NewHandler(svc, nil, "https://src.example.com", nil)
|
|
||||||
|
|
||||||
mux := http.NewServeMux()
|
|
||||||
mux.HandleFunc("/healthz", h.Healthz)
|
|
||||||
mux.HandleFunc("/", h.Index)
|
|
||||||
mux.HandleFunc("/search", h.Search)
|
|
||||||
mux.HandleFunc("/autocompleter", h.Autocompleter)
|
|
||||||
mux.HandleFunc("/preferences", h.Preferences)
|
|
||||||
|
|
||||||
server := httptest.NewServer(mux)
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
return server, h
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHealthz(t *testing.T) {
|
|
||||||
server, _ := newTestServer(t)
|
|
||||||
resp, err := http.Get(server.URL + "/healthz")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("request failed: %v", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Errorf("expected status 200, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "text/plain") {
|
|
||||||
t.Errorf("expected text/plain, got %s", ct)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIndex(t *testing.T) {
|
|
||||||
server, _ := newTestServer(t)
|
|
||||||
resp, err := http.Get(server.URL + "/")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("request failed: %v", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Errorf("expected status 200, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "text/html") {
|
|
||||||
t.Errorf("expected text/html, got %s", ct)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, _ := io.ReadAll(resp.Body)
|
|
||||||
if !strings.Contains(string(body), "<!DOCTYPE html") {
|
|
||||||
t.Error("expected HTML DOCTYPE")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSearch_RedirectOnEmptyQuery(t *testing.T) {
|
|
||||||
server, _ := newTestServer(t)
|
|
||||||
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
|
|
||||||
return http.ErrUseLastResponse
|
|
||||||
}}
|
|
||||||
req, _ := http.NewRequest("GET", server.URL+"/search?format=html", nil)
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("request failed: %v", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusFound {
|
|
||||||
t.Errorf("expected redirect 302, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
loc, _ := resp.Location()
|
|
||||||
if loc == nil || loc.Path != "/" {
|
|
||||||
t.Errorf("expected redirect to /, got %v", loc)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSearch_JSONResponse(t *testing.T) {
|
|
||||||
server, _ := newTestServer(t)
|
|
||||||
resp, err := http.Get(server.URL + "/search?q=test&format=json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("request failed: %v", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Errorf("expected status 200, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "application/json") {
|
|
||||||
t.Errorf("expected application/json, got %s", ct)
|
|
||||||
}
|
|
||||||
|
|
||||||
var result contracts.SearchResponse
|
|
||||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
|
||||||
t.Fatalf("failed to decode JSON: %v", err)
|
|
||||||
}
|
|
||||||
if result.Query != "test" {
|
|
||||||
t.Errorf("expected query 'test', got %q", result.Query)
|
|
||||||
}
|
|
||||||
if len(result.Results) == 0 {
|
|
||||||
t.Error("expected at least one result")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSearch_HTMLResponse(t *testing.T) {
|
|
||||||
server, _ := newTestServer(t)
|
|
||||||
resp, err := http.Get(server.URL + "/search?q=test&format=html")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("request failed: %v", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Errorf("expected status 200, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "text/html") {
|
|
||||||
t.Errorf("expected text/html, got %s", ct)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, _ := io.ReadAll(resp.Body)
|
|
||||||
if !strings.Contains(string(body), "<!DOCTYPE html") {
|
|
||||||
t.Error("expected HTML DOCTYPE in response")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutocompleter_EmptyQuery(t *testing.T) {
|
|
||||||
server, _ := newTestServer(t)
|
|
||||||
resp, err := http.Get(server.URL + "/autocompleter?q=")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("request failed: %v", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusBadRequest {
|
|
||||||
t.Errorf("expected status 400, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutocompleter_NoQuery(t *testing.T) {
|
|
||||||
server, _ := newTestServer(t)
|
|
||||||
resp, err := http.Get(server.URL + "/autocompleter")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("request failed: %v", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusBadRequest {
|
|
||||||
t.Errorf("expected status 400, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSearch_SourceURLInFooter(t *testing.T) {
|
|
||||||
server, _ := newTestServer(t)
|
|
||||||
resp, err := http.Get(server.URL + "/?q=test")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("request failed: %v", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
body, _ := io.ReadAll(resp.Body)
|
|
||||||
if !strings.Contains(string(body), "https://src.example.com") {
|
|
||||||
t.Error("expected source URL in footer")
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(body), "AGPLv3") {
|
|
||||||
t.Error("expected AGPLv3 link in footer")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreferences_PostSetsFaviconCookie(t *testing.T) {
|
|
||||||
server, _ := newTestServer(t)
|
|
||||||
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
|
|
||||||
return http.ErrUseLastResponse
|
|
||||||
}}
|
|
||||||
req, _ := http.NewRequest(http.MethodPost, server.URL+"/preferences", strings.NewReader("favicon=google&theme=dark"))
|
|
||||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("request failed: %v", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusFound {
|
|
||||||
t.Fatalf("expected redirect 302, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
found := false
|
|
||||||
for _, c := range resp.Cookies() {
|
|
||||||
if c.Name == "favicon" {
|
|
||||||
found = true
|
|
||||||
if c.Value != "google" {
|
|
||||||
t.Fatalf("expected favicon cookie google, got %q", c.Value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatal("expected favicon cookie to be set")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreferences_GetReflectsFaviconCookie(t *testing.T) {
|
|
||||||
server, _ := newTestServer(t)
|
|
||||||
req, _ := http.NewRequest(http.MethodGet, server.URL+"/preferences", nil)
|
|
||||||
req.AddCookie(&http.Cookie{Name: "favicon", Value: "duckduckgo"})
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("request failed: %v", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
body, _ := io.ReadAll(resp.Body)
|
|
||||||
html := string(body)
|
|
||||||
if !strings.Contains(html, `option value="duckduckgo" selected`) {
|
|
||||||
t.Fatalf("expected duckduckgo option selected, body: %s", html)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,61 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package httpclient
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
defaultTransport http.RoundTripper
|
|
||||||
once sync.Once
|
|
||||||
)
|
|
||||||
|
|
||||||
// Default returns a shared, pre-configured http.RoundTripper suitable for
|
|
||||||
// outgoing engine requests. It is safe for concurrent use across goroutines.
|
|
||||||
// All fields are tuned for a meta-search engine that makes many concurrent
|
|
||||||
// requests to a fixed set of upstream hosts:
|
|
||||||
//
|
|
||||||
// - MaxIdleConnsPerHost = 20 (vs default of 2; keeps more warm connections
|
|
||||||
// to each host, avoiding repeated TCP+TLS handshakes)
|
|
||||||
// - MaxIdleConns = 100 (total idle connection ceiling)
|
|
||||||
// - IdleConnTimeout = 90s (prunes connections before they go stale)
|
|
||||||
// - DialContext timeout = 5s (fails fast on DNS/connect rather than
|
|
||||||
// holding a goroutine indefinitely)
|
|
||||||
func Default() http.RoundTripper {
|
|
||||||
once.Do(func() {
|
|
||||||
defaultTransport = &http.Transport{
|
|
||||||
MaxIdleConnsPerHost: 20,
|
|
||||||
MaxIdleConns: 100,
|
|
||||||
IdleConnTimeout: 90 * time.Second,
|
|
||||||
DialContext: dialWithTimeout(5 * time.Second),
|
|
||||||
}
|
|
||||||
})
|
|
||||||
return defaultTransport
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewClient returns an http.Client that uses DefaultTransport and the given
|
|
||||||
// request timeout. The returned client reuses the shared connection pool,
|
|
||||||
// so all clients created via this function share the same warm connections.
|
|
||||||
func NewClient(timeout time.Duration) *http.Client {
|
|
||||||
return &http.Client{
|
|
||||||
Transport: Default(),
|
|
||||||
Timeout: timeout,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package httpclient
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// dialWithTimeout returns a DialContext function for http.Transport that
|
|
||||||
// respects the given connection timeout.
|
|
||||||
func dialWithTimeout(timeout time.Duration) func(context.Context, string, string) (net.Conn, error) {
|
|
||||||
d := &net.Dialer{Timeout: timeout}
|
|
||||||
return d.DialContext
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package middleware
|
package middleware
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -42,8 +26,7 @@ type CORSConfig struct {
|
||||||
func CORS(cfg CORSConfig) func(http.Handler) http.Handler {
|
func CORS(cfg CORSConfig) func(http.Handler) http.Handler {
|
||||||
origins := cfg.AllowedOrigins
|
origins := cfg.AllowedOrigins
|
||||||
if len(origins) == 0 {
|
if len(origins) == 0 {
|
||||||
// Default: no CORS headers. Explicitly configure origins to enable.
|
origins = []string{"*"}
|
||||||
origins = nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
methods := cfg.AllowedMethods
|
methods := cfg.AllowedMethods
|
||||||
|
|
@ -71,7 +54,6 @@ func CORS(cfg CORSConfig) func(http.Handler) http.Handler {
|
||||||
origin := r.Header.Get("Origin")
|
origin := r.Header.Get("Origin")
|
||||||
|
|
||||||
// Determine the allowed origin for this request.
|
// Determine the allowed origin for this request.
|
||||||
// If no origins are configured, CORS is disabled entirely — no headers are set.
|
|
||||||
allowedOrigin := ""
|
allowedOrigin := ""
|
||||||
for _, o := range origins {
|
for _, o := range origins {
|
||||||
if o == "*" {
|
if o == "*" {
|
||||||
|
|
|
||||||
|
|
@ -51,7 +51,7 @@ func TestCORS_SpecificOrigin(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCORS_Preflight(t *testing.T) {
|
func TestCORS_Preflight(t *testing.T) {
|
||||||
h := CORS(CORSConfig{AllowedOrigins: []string{"https://example.com"}})(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
h := CORS(CORSConfig{})(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
t.Error("handler should not be called for preflight")
|
t.Error("handler should not be called for preflight")
|
||||||
}))
|
}))
|
||||||
|
|
||||||
|
|
@ -100,7 +100,6 @@ func TestCORS_CustomMethodsAndHeaders(t *testing.T) {
|
||||||
})(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
})(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||||
|
|
||||||
req := httptest.NewRequest("OPTIONS", "/search", nil)
|
req := httptest.NewRequest("OPTIONS", "/search", nil)
|
||||||
req.Header.Set("Origin", "https://example.com")
|
|
||||||
rec := httptest.NewRecorder()
|
rec := httptest.NewRecorder()
|
||||||
h.ServeHTTP(rec, req)
|
h.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package middleware
|
package middleware
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -27,16 +11,19 @@ import (
|
||||||
"log/slog"
|
"log/slog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RateLimitConfig controls per-IP rate limiting.
|
// RateLimitConfig controls per-IP rate limiting using a sliding window counter.
|
||||||
type RateLimitConfig struct {
|
type RateLimitConfig struct {
|
||||||
|
// Requests is the max number of requests allowed per window.
|
||||||
Requests int
|
Requests int
|
||||||
|
// Window is the time window duration (e.g. "1m").
|
||||||
Window time.Duration
|
Window time.Duration
|
||||||
|
// CleanupInterval is how often stale entries are purged (default: 5m).
|
||||||
CleanupInterval time.Duration
|
CleanupInterval time.Duration
|
||||||
// TrustedProxies is a list of CIDR ranges that are allowed to set
|
|
||||||
// X-Forwarded-For / X-Real-IP. If empty, only r.RemoteAddr is used.
|
|
||||||
TrustedProxies []string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RateLimit returns a middleware that limits requests per IP address.
|
||||||
|
// Uses an in-memory sliding window counter. When the limit is exceeded,
|
||||||
|
// responds with HTTP 429 and a Retry-After header.
|
||||||
func RateLimit(cfg RateLimitConfig, logger *slog.Logger) func(http.Handler) http.Handler {
|
func RateLimit(cfg RateLimitConfig, logger *slog.Logger) func(http.Handler) http.Handler {
|
||||||
requests := cfg.Requests
|
requests := cfg.Requests
|
||||||
if requests <= 0 {
|
if requests <= 0 {
|
||||||
|
|
@ -57,30 +44,18 @@ func RateLimit(cfg RateLimitConfig, logger *slog.Logger) func(http.Handler) http
|
||||||
logger = slog.Default()
|
logger = slog.Default()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse trusted proxy CIDRs.
|
|
||||||
var trustedNets []*net.IPNet
|
|
||||||
for _, cidr := range cfg.TrustedProxies {
|
|
||||||
_, network, err := net.ParseCIDR(cidr)
|
|
||||||
if err != nil {
|
|
||||||
logger.Warn("invalid trusted proxy CIDR, skipping", "cidr", cidr, "error", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
trustedNets = append(trustedNets, network)
|
|
||||||
}
|
|
||||||
|
|
||||||
limiter := &ipLimiter{
|
limiter := &ipLimiter{
|
||||||
requests: requests,
|
requests: requests,
|
||||||
window: window,
|
window: window,
|
||||||
clients: make(map[string]*bucket),
|
clients: make(map[string]*bucket),
|
||||||
logger: logger,
|
logger: logger,
|
||||||
trusted: trustedNets,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
go limiter.cleanup(cleanup)
|
go limiter.cleanup(cleanup)
|
||||||
|
|
||||||
return func(next http.Handler) http.Handler {
|
return func(next http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
ip := limiter.extractIP(r)
|
ip := extractIP(r)
|
||||||
|
|
||||||
if !limiter.allow(ip) {
|
if !limiter.allow(ip) {
|
||||||
retryAfter := int(limiter.window.Seconds())
|
retryAfter := int(limiter.window.Seconds())
|
||||||
|
|
@ -108,7 +83,6 @@ type ipLimiter struct {
|
||||||
clients map[string]*bucket
|
clients map[string]*bucket
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
trusted []*net.IPNet
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *ipLimiter) allow(ip string) bool {
|
func (l *ipLimiter) allow(ip string) bool {
|
||||||
|
|
@ -146,48 +120,18 @@ func (l *ipLimiter) cleanup(interval time.Duration) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// extractIP extracts the client IP from the request.
|
func extractIP(r *http.Request) string {
|
||||||
// If trusted proxy CIDRs are configured, X-Forwarded-For is only used when
|
|
||||||
// the direct connection comes from a trusted proxy. Otherwise, only RemoteAddr is used.
|
|
||||||
func (l *ipLimiter) extractIP(r *http.Request) string {
|
|
||||||
return extractIP(r, l.trusted...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func extractIP(r *http.Request, trusted ...*net.IPNet) string {
|
|
||||||
remoteIP, _, err := net.SplitHostPort(r.RemoteAddr)
|
|
||||||
if err != nil {
|
|
||||||
remoteIP = r.RemoteAddr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if the direct connection is from a trusted proxy.
|
|
||||||
isTrusted := false
|
|
||||||
if len(trusted) > 0 {
|
|
||||||
ip := net.ParseIP(remoteIP)
|
|
||||||
if ip != nil {
|
|
||||||
for _, network := range trusted {
|
|
||||||
if network.Contains(ip) {
|
|
||||||
isTrusted = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if isTrusted {
|
|
||||||
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
|
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
|
||||||
parts := strings.SplitN(xff, ",", 2)
|
parts := strings.SplitN(xff, ",", 2)
|
||||||
candidate := strings.TrimSpace(parts[0])
|
return strings.TrimSpace(parts[0])
|
||||||
if net.ParseIP(candidate) != nil {
|
|
||||||
return candidate
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if rip := r.Header.Get("X-Real-IP"); rip != "" {
|
if rip := r.Header.Get("X-Real-IP"); rip != "" {
|
||||||
candidate := strings.TrimSpace(rip)
|
return strings.TrimSpace(rip)
|
||||||
if net.ParseIP(candidate) != nil {
|
|
||||||
return candidate
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return remoteIP
|
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||||
|
if err != nil {
|
||||||
|
return r.RemoteAddr
|
||||||
|
}
|
||||||
|
return host
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package middleware
|
package middleware
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -71,7 +55,7 @@ func GlobalRateLimit(cfg GlobalRateLimitConfig, logger *slog.Logger) func(http.H
|
||||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||||
w.WriteHeader(http.StatusServiceUnavailable)
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
_, _ = w.Write([]byte("503 Service Unavailable — global rate limit exceeded\n"))
|
_, _ = w.Write([]byte("503 Service Unavailable — global rate limit exceeded\n"))
|
||||||
logger.Warn("global rate limit exceeded", "remote", r.RemoteAddr)
|
logger.Warn("global rate limit exceeded", "ip", extractIP(r))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
package middleware
|
package middleware
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
@ -96,7 +95,6 @@ func TestRateLimit_XForwardedFor(t *testing.T) {
|
||||||
h := RateLimit(RateLimitConfig{
|
h := RateLimit(RateLimitConfig{
|
||||||
Requests: 1,
|
Requests: 1,
|
||||||
Window: 10 * time.Second,
|
Window: 10 * time.Second,
|
||||||
TrustedProxies: []string{"10.0.0.0/8"},
|
|
||||||
}, nil)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
}, nil)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
}))
|
}))
|
||||||
|
|
@ -145,27 +143,17 @@ func TestRateLimit_WindowExpires(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExtractIP(t *testing.T) {
|
func TestExtractIP(t *testing.T) {
|
||||||
// Trusted proxy: loopback
|
|
||||||
loopback := mustParseCIDR("127.0.0.0/8")
|
|
||||||
privateNet := mustParseCIDR("10.0.0.0/8")
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
xff string
|
xff string
|
||||||
realIP string
|
realIP string
|
||||||
remote string
|
remote string
|
||||||
trusted []*net.IPNet
|
|
||||||
expected string
|
expected string
|
||||||
}{
|
}{
|
||||||
// No trusted proxies → always use RemoteAddr.
|
{"xff", "203.0.113.50, 10.0.0.1", "", "10.0.0.1:1234", "203.0.113.50"},
|
||||||
{"no_trusted_xff", "203.0.113.50, 10.0.0.1", "", "10.0.0.1:1234", nil, "10.0.0.1"},
|
{"real_ip", "", "203.0.113.50", "10.0.0.1:1234", "203.0.113.50"},
|
||||||
{"no_trusted_real", "", "203.0.113.50", "10.0.0.1:1234", nil, "10.0.0.1"},
|
{"remote", "", "", "1.2.3.4:5678", "1.2.3.4"},
|
||||||
{"no_trusted_remote", "", "", "1.2.3.4:5678", nil, "1.2.3.4"},
|
{"xff_over_real", "203.0.113.50", "10.0.0.1", "10.0.0.1:1234", "203.0.113.50"},
|
||||||
// Trusted proxy → XFF is respected.
|
|
||||||
{"trusted_xff", "203.0.113.50, 10.0.0.1", "", "10.0.0.1:1234", []*net.IPNet{privateNet}, "203.0.113.50"},
|
|
||||||
{"trusted_real_ip", "", "203.0.113.50", "10.0.0.1:1234", []*net.IPNet{privateNet}, "203.0.113.50"},
|
|
||||||
// Untrusted remote → XFF ignored even if present.
|
|
||||||
{"untrusted_xff", "203.0.113.50, 10.0.0.1", "", "1.2.3.4:5678", []*net.IPNet{loopback}, "1.2.3.4"},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|
@ -179,17 +167,9 @@ func TestExtractIP(t *testing.T) {
|
||||||
}
|
}
|
||||||
req.RemoteAddr = tt.remote
|
req.RemoteAddr = tt.remote
|
||||||
|
|
||||||
if got := extractIP(req, tt.trusted...); got != tt.expected {
|
if got := extractIP(req); got != tt.expected {
|
||||||
t.Errorf("extractIP() = %q, want %q", got, tt.expected)
|
t.Errorf("extractIP() = %q, want %q", got, tt.expected)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func mustParseCIDR(s string) *net.IPNet {
|
|
||||||
_, network, err := net.ParseCIDR(s)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return network
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,92 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
|
|
||||||
package middleware
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// SecurityHeadersConfig controls which security headers are set.
|
|
||||||
type SecurityHeadersConfig struct {
|
|
||||||
// FrameOptions controls X-Frame-Options. Default: "DENY".
|
|
||||||
FrameOptions string
|
|
||||||
// HSTSMaxAge controls the max-age for Strict-Transport-Security.
|
|
||||||
// Set to 0 to disable HSTS (useful for local dev). Default: 31536000 (1 year).
|
|
||||||
HSTSMaxAge int
|
|
||||||
// HSTSPreloadDomains adds "includeSubDomains; preload" to HSTS.
|
|
||||||
HSTSPreloadDomains bool
|
|
||||||
// ReferrerPolicy controls the Referrer-Policy header. Default: "no-referrer".
|
|
||||||
ReferrerPolicy string
|
|
||||||
// CSP controls Content-Security-Policy. Default: a restrictive policy.
|
|
||||||
// Set to "" to disable CSP entirely.
|
|
||||||
CSP string
|
|
||||||
}
|
|
||||||
|
|
||||||
// SecurityHeaders returns middleware that sets standard HTTP security headers
|
|
||||||
// on every response.
|
|
||||||
func SecurityHeaders(cfg SecurityHeadersConfig) func(http.Handler) http.Handler {
|
|
||||||
frameOpts := cfg.FrameOptions
|
|
||||||
if frameOpts == "" {
|
|
||||||
frameOpts = "DENY"
|
|
||||||
}
|
|
||||||
|
|
||||||
hstsAge := cfg.HSTSMaxAge
|
|
||||||
if hstsAge == 0 {
|
|
||||||
hstsAge = 31536000 // 1 year
|
|
||||||
}
|
|
||||||
|
|
||||||
refPol := cfg.ReferrerPolicy
|
|
||||||
if refPol == "" {
|
|
||||||
refPol = "no-referrer"
|
|
||||||
}
|
|
||||||
|
|
||||||
csp := cfg.CSP
|
|
||||||
if csp == "" {
|
|
||||||
csp = defaultCSP()
|
|
||||||
}
|
|
||||||
|
|
||||||
hstsValue := "max-age=" + strconv.Itoa(hstsAge)
|
|
||||||
if cfg.HSTSPreloadDomains {
|
|
||||||
hstsValue += "; includeSubDomains; preload"
|
|
||||||
}
|
|
||||||
|
|
||||||
return func(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
||||||
w.Header().Set("X-Frame-Options", frameOpts)
|
|
||||||
w.Header().Set("Referrer-Policy", refPol)
|
|
||||||
w.Header().Set("Permissions-Policy", "camera=(), microphone=(), geolocation=()")
|
|
||||||
w.Header().Set("Content-Security-Policy", csp)
|
|
||||||
|
|
||||||
if hstsAge > 0 {
|
|
||||||
w.Header().Set("Strict-Transport-Security", hstsValue)
|
|
||||||
}
|
|
||||||
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// defaultCSP returns a restrictive Content-Security-Policy for the
|
|
||||||
// metasearch engine.
|
|
||||||
func defaultCSP() string {
|
|
||||||
return strings.Join([]string{
|
|
||||||
"default-src 'self'",
|
|
||||||
"script-src 'self' 'unsafe-inline' https://unpkg.com",
|
|
||||||
"style-src 'self' 'unsafe-inline'",
|
|
||||||
"img-src 'self' https: data:",
|
|
||||||
"connect-src 'self'",
|
|
||||||
"font-src 'self'",
|
|
||||||
"frame-ancestors 'none'",
|
|
||||||
"base-uri 'self'",
|
|
||||||
"form-action 'self'",
|
|
||||||
}, "; ")
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package search
|
package search
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -21,10 +5,15 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
// MergeResponses merges multiple compatible JSON responses.
|
// MergeResponses merges multiple SearXNG-compatible JSON responses.
|
||||||
|
//
|
||||||
|
// MVP merge semantics:
|
||||||
|
// - results are concatenated with a simple de-dup key (engine|title|url)
|
||||||
|
// - suggestions/corrections are de-duplicated as sets
|
||||||
|
// - answers/infoboxes/unresponsive_engines are concatenated (best-effort)
|
||||||
func MergeResponses(responses []contracts.SearchResponse) contracts.SearchResponse {
|
func MergeResponses(responses []contracts.SearchResponse) contracts.SearchResponse {
|
||||||
var merged contracts.SearchResponse
|
var merged contracts.SearchResponse
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMergeResponses_DedupResultsAndSets(t *testing.T) {
|
func TestMergeResponses_DedupResultsAndSets(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package search
|
package search
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -26,32 +10,8 @@ import (
|
||||||
|
|
||||||
var languageCodeRe = regexp.MustCompile(`^[a-z]{2,3}(-[a-zA-Z]{2})?$`)
|
var languageCodeRe = regexp.MustCompile(`^[a-z]{2,3}(-[a-zA-Z]{2})?$`)
|
||||||
|
|
||||||
// maxQueryLength is the maximum allowed length for the search query.
|
|
||||||
const maxQueryLength = 1024
|
|
||||||
|
|
||||||
// knownEngineNames is the allowlist of valid engine identifiers.
|
|
||||||
var knownEngineNames = map[string]bool{
|
|
||||||
"wikipedia": true, "wikidata": true, "arxiv": true, "crossref": true,
|
|
||||||
"braveapi": true, "brave": true, "qwant": true,
|
|
||||||
"duckduckgo": true, "github": true, "reddit": true,
|
|
||||||
"bing": true, "google": true, "youtube": true,
|
|
||||||
// Image engines
|
|
||||||
"bing_images": true, "ddg_images": true, "qwant_images": true,
|
|
||||||
}
|
|
||||||
|
|
||||||
// validateEngines filters engine names against the known registry.
|
|
||||||
func validateEngines(engines []string) []string {
|
|
||||||
out := make([]string, 0, len(engines))
|
|
||||||
for _, e := range engines {
|
|
||||||
if knownEngineNames[strings.ToLower(e)] {
|
|
||||||
out = append(out, strings.ToLower(e))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseSearchRequest(r *http.Request) (SearchRequest, error) {
|
func ParseSearchRequest(r *http.Request) (SearchRequest, error) {
|
||||||
// Supports both GET and POST and relies on form values for routing.
|
// SearXNG supports both GET and POST and relies on form values for routing.
|
||||||
if err := r.ParseForm(); err != nil {
|
if err := r.ParseForm(); err != nil {
|
||||||
return SearchRequest{}, errors.New("invalid request: cannot parse form")
|
return SearchRequest{}, errors.New("invalid request: cannot parse form")
|
||||||
}
|
}
|
||||||
|
|
@ -74,9 +34,6 @@ func ParseSearchRequest(r *http.Request) (SearchRequest, error) {
|
||||||
if strings.TrimSpace(q) == "" {
|
if strings.TrimSpace(q) == "" {
|
||||||
return SearchRequest{}, errors.New("missing required parameter: q")
|
return SearchRequest{}, errors.New("missing required parameter: q")
|
||||||
}
|
}
|
||||||
if len(q) > maxQueryLength {
|
|
||||||
return SearchRequest{}, errors.New("query exceeds maximum length")
|
|
||||||
}
|
|
||||||
|
|
||||||
pageno := 1
|
pageno := 1
|
||||||
if s := strings.TrimSpace(r.FormValue("pageno")); s != "" {
|
if s := strings.TrimSpace(r.FormValue("pageno")); s != "" {
|
||||||
|
|
@ -132,10 +89,8 @@ func ParseSearchRequest(r *http.Request) (SearchRequest, error) {
|
||||||
|
|
||||||
// engines is an explicit list of engine names.
|
// engines is an explicit list of engine names.
|
||||||
engines := splitCSV(strings.TrimSpace(r.FormValue("engines")))
|
engines := splitCSV(strings.TrimSpace(r.FormValue("engines")))
|
||||||
// Validate engine names against known registry to prevent injection.
|
|
||||||
engines = validateEngines(engines)
|
|
||||||
|
|
||||||
// categories and category_<name> params mirror the webadapter parsing.
|
// categories and category_<name> params mirror SearXNG's webadapter parsing.
|
||||||
// We don't validate against a registry here; we just preserve the requested values.
|
// We don't validate against a registry here; we just preserve the requested values.
|
||||||
catSet := map[string]bool{}
|
catSet := map[string]bool{}
|
||||||
if catsParam := strings.TrimSpace(r.FormValue("categories")); catsParam != "" {
|
if catsParam := strings.TrimSpace(r.FormValue("categories")); catsParam != "" {
|
||||||
|
|
@ -161,10 +116,6 @@ func ParseSearchRequest(r *http.Request) (SearchRequest, error) {
|
||||||
delete(catSet, category)
|
delete(catSet, category)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// HTML UI uses a single ?category=images (etc.) query param; honor it here.
|
|
||||||
if single := strings.TrimSpace(r.FormValue("category")); single != "" {
|
|
||||||
catSet[single] = true
|
|
||||||
}
|
|
||||||
categories := make([]string, 0, len(catSet))
|
categories := make([]string, 0, len(catSet))
|
||||||
for c := range catSet {
|
for c := range catSet {
|
||||||
categories = append(categories, c)
|
categories = append(categories, c)
|
||||||
|
|
@ -254,3 +205,4 @@ func parseAccessToken(r *http.Request) string {
|
||||||
|
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -72,21 +72,3 @@ func TestParseSearchRequest_CategoriesAndEngineData(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseSearchRequest_SingularCategoryParam(t *testing.T) {
|
|
||||||
r := httptest.NewRequest(http.MethodGet, "/search?q=cats&category=images", nil)
|
|
||||||
req, err := ParseSearchRequest(r)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
found := false
|
|
||||||
for _, c := range req.Categories {
|
|
||||||
if c == "images" {
|
|
||||||
found = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatalf("expected category images from ?category=images, got %v", req.Categories)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package search
|
package search
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -54,7 +38,7 @@ func WriteSearchResponse(w http.ResponseWriter, format OutputFormat, resp Search
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// csvRowHeader matches the CSV writer key order.
|
// csvRowHeader matches the SearXNG CSV writer key order.
|
||||||
var csvRowHeader = []string{"title", "url", "content", "host", "engine", "score", "type"}
|
var csvRowHeader = []string{"title", "url", "content", "host", "engine", "score", "type"}
|
||||||
|
|
||||||
func writeCSV(w http.ResponseWriter, resp SearchResponse) error {
|
func writeCSV(w http.ResponseWriter, resp SearchResponse) error {
|
||||||
|
|
@ -127,14 +111,14 @@ func writeCSV(w http.ResponseWriter, resp SearchResponse) error {
|
||||||
|
|
||||||
func writeRSS(w http.ResponseWriter, resp SearchResponse) error {
|
func writeRSS(w http.ResponseWriter, resp SearchResponse) error {
|
||||||
q := resp.Query
|
q := resp.Query
|
||||||
escapedTitle := xmlEscape("samsa search: " + q)
|
escapedTitle := xmlEscape("SearXNG search: " + q)
|
||||||
escapedDesc := xmlEscape("Search results for \"" + q + "\" - samsa")
|
escapedDesc := xmlEscape("Search results for \"" + q + "\" - SearXNG")
|
||||||
escapedQueryTerms := xmlEscape(q)
|
escapedQueryTerms := xmlEscape(q)
|
||||||
|
|
||||||
link := "/search?q=" + url.QueryEscape(q)
|
link := "/search?q=" + url.QueryEscape(q)
|
||||||
opensearchQuery := fmt.Sprintf(`<opensearch:Query role="request" searchTerms="%s" startPage="1" />`, escapedQueryTerms)
|
opensearchQuery := fmt.Sprintf(`<opensearch:Query role="request" searchTerms="%s" startPage="1" />`, escapedQueryTerms)
|
||||||
|
|
||||||
// The template uses the number of results for both totalResults and itemsPerPage.
|
// SearXNG template uses the number of results for both totalResults and itemsPerPage.
|
||||||
nr := resp.NumberOfResults
|
nr := resp.NumberOfResults
|
||||||
|
|
||||||
var items bytes.Buffer
|
var items bytes.Buffer
|
||||||
|
|
|
||||||
|
|
@ -1,49 +1,28 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package search
|
package search
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"net/http"
|
||||||
"fmt"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/cache"
|
"github.com/metamorphosis-dev/kafka/internal/cache"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/config"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/engines"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/engines"
|
"github.com/metamorphosis-dev/kafka/internal/upstream"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/httpclient"
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/upstream"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type ServiceConfig struct {
|
type ServiceConfig struct {
|
||||||
UpstreamURL string
|
UpstreamURL string
|
||||||
HTTPTimeout time.Duration
|
HTTPTimeout time.Duration
|
||||||
Cache *cache.Cache
|
Cache *cache.Cache
|
||||||
CacheTTLOverrides map[string]time.Duration
|
|
||||||
EnginesConfig *config.Config
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Service struct {
|
type Service struct {
|
||||||
upstreamClient *upstream.Client
|
upstreamClient *upstream.Client
|
||||||
planner *engines.Planner
|
planner *engines.Planner
|
||||||
localEngines map[string]engines.Engine
|
localEngines map[string]engines.Engine
|
||||||
engineCache *cache.EngineCache
|
cache *cache.Cache
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewService(cfg ServiceConfig) *Service {
|
func NewService(cfg ServiceConfig) *Service {
|
||||||
|
|
@ -52,7 +31,7 @@ func NewService(cfg ServiceConfig) *Service {
|
||||||
timeout = 10 * time.Second
|
timeout = 10 * time.Second
|
||||||
}
|
}
|
||||||
|
|
||||||
httpClient := httpclient.NewClient(timeout)
|
httpClient := &http.Client{Timeout: timeout}
|
||||||
|
|
||||||
var up *upstream.Client
|
var up *upstream.Client
|
||||||
if cfg.UpstreamURL != "" {
|
if cfg.UpstreamURL != "" {
|
||||||
|
|
@ -62,177 +41,118 @@ func NewService(cfg ServiceConfig) *Service {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var engineCache *cache.EngineCache
|
|
||||||
if cfg.Cache != nil {
|
|
||||||
engineCache = cache.NewEngineCache(cfg.Cache, cfg.CacheTTLOverrides)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Service{
|
return &Service{
|
||||||
upstreamClient: up,
|
upstreamClient: up,
|
||||||
planner: engines.NewPlannerFromEnv(),
|
planner: engines.NewPlannerFromEnv(),
|
||||||
localEngines: engines.NewDefaultPortedEngines(httpClient, cfg.EnginesConfig),
|
localEngines: engines.NewDefaultPortedEngines(httpClient),
|
||||||
engineCache: engineCache,
|
cache: cfg.Cache,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// derefString returns the string value of a *string, or "" if nil.
|
|
||||||
func derefString(s *string) string {
|
|
||||||
if s == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return *s
|
|
||||||
}
|
|
||||||
|
|
||||||
// Search executes the request against local engines (in parallel) and
|
// Search executes the request against local engines (in parallel) and
|
||||||
// optionally the upstream instance for unported engines.
|
// optionally upstream SearXNG for unported engines.
|
||||||
|
//
|
||||||
|
// Individual engine failures are reported as unresponsive_engines rather
|
||||||
|
// than aborting the entire search.
|
||||||
|
//
|
||||||
|
// If a Valkey cache is configured and contains a cached response for this
|
||||||
|
// request, the cached result is returned without hitting any engines.
|
||||||
func (s *Service) Search(ctx context.Context, req SearchRequest) (SearchResponse, error) {
|
func (s *Service) Search(ctx context.Context, req SearchRequest) (SearchResponse, error) {
|
||||||
queryHash := cache.QueryHash(
|
// Check cache first.
|
||||||
req.Query,
|
if s.cache != nil {
|
||||||
int(req.Pageno),
|
cacheKey := cache.Key(req)
|
||||||
int(req.Safesearch),
|
if cached, hit := s.cache.Get(ctx, cacheKey); hit {
|
||||||
req.Language,
|
return cached, nil
|
||||||
derefString(req.TimeRange),
|
}
|
||||||
)
|
}
|
||||||
|
|
||||||
|
merged, err := s.executeSearch(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
return SearchResponse{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store in cache.
|
||||||
|
if s.cache != nil {
|
||||||
|
cacheKey := cache.Key(req)
|
||||||
|
s.cache.Set(ctx, cacheKey, merged)
|
||||||
|
}
|
||||||
|
|
||||||
|
return merged, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// executeSearch runs the actual engine queries and merges results.
|
||||||
|
func (s *Service) executeSearch(ctx context.Context, req SearchRequest) (SearchResponse, error) {
|
||||||
localEngineNames, upstreamEngineNames, _ := s.planner.Plan(req)
|
localEngineNames, upstreamEngineNames, _ := s.planner.Plan(req)
|
||||||
|
|
||||||
// Phase 1: Parallel cache lookups — classify each engine as fresh/stale/miss
|
// Run all local engines concurrently.
|
||||||
type cacheResult struct {
|
type engineResult struct {
|
||||||
engine string
|
name string
|
||||||
cached cache.CachedEngineResponse
|
resp contracts.SearchResponse
|
||||||
hit bool
|
err error
|
||||||
fresh *contracts.SearchResponse // nil if no fresh response
|
|
||||||
fetchErr error
|
|
||||||
unmarshalErr bool // true if hit but unmarshal failed (treat as miss)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cacheResults := make([]cacheResult, len(localEngineNames))
|
localResults := make([]engineResult, 0, len(localEngineNames))
|
||||||
|
|
||||||
var lookupWg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
for i, name := range localEngineNames {
|
var mu sync.Mutex
|
||||||
lookupWg.Add(1)
|
|
||||||
go func(i int, name string) {
|
|
||||||
defer lookupWg.Done()
|
|
||||||
|
|
||||||
result := cacheResult{engine: name}
|
for _, name := range localEngineNames {
|
||||||
|
|
||||||
if s.engineCache != nil {
|
|
||||||
cached, ok := s.engineCache.Get(ctx, name, queryHash)
|
|
||||||
if ok {
|
|
||||||
result.hit = true
|
|
||||||
result.cached = cached
|
|
||||||
if !s.engineCache.IsStale(cached, name) {
|
|
||||||
// Fresh cache hit — deserialize and use directly
|
|
||||||
var resp contracts.SearchResponse
|
|
||||||
if err := json.Unmarshal(cached.Response, &resp); err == nil {
|
|
||||||
result.fresh = &resp
|
|
||||||
} else {
|
|
||||||
// Unmarshal failed — treat as cache miss (will fetch fresh synchronously)
|
|
||||||
result.unmarshalErr = true
|
|
||||||
result.hit = false // treat as miss
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// If stale: result.fresh stays zero, result.cached has stale data
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheResults[i] = result
|
|
||||||
}(i, name)
|
|
||||||
}
|
|
||||||
lookupWg.Wait()
|
|
||||||
|
|
||||||
// Phase 2: Fetch fresh for misses and stale entries
|
|
||||||
var fetchWg sync.WaitGroup
|
|
||||||
for i, name := range localEngineNames {
|
|
||||||
cr := cacheResults[i]
|
|
||||||
|
|
||||||
// Fresh hit — nothing to do in phase 2
|
|
||||||
if cr.hit && cr.fresh != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stale hit — return stale immediately, refresh in background
|
|
||||||
if cr.hit && len(cr.cached.Response) > 0 && s.engineCache != nil && s.engineCache.IsStale(cr.cached, name) {
|
|
||||||
fetchWg.Add(1)
|
|
||||||
go func(name string) {
|
|
||||||
defer fetchWg.Done()
|
|
||||||
eng, ok := s.localEngines[name]
|
eng, ok := s.localEngines[name]
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
mu.Lock()
|
||||||
}
|
localResults = append(localResults, engineResult{
|
||||||
freshResp, err := eng.Search(ctx, req)
|
name: name,
|
||||||
if err != nil {
|
resp: unresponsiveResponse(req.Query, name, "engine_not_registered"),
|
||||||
s.engineCache.Logger().Debug("background refresh failed", "engine", name, "error", err)
|
})
|
||||||
return
|
mu.Unlock()
|
||||||
}
|
|
||||||
s.engineCache.Set(ctx, name, queryHash, freshResp)
|
|
||||||
}(name)
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cache miss — fetch fresh synchronously
|
wg.Add(1)
|
||||||
if !cr.hit {
|
go func(name string, eng engines.Engine) {
|
||||||
fetchWg.Add(1)
|
defer wg.Done()
|
||||||
go func(i int, name string) {
|
|
||||||
defer fetchWg.Done()
|
|
||||||
|
|
||||||
eng, ok := s.localEngines[name]
|
r, err := eng.Search(ctx, req)
|
||||||
if !ok {
|
|
||||||
cacheResults[i] = cacheResult{
|
mu.Lock()
|
||||||
engine: name,
|
defer mu.Unlock()
|
||||||
fetchErr: fmt.Errorf("engine not registered: %s", name),
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
freshResp, err := eng.Search(ctx, req)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cacheResults[i] = cacheResult{
|
localResults = append(localResults, engineResult{
|
||||||
engine: name,
|
name: name,
|
||||||
fetchErr: err,
|
resp: unresponsiveResponse(req.Query, name, err.Error()),
|
||||||
}
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
localResults = append(localResults, engineResult{name: name, resp: r})
|
||||||
// Cache the fresh response
|
}(name, eng)
|
||||||
if s.engineCache != nil {
|
|
||||||
s.engineCache.Set(ctx, name, queryHash, freshResp)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cacheResults[i] = cacheResult{
|
wg.Wait()
|
||||||
engine: name,
|
|
||||||
fresh: &freshResp,
|
|
||||||
hit: false,
|
|
||||||
}
|
|
||||||
}(i, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fetchWg.Wait()
|
|
||||||
|
|
||||||
// Phase 3: Collect responses for merge
|
// Collect successful responses and determine upstream fallbacks.
|
||||||
responses := make([]contracts.SearchResponse, 0, len(cacheResults))
|
responses := make([]contracts.SearchResponse, 0, len(localResults)+1)
|
||||||
|
upstreamSet := map[string]bool{}
|
||||||
|
for _, e := range upstreamEngineNames {
|
||||||
|
upstreamSet[e] = true
|
||||||
|
}
|
||||||
|
|
||||||
for _, cr := range cacheResults {
|
for _, lr := range localResults {
|
||||||
if cr.fetchErr != nil {
|
responses = append(responses, lr.resp)
|
||||||
responses = append(responses, unresponsiveResponse(req.Query, cr.engine, cr.fetchErr.Error()))
|
|
||||||
continue
|
// If a local engine returned nothing (e.g. qwant anti-bot), fall back
|
||||||
}
|
// to upstream if available.
|
||||||
// Use fresh data if available (fresh hit or freshly fetched), otherwise use stale cached
|
if shouldFallbackToUpstream(lr.name, lr.resp) && !upstreamSet[lr.name] {
|
||||||
if cr.fresh != nil {
|
upstreamEngineNames = append(upstreamEngineNames, lr.name)
|
||||||
responses = append(responses, *cr.fresh)
|
upstreamSet[lr.name] = true
|
||||||
} else if cr.hit && len(cr.cached.Response) > 0 {
|
|
||||||
var resp contracts.SearchResponse
|
|
||||||
if err := json.Unmarshal(cr.cached.Response, &resp); err == nil {
|
|
||||||
responses = append(responses, resp)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Upstream proxy for unported (or fallback) engines.
|
// Upstream proxy for unported (or fallback) engines.
|
||||||
// ... rest of the existing code is UNCHANGED ...
|
|
||||||
if s.upstreamClient != nil && len(upstreamEngineNames) > 0 {
|
if s.upstreamClient != nil && len(upstreamEngineNames) > 0 {
|
||||||
r, err := s.upstreamClient.SearchJSON(ctx, req, upstreamEngineNames)
|
r, err := s.upstreamClient.SearchJSON(ctx, req, upstreamEngineNames)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
// Upstream failure is treated as a single unresponsive engine entry.
|
||||||
responses = append(responses, contracts.SearchResponse{
|
responses = append(responses, contracts.SearchResponse{
|
||||||
Query: req.Query,
|
Query: req.Query,
|
||||||
UnresponsiveEngines: [][2]string{{"upstream", err.Error()}},
|
UnresponsiveEngines: [][2]string{{"upstream", err.Error()}},
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,8 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/engines"
|
"github.com/metamorphosis-dev/kafka/internal/engines"
|
||||||
)
|
)
|
||||||
|
|
||||||
// mockEngine is a test engine that returns a predefined response or error.
|
// mockEngine is a test engine that returns a predefined response or error.
|
||||||
|
|
|
||||||
|
|
@ -1,29 +1,13 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package search
|
package search
|
||||||
|
|
||||||
import "github.com/metamorphosis-dev/samsa/internal/contracts"
|
import "github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
|
|
||||||
// Re-export the JSON contract types so the rest of the code can stay in the
|
// Re-export the JSON contract types so the rest of the code can stay in the
|
||||||
// `internal/search` namespace without creating an import cycle.
|
// `internal/search` namespace without creating an import cycle.
|
||||||
type OutputFormat = contracts.OutputFormat
|
type OutputFormat = contracts.OutputFormat
|
||||||
|
|
||||||
const (
|
const (
|
||||||
FormatHTML = contracts.FormatHTML // accepted for compatibility
|
FormatHTML = contracts.FormatHTML // accepted for compatibility (not yet implemented)
|
||||||
FormatJSON = contracts.FormatJSON
|
FormatJSON = contracts.FormatJSON
|
||||||
FormatCSV = contracts.FormatCSV
|
FormatCSV = contracts.FormatCSV
|
||||||
FormatRSS = contracts.FormatRSS
|
FormatRSS = contracts.FormatRSS
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,3 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package upstream
|
package upstream
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -27,8 +11,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/httpclient"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Client struct {
|
type Client struct {
|
||||||
|
|
@ -45,9 +28,6 @@ func NewClient(baseURL string, timeout time.Duration) (*Client, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("invalid upstream base URL: %w", err)
|
return nil, fmt.Errorf("invalid upstream base URL: %w", err)
|
||||||
}
|
}
|
||||||
if u.Scheme != "http" && u.Scheme != "https" {
|
|
||||||
return nil, fmt.Errorf("upstream URL must use http or https, got %q", u.Scheme)
|
|
||||||
}
|
|
||||||
// Normalize: trim trailing slash to make URL concatenation predictable.
|
// Normalize: trim trailing slash to make URL concatenation predictable.
|
||||||
base := strings.TrimRight(u.String(), "/")
|
base := strings.TrimRight(u.String(), "/")
|
||||||
|
|
||||||
|
|
@ -57,7 +37,9 @@ func NewClient(baseURL string, timeout time.Duration) (*Client, error) {
|
||||||
|
|
||||||
return &Client{
|
return &Client{
|
||||||
baseURL: base,
|
baseURL: base,
|
||||||
http: httpclient.NewClient(timeout),
|
http: &http.Client{
|
||||||
|
Timeout: timeout,
|
||||||
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -86,7 +68,7 @@ func (c *Client) SearchJSON(ctx context.Context, req contracts.SearchRequest, en
|
||||||
|
|
||||||
for engineName, kv := range req.EngineData {
|
for engineName, kv := range req.EngineData {
|
||||||
for key, value := range kv {
|
for key, value := range kv {
|
||||||
// Mirror the naming convention: `engine_data-<engine>-<key>=<value>`
|
// Mirror SearXNG's naming: `engine_data-<engine>-<key>=<value>`
|
||||||
form.Set(fmt.Sprintf("engine_data-%s-%s", engineName, key), value)
|
form.Set(fmt.Sprintf("engine_data-%s-%s", engineName, key), value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -110,7 +92,7 @@ func (c *Client) SearchJSON(ctx context.Context, req contracts.SearchRequest, en
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return contracts.SearchResponse{}, fmt.Errorf("upstream search failed with status %d", resp.StatusCode)
|
return contracts.SearchResponse{}, fmt.Errorf("upstream search failed: status=%d body=%q", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Decode upstream JSON into our contract types.
|
// Decode upstream JSON into our contract types.
|
||||||
|
|
|
||||||
|
|
@ -1,127 +0,0 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License,// or (at your option) any later version.
|
|
||||||
|
|
||||||
package util
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// SafeURLScheme validates that a URL is well-formed and uses an acceptable scheme.
|
|
||||||
// Returns the parsed URL on success, or an error.
|
|
||||||
func SafeURLScheme(raw string) (*url.URL, error) {
|
|
||||||
u, err := url.Parse(raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if u.Scheme != "http" && u.Scheme != "https" {
|
|
||||||
return nil, fmt.Errorf("URL must use http or https, got %q", u.Scheme)
|
|
||||||
}
|
|
||||||
return u, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsPrivateIP returns true if the IP address is in a private, loopback,
|
|
||||||
// link-local, or otherwise non-routable range.
|
|
||||||
func IsPrivateIP(host string) bool {
|
|
||||||
// Strip port if present.
|
|
||||||
h, _, err := net.SplitHostPort(host)
|
|
||||||
if err != nil {
|
|
||||||
h = host
|
|
||||||
}
|
|
||||||
|
|
||||||
// Resolve hostname to IPs.
|
|
||||||
ips, err := net.LookupIP(h)
|
|
||||||
if err != nil || len(ips) == 0 {
|
|
||||||
// If we can't resolve, reject to be safe.
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, ip := range ips {
|
|
||||||
if isPrivateIPAddr(ip) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func isPrivateIPAddr(ip net.IP) bool {
|
|
||||||
privateRanges := []struct {
|
|
||||||
network *net.IPNet
|
|
||||||
}{
|
|
||||||
// Loopback
|
|
||||||
{mustParseCIDR("127.0.0.0/8")},
|
|
||||||
{mustParseCIDR("::1/128")},
|
|
||||||
// RFC 1918
|
|
||||||
{mustParseCIDR("10.0.0.0/8")},
|
|
||||||
{mustParseCIDR("172.16.0.0/12")},
|
|
||||||
{mustParseCIDR("192.168.0.0/16")},
|
|
||||||
// RFC 6598 (Carrier-grade NAT)
|
|
||||||
{mustParseCIDR("100.64.0.0/10")},
|
|
||||||
// Link-local
|
|
||||||
{mustParseCIDR("169.254.0.0/16")},
|
|
||||||
{mustParseCIDR("fe80::/10")},
|
|
||||||
// IPv6 unique local
|
|
||||||
{mustParseCIDR("fc00::/7")},
|
|
||||||
// IPv4-mapped IPv6 loopback
|
|
||||||
{mustParseCIDR("::ffff:127.0.0.0/104")},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, r := range privateRanges {
|
|
||||||
if r.network.Contains(ip) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustParseCIDR(s string) *net.IPNet {
|
|
||||||
_, network, err := net.ParseCIDR(s)
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("validate: invalid CIDR %q: %v", s, err))
|
|
||||||
}
|
|
||||||
return network
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidatePublicURL checks that a URL is well-formed, uses http or https,
|
|
||||||
// and does not point to a private/reserved IP range.
|
|
||||||
func ValidatePublicURL(raw string) error {
|
|
||||||
u, err := url.Parse(raw)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("invalid URL: %w", err)
|
|
||||||
}
|
|
||||||
if u.Scheme != "http" && u.Scheme != "https" {
|
|
||||||
return fmt.Errorf("URL must use http or https, got %q", u.Scheme)
|
|
||||||
}
|
|
||||||
if u.Host == "" {
|
|
||||||
return fmt.Errorf("URL must have a host")
|
|
||||||
}
|
|
||||||
if IsPrivateIP(u.Host) {
|
|
||||||
return fmt.Errorf("URL points to a private or reserved address: %s", u.Host)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// SanitizeResultURL ensures a URL is safe for rendering in an href attribute.
|
|
||||||
// It rejects javascript:, data:, vbscript: and other dangerous schemes.
|
|
||||||
func SanitizeResultURL(raw string) string {
|
|
||||||
if raw == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
u, err := url.Parse(raw)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
switch strings.ToLower(u.Scheme) {
|
|
||||||
case "http", "https", "":
|
|
||||||
return raw
|
|
||||||
default:
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
516
internal/views/static/css/kafka.css
Normal file
516
internal/views/static/css/kafka.css
Normal file
|
|
@ -0,0 +1,516 @@
|
||||||
|
/* kafka — clean, minimal search engine CSS */
|
||||||
|
/* Inspired by SearXNG's simple theme class conventions */
|
||||||
|
|
||||||
|
:root {
|
||||||
|
--color-base: #f5f5f5;
|
||||||
|
--color-base-font: #444;
|
||||||
|
--color-base-background: #fff;
|
||||||
|
--color-header-background: #f7f7f7;
|
||||||
|
--color-header-border: #ddd;
|
||||||
|
--color-search-border: #bbb;
|
||||||
|
--color-search-focus: #3498db;
|
||||||
|
--color-result-url: #1a0dab;
|
||||||
|
--color-result-url-visited: #609;
|
||||||
|
--color-result-content: #545454;
|
||||||
|
--color-result-title: #1a0dab;
|
||||||
|
--color-result-title-visited: #609;
|
||||||
|
--color-result-engine: #666;
|
||||||
|
--color-result-border: #eee;
|
||||||
|
--color-link: #3498db;
|
||||||
|
--color-link-visited: #609;
|
||||||
|
--color-sidebar-background: #f7f7f7;
|
||||||
|
--color-sidebar-border: #ddd;
|
||||||
|
--color-infobox-background: #f9f9f9;
|
||||||
|
--color-infobox-border: #ddd;
|
||||||
|
--color-pagination-current: #3498db;
|
||||||
|
--color-pagination-border: #ddd;
|
||||||
|
--color-error: #c0392b;
|
||||||
|
--color-error-background: #fdecea;
|
||||||
|
--color-suggestion: #666;
|
||||||
|
--color-footer: #888;
|
||||||
|
--color-btn-background: #fff;
|
||||||
|
--color-btn-border: #ddd;
|
||||||
|
--color-btn-hover: #eee;
|
||||||
|
--radius: 4px;
|
||||||
|
--max-width: 800px;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (prefers-color-scheme: dark) {
|
||||||
|
:root {
|
||||||
|
--color-base: #222;
|
||||||
|
--color-base-font: #dcdcdc;
|
||||||
|
--color-base-background: #2b2b2b;
|
||||||
|
--color-header-background: #333;
|
||||||
|
--color-header-border: #444;
|
||||||
|
--color-search-border: #555;
|
||||||
|
--color-search-focus: #5dade2;
|
||||||
|
--color-result-url: #8ab4f8;
|
||||||
|
--color-result-url-visited: #b39ddb;
|
||||||
|
--color-result-content: #b0b0b0;
|
||||||
|
--color-result-title: #8ab4f8;
|
||||||
|
--color-result-title-visited: #b39ddb;
|
||||||
|
--color-result-engine: #999;
|
||||||
|
--color-result-border: #3a3a3a;
|
||||||
|
--color-link: #5dade2;
|
||||||
|
--color-link-visited: #b39ddb;
|
||||||
|
--color-sidebar-background: #333;
|
||||||
|
--color-sidebar-border: #444;
|
||||||
|
--color-infobox-background: #333;
|
||||||
|
--color-infobox-border: #444;
|
||||||
|
--color-pagination-current: #5dade2;
|
||||||
|
--color-pagination-border: #444;
|
||||||
|
--color-error: #e74c3c;
|
||||||
|
--color-error-background: #3b1a1a;
|
||||||
|
--color-suggestion: #999;
|
||||||
|
--color-footer: #666;
|
||||||
|
--color-btn-background: #333;
|
||||||
|
--color-btn-border: #555;
|
||||||
|
--color-btn-hover: #444;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
*, *::before, *::after {
|
||||||
|
box-sizing: border-box;
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
html {
|
||||||
|
font-size: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
|
||||||
|
color: var(--color-base-font);
|
||||||
|
background: var(--color-base);
|
||||||
|
line-height: 1.6;
|
||||||
|
min-height: 100vh;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
|
||||||
|
main {
|
||||||
|
flex: 1;
|
||||||
|
max-width: var(--max-width);
|
||||||
|
width: 100%;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
footer {
|
||||||
|
text-align: center;
|
||||||
|
padding: 1.5rem;
|
||||||
|
color: var(--color-footer);
|
||||||
|
font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
footer a {
|
||||||
|
color: var(--color-link);
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
footer a:hover {
|
||||||
|
text-decoration: underline;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Index / Homepage */
|
||||||
|
.index {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
min-height: 60vh;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.index .title h1 {
|
||||||
|
font-size: 2rem;
|
||||||
|
font-weight: 300;
|
||||||
|
margin-bottom: 2rem;
|
||||||
|
letter-spacing: 0.05em;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Search form */
|
||||||
|
#search {
|
||||||
|
width: 100%;
|
||||||
|
max-width: 600px;
|
||||||
|
margin-bottom: 2rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
#search form {
|
||||||
|
display: flex;
|
||||||
|
gap: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
#search input[type="text"],
|
||||||
|
#q {
|
||||||
|
flex: 1;
|
||||||
|
padding: 0.7rem 1rem;
|
||||||
|
font-size: 1rem;
|
||||||
|
border: 1px solid var(--color-search-border);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
background: var(--color-base-background);
|
||||||
|
color: var(--color-base-font);
|
||||||
|
outline: none;
|
||||||
|
transition: border-color 0.2s;
|
||||||
|
}
|
||||||
|
|
||||||
|
#search input[type="text"]:focus,
|
||||||
|
#q:focus {
|
||||||
|
border-color: var(--color-search-focus);
|
||||||
|
box-shadow: 0 0 0 2px rgba(52, 152, 219, 0.15);
|
||||||
|
}
|
||||||
|
|
||||||
|
#search button[type="submit"] {
|
||||||
|
padding: 0.7rem 1.2rem;
|
||||||
|
font-size: 1rem;
|
||||||
|
border: 1px solid var(--color-search-border);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
background: var(--color-btn-background);
|
||||||
|
color: var(--color-base-font);
|
||||||
|
cursor: pointer;
|
||||||
|
transition: background 0.2s;
|
||||||
|
}
|
||||||
|
|
||||||
|
#search button[type="submit"]:hover {
|
||||||
|
background: var(--color-btn-hover);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Results page search bar (compact) */
|
||||||
|
.search_on_results #search {
|
||||||
|
max-width: 100%;
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Results area */
|
||||||
|
#results {
|
||||||
|
display: flex;
|
||||||
|
gap: 1.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
#sidebar {
|
||||||
|
flex: 0 0 200px;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
#sidebar p {
|
||||||
|
margin-bottom: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
#urls {
|
||||||
|
flex: 1;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Result count */
|
||||||
|
#result_count {
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Individual result */
|
||||||
|
.result {
|
||||||
|
padding: 0.8rem 0;
|
||||||
|
border-bottom: 1px solid var(--color-result-border);
|
||||||
|
word-wrap: break-word;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result:last-child {
|
||||||
|
border-bottom: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result_header {
|
||||||
|
margin-bottom: 0.2rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result_header a {
|
||||||
|
font-size: 1.1rem;
|
||||||
|
font-weight: 400;
|
||||||
|
color: var(--color-result-title);
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result_header a:visited {
|
||||||
|
color: var(--color-result-title-visited);
|
||||||
|
}
|
||||||
|
|
||||||
|
.result_header a:hover {
|
||||||
|
text-decoration: underline;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result_url {
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: var(--color-result-url);
|
||||||
|
margin-bottom: 0.2rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result_url a {
|
||||||
|
color: var(--color-result-url);
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result_url a:visited {
|
||||||
|
color: var(--color-result-url-visited);
|
||||||
|
}
|
||||||
|
|
||||||
|
.result_content {
|
||||||
|
font-size: 0.9rem;
|
||||||
|
color: var(--color-result-content);
|
||||||
|
max-width: 600px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result_content p {
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result_engine {
|
||||||
|
font-size: 0.75rem;
|
||||||
|
color: var(--color-result-engine);
|
||||||
|
margin-top: 0.3rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.engine {
|
||||||
|
display: inline-block;
|
||||||
|
padding: 0.1rem 0.4rem;
|
||||||
|
background: var(--color-sidebar-background);
|
||||||
|
border: 1px solid var(--color-sidebar-border);
|
||||||
|
border-radius: 2px;
|
||||||
|
font-size: 0.7rem;
|
||||||
|
color: var(--color-result-engine);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* No results */
|
||||||
|
.no_results {
|
||||||
|
text-align: center;
|
||||||
|
padding: 3rem 1rem;
|
||||||
|
color: var(--color-suggestion);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Suggestions */
|
||||||
|
#suggestions {
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.suggestion {
|
||||||
|
display: inline-block;
|
||||||
|
margin: 0.2rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.suggestion a {
|
||||||
|
display: inline-block;
|
||||||
|
padding: 0.3rem 0.6rem;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
border: 1px solid var(--color-pagination-border);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
color: var(--color-link);
|
||||||
|
text-decoration: none;
|
||||||
|
background: var(--color-btn-background);
|
||||||
|
transition: background 0.2s;
|
||||||
|
}
|
||||||
|
|
||||||
|
.suggestion a:hover {
|
||||||
|
background: var(--color-btn-hover);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Infoboxes */
|
||||||
|
#infoboxes {
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.infobox {
|
||||||
|
background: var(--color-infobox-background);
|
||||||
|
border: 1px solid var(--color-infobox-border);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
padding: 0.8rem;
|
||||||
|
margin-bottom: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.infobox .title {
|
||||||
|
font-weight: 600;
|
||||||
|
margin-bottom: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Errors */
|
||||||
|
.dialog-error {
|
||||||
|
background: var(--color-error-background);
|
||||||
|
color: var(--color-error);
|
||||||
|
border: 1px solid var(--color-error);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
padding: 0.8rem 1rem;
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Unresponsive engines */
|
||||||
|
.unresponsive_engines {
|
||||||
|
font-size: 0.8rem;
|
||||||
|
color: var(--color-suggestion);
|
||||||
|
margin-top: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.unresponsive_engines li {
|
||||||
|
margin: 0.1rem 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Corrections */
|
||||||
|
.correction {
|
||||||
|
font-size: 0.9rem;
|
||||||
|
margin-bottom: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Pagination */
|
||||||
|
#pagination {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 0.3rem;
|
||||||
|
padding: 1.5rem 0;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
#pagination button,
|
||||||
|
#pagination .page_number,
|
||||||
|
#pagination .page_number_current {
|
||||||
|
padding: 0.4rem 0.8rem;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
border: 1px solid var(--color-pagination-border);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
background: var(--color-btn-background);
|
||||||
|
color: var(--color-base-font);
|
||||||
|
cursor: pointer;
|
||||||
|
text-decoration: none;
|
||||||
|
transition: background 0.2s;
|
||||||
|
}
|
||||||
|
|
||||||
|
#pagination button:hover,
|
||||||
|
#pagination .page_number:hover {
|
||||||
|
background: var(--color-btn-hover);
|
||||||
|
}
|
||||||
|
|
||||||
|
#pagination .page_number_current {
|
||||||
|
background: var(--color-pagination-current);
|
||||||
|
color: #fff;
|
||||||
|
border-color: var(--color-pagination-current);
|
||||||
|
cursor: default;
|
||||||
|
}
|
||||||
|
|
||||||
|
.previous_page, .next_page {
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Back to top */
|
||||||
|
#backToTop {
|
||||||
|
text-align: center;
|
||||||
|
margin: 1rem 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
#backToTop a {
|
||||||
|
color: var(--color-link);
|
||||||
|
text-decoration: none;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* HTMX loading indicator */
|
||||||
|
.htmx-indicator {
|
||||||
|
display: none;
|
||||||
|
text-align: center;
|
||||||
|
padding: 2rem;
|
||||||
|
color: var(--color-suggestion);
|
||||||
|
}
|
||||||
|
|
||||||
|
.htmx-request .htmx-indicator,
|
||||||
|
.htmx-request.htmx-indicator {
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Autocomplete dropdown */
|
||||||
|
#search {
|
||||||
|
position: relative;
|
||||||
|
}
|
||||||
|
|
||||||
|
#autocomplete-dropdown {
|
||||||
|
position: absolute;
|
||||||
|
top: 100%;
|
||||||
|
left: 0;
|
||||||
|
right: 0;
|
||||||
|
background: var(--color-base-background);
|
||||||
|
border: 1px solid var(--color-search-border);
|
||||||
|
border-top: none;
|
||||||
|
border-radius: 0 0 var(--radius) var(--radius);
|
||||||
|
box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
|
||||||
|
z-index: 100;
|
||||||
|
max-height: 320px;
|
||||||
|
overflow-y: auto;
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
#autocomplete-dropdown.open {
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
|
||||||
|
.autocomplete-suggestion {
|
||||||
|
padding: 0.6rem 1rem;
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 0.95rem;
|
||||||
|
color: var(--color-base-font);
|
||||||
|
border-bottom: 1px solid var(--color-result-border);
|
||||||
|
transition: background 0.15s;
|
||||||
|
}
|
||||||
|
|
||||||
|
.autocomplete-suggestion:last-child {
|
||||||
|
border-bottom: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.autocomplete-suggestion:hover,
|
||||||
|
.autocomplete-suggestion.active {
|
||||||
|
background: var(--color-header-background);
|
||||||
|
}
|
||||||
|
|
||||||
|
.autocomplete-suggestion mark {
|
||||||
|
background: none;
|
||||||
|
color: var(--color-link);
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
.autocomplete-footer {
|
||||||
|
padding: 0.4rem 1rem;
|
||||||
|
font-size: 0.75rem;
|
||||||
|
color: var(--color-suggestion);
|
||||||
|
border-top: 1px solid var(--color-result-border);
|
||||||
|
background: var(--color-header-background);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Responsive */
|
||||||
|
@media (max-width: 768px) {
|
||||||
|
#results {
|
||||||
|
flex-direction: column-reverse;
|
||||||
|
}
|
||||||
|
|
||||||
|
#sidebar {
|
||||||
|
flex: none;
|
||||||
|
border-top: 1px solid var(--color-sidebar-border);
|
||||||
|
padding-top: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.index .title h1 {
|
||||||
|
font-size: 1.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
main {
|
||||||
|
padding: 0.5rem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Print */
|
||||||
|
@media print {
|
||||||
|
footer, #pagination, #search button, #backToTop, .htmx-indicator {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
background: #fff;
|
||||||
|
color: #000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result a {
|
||||||
|
color: #000;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,971 +0,0 @@
|
||||||
/* ============================================================
|
|
||||||
kafka — clean, minimal search UI
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
:root {
|
|
||||||
--bg: #ffffff;
|
|
||||||
--bg-secondary: #f8f9fa;
|
|
||||||
--bg-tertiary: #f1f3f5;
|
|
||||||
--border: #e9ecef;
|
|
||||||
--border-focus: #cad1d8;
|
|
||||||
--text-primary: #1a1a1a;
|
|
||||||
--text-secondary: #5c6370;
|
|
||||||
--text-muted: #8b929e;
|
|
||||||
--accent: #0d9488;
|
|
||||||
--accent-hover: #0f766e;
|
|
||||||
--accent-soft: #f0fdfa;
|
|
||||||
--shadow-sm: 0 1px 3px rgba(0,0,0,0.06);
|
|
||||||
--shadow-md: 0 4px 12px rgba(0,0,0,0.08);
|
|
||||||
--radius-sm: 8px;
|
|
||||||
--radius-md: 12px;
|
|
||||||
--radius-lg: 16px;
|
|
||||||
--radius-full: 9999px;
|
|
||||||
--font: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
||||||
--font-mono: "IBM Plex Mono", ui-monospace, monospace;
|
|
||||||
}
|
|
||||||
|
|
||||||
:root[data-theme="dark"],
|
|
||||||
html[data-theme="dark"],
|
|
||||||
@media (prefers-color-scheme: dark) {
|
|
||||||
--bg: #0f0f0f !important;
|
|
||||||
--bg-secondary: #1a1a1a !important;
|
|
||||||
--bg-tertiary: #242424 !important;
|
|
||||||
--border: #2e2e2e !important;
|
|
||||||
--border-focus: #404040 !important;
|
|
||||||
--text-primary: #e8eaed !important;
|
|
||||||
--text-secondary: #9aa0a6 !important;
|
|
||||||
--text-muted: #6b7280 !important;
|
|
||||||
--accent: #14b8a6 !important;
|
|
||||||
--accent-hover: #2dd4bf !important;
|
|
||||||
--accent-soft: #134e4a !important;
|
|
||||||
--shadow-sm: 0 1px 3px rgba(0,0,0,0.3) !important;
|
|
||||||
--shadow-md: 0 4px 12px rgba(0,0,0,0.4) !important;
|
|
||||||
}
|
|
||||||
|
|
||||||
*, *::before, *::after {
|
|
||||||
box-sizing: border-box;
|
|
||||||
margin: 0;
|
|
||||||
padding: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
html {
|
|
||||||
font-size: 16px;
|
|
||||||
scroll-behavior: smooth;
|
|
||||||
}
|
|
||||||
|
|
||||||
body {
|
|
||||||
font-family: var(--font);
|
|
||||||
background: var(--bg);
|
|
||||||
color: var(--text-primary);
|
|
||||||
line-height: 1.5;
|
|
||||||
min-height: 100vh;
|
|
||||||
-webkit-font-smoothing: antialiased;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Header
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.site-header {
|
|
||||||
position: sticky;
|
|
||||||
top: 0;
|
|
||||||
z-index: 100;
|
|
||||||
height: 56px;
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: space-between;
|
|
||||||
padding: 0 1.5rem;
|
|
||||||
background: var(--bg);
|
|
||||||
border-bottom: 1px solid var(--border);
|
|
||||||
}
|
|
||||||
|
|
||||||
.site-logo {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 0.5rem;
|
|
||||||
text-decoration: none;
|
|
||||||
color: var(--text-primary);
|
|
||||||
}
|
|
||||||
|
|
||||||
.site-logo-mark {
|
|
||||||
width: 24px;
|
|
||||||
height: 24px;
|
|
||||||
color: var(--accent);
|
|
||||||
}
|
|
||||||
|
|
||||||
.site-name {
|
|
||||||
font-family: var(--font-mono);
|
|
||||||
font-size: 1.1rem;
|
|
||||||
font-weight: 700;
|
|
||||||
letter-spacing: -0.02em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.settings-link {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: center;
|
|
||||||
width: 36px;
|
|
||||||
height: 36px;
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
color: var(--text-secondary);
|
|
||||||
transition: background 0.15s, color 0.15s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.settings-link:hover {
|
|
||||||
background: var(--bg-secondary);
|
|
||||||
color: var(--text-primary);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Main Layout
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
main {
|
|
||||||
max-width: 800px;
|
|
||||||
margin: 0 auto;
|
|
||||||
padding: 0 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Homepage
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.page-home {
|
|
||||||
min-height: calc(100vh - 56px);
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: center;
|
|
||||||
padding: 4rem 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.home-container {
|
|
||||||
width: 100%;
|
|
||||||
max-width: 640px;
|
|
||||||
text-align: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.home-logo {
|
|
||||||
display: inline-flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 0.75rem;
|
|
||||||
text-decoration: none;
|
|
||||||
color: var(--text-primary);
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.home-logo svg {
|
|
||||||
width: 48px;
|
|
||||||
height: 48px;
|
|
||||||
color: var(--accent);
|
|
||||||
}
|
|
||||||
|
|
||||||
.home-logo-text {
|
|
||||||
font-family: var(--font-mono);
|
|
||||||
font-size: 2rem;
|
|
||||||
font-weight: 700;
|
|
||||||
letter-spacing: -0.02em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.home-tagline {
|
|
||||||
color: var(--text-muted);
|
|
||||||
margin-bottom: 2rem;
|
|
||||||
font-size: 0.95rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Search Form
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.search-form {
|
|
||||||
width: 100%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-box {
|
|
||||||
position: relative;
|
|
||||||
width: 100%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-box input[type="text"] {
|
|
||||||
width: 100%;
|
|
||||||
padding: 1rem 3.5rem 1rem 1.25rem;
|
|
||||||
font-size: 1.05rem;
|
|
||||||
font-family: inherit;
|
|
||||||
border: 2px solid var(--border);
|
|
||||||
border-radius: var(--radius-full);
|
|
||||||
background: var(--bg);
|
|
||||||
color: var(--text-primary);
|
|
||||||
outline: none;
|
|
||||||
transition: border-color 0.2s, box-shadow 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-box input[type="text"]:focus {
|
|
||||||
border-color: var(--accent);
|
|
||||||
box-shadow: var(--shadow-md), 0 0 0 3px rgba(13, 148, 136, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-box input[type="text"]::placeholder {
|
|
||||||
color: var(--text-muted);
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-btn {
|
|
||||||
position: absolute;
|
|
||||||
right: 6px;
|
|
||||||
top: 50%;
|
|
||||||
transform: translateY(-50%);
|
|
||||||
width: 42px;
|
|
||||||
height: 42px;
|
|
||||||
border: none;
|
|
||||||
border-radius: var(--radius-full);
|
|
||||||
background: var(--accent);
|
|
||||||
color: #fff;
|
|
||||||
cursor: pointer;
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: center;
|
|
||||||
transition: background 0.15s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-btn:hover {
|
|
||||||
background: var(--accent-hover);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Results Page
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.page-results {
|
|
||||||
padding-top: 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.results-container {
|
|
||||||
max-width: 768px;
|
|
||||||
margin: 0 auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.results-header {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 1rem;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
position: sticky;
|
|
||||||
top: 56px;
|
|
||||||
background: var(--bg);
|
|
||||||
padding: 0.75rem 0;
|
|
||||||
z-index: 50;
|
|
||||||
}
|
|
||||||
|
|
||||||
.results-logo {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 0.4rem;
|
|
||||||
text-decoration: none;
|
|
||||||
color: var(--text-primary);
|
|
||||||
flex-shrink: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.results-logo svg {
|
|
||||||
width: 20px;
|
|
||||||
height: 20px;
|
|
||||||
color: var(--accent);
|
|
||||||
}
|
|
||||||
|
|
||||||
.results-logo span {
|
|
||||||
font-family: var(--font-mono);
|
|
||||||
font-size: 1rem;
|
|
||||||
font-weight: 700;
|
|
||||||
}
|
|
||||||
|
|
||||||
.header-search {
|
|
||||||
flex: 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
.header-search .search-box input {
|
|
||||||
padding: 0.65rem 2.5rem 0.65rem 1rem;
|
|
||||||
font-size: 0.95rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.header-search .search-btn {
|
|
||||||
width: 34px;
|
|
||||||
height: 34px;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Category Tabs
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.category-tabs {
|
|
||||||
display: flex;
|
|
||||||
gap: 0.25rem;
|
|
||||||
margin-bottom: 1.5rem;
|
|
||||||
border-bottom: 1px solid var(--border);
|
|
||||||
padding-bottom: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.category-tab {
|
|
||||||
padding: 0.5rem 0.85rem;
|
|
||||||
font-size: 0.875rem;
|
|
||||||
font-weight: 500;
|
|
||||||
color: var(--text-secondary);
|
|
||||||
text-decoration: none;
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
transition: background 0.15s, color 0.15s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.category-tab:hover {
|
|
||||||
background: var(--bg-secondary);
|
|
||||||
color: var(--text-primary);
|
|
||||||
}
|
|
||||||
|
|
||||||
.category-tab.active {
|
|
||||||
background: var(--accent-soft);
|
|
||||||
color: var(--accent);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Results Meta
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.results-meta {
|
|
||||||
font-size: 0.85rem;
|
|
||||||
color: var(--text-muted);
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Result Cards
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.result {
|
|
||||||
padding: 0.85rem 0;
|
|
||||||
border-bottom: 1px solid var(--border);
|
|
||||||
}
|
|
||||||
|
|
||||||
.result[data-engine] {
|
|
||||||
border-left: 3px solid var(--accent, transparent);
|
|
||||||
}
|
|
||||||
|
|
||||||
.result:last-child {
|
|
||||||
border-bottom: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.result_header {
|
|
||||||
margin-bottom: 0.2rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.result_header a {
|
|
||||||
font-size: 1rem;
|
|
||||||
font-weight: 500;
|
|
||||||
color: #2563eb;
|
|
||||||
text-decoration: none;
|
|
||||||
line-height: 1.4;
|
|
||||||
}
|
|
||||||
|
|
||||||
.result_header a:hover {
|
|
||||||
text-decoration: underline;
|
|
||||||
}
|
|
||||||
|
|
||||||
.result_url {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 0.4rem;
|
|
||||||
font-size: 0.8rem;
|
|
||||||
color: var(--text-muted);
|
|
||||||
margin-bottom: 0.3rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.result-favicon {
|
|
||||||
width: 14px;
|
|
||||||
height: 14px;
|
|
||||||
border-radius: 2px;
|
|
||||||
background: var(--bg-tertiary);
|
|
||||||
flex-shrink: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.result_url a {
|
|
||||||
color: var(--text-muted);
|
|
||||||
text-decoration: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.result_url a:hover {
|
|
||||||
color: var(--accent);
|
|
||||||
}
|
|
||||||
|
|
||||||
.engine-badge {
|
|
||||||
flex-shrink: 0;
|
|
||||||
padding: 0.1rem 0.35rem;
|
|
||||||
background: var(--bg-secondary);
|
|
||||||
border-radius: 4px;
|
|
||||||
font-size: 0.7rem;
|
|
||||||
color: var(--text-muted);
|
|
||||||
margin-left: auto;
|
|
||||||
border-left: 2.5px solid transparent;
|
|
||||||
transition: border-color 0.15s ease;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Engine Accent Colors
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.result[data-engine="google"],
|
|
||||||
.engine-badge[data-engine="google"] { --accent: #4285f4; }
|
|
||||||
.result[data-engine="bing"],
|
|
||||||
.engine-badge[data-engine="bing"] { --accent: #00897b; }
|
|
||||||
.result[data-engine="duckduckgo"],
|
|
||||||
.engine-badge[data-engine="duckduckgo"] { --accent: #de5833; }
|
|
||||||
.result[data-engine="brave"],
|
|
||||||
.engine-badge[data-engine="brave"] { --accent: #fb542b; }
|
|
||||||
.result[data-engine="braveapi"],
|
|
||||||
.engine-badge[data-engine="braveapi"] { --accent: #ff6600; }
|
|
||||||
.result[data-engine="qwant"],
|
|
||||||
.engine-badge[data-engine="qwant"] { --accent: #5c97ff; }
|
|
||||||
.result[data-engine="wikipedia"],
|
|
||||||
.engine-badge[data-engine="wikipedia"] { --accent: #333333; }
|
|
||||||
.result[data-engine="github"],
|
|
||||||
.engine-badge[data-engine="github"] { --accent: #8b5cf6; }
|
|
||||||
.result[data-engine="reddit"],
|
|
||||||
.engine-badge[data-engine="reddit"] { --accent: #ff4500; }
|
|
||||||
.result[data-engine="youtube"],
|
|
||||||
.engine-badge[data-engine="youtube"] { --accent: #ff0000; }
|
|
||||||
.result[data-engine="stackoverflow"],
|
|
||||||
.engine-badge[data-engine="stackoverflow"] { --accent: #f48024; }
|
|
||||||
.result[data-engine="arxiv"],
|
|
||||||
.engine-badge[data-engine="arxiv"] { --accent: #b31b1b; }
|
|
||||||
.result[data-engine="crossref"],
|
|
||||||
.engine-badge[data-engine="crossref"] { --accent: #00354d; }
|
|
||||||
.result[data-engine="bing_images"],
|
|
||||||
.engine-badge[data-engine="bing_images"] { --accent: #00897b; }
|
|
||||||
.result[data-engine="ddg_images"],
|
|
||||||
.engine-badge[data-engine="ddg_images"] { --accent: #de5833; }
|
|
||||||
.result[data-engine="qwant_images"],
|
|
||||||
.engine-badge[data-engine="qwant_images"] { --accent: #5c97ff; }
|
|
||||||
|
|
||||||
.engine-badge[data-engine] {
|
|
||||||
border-left-color: var(--accent, transparent);
|
|
||||||
color: var(--accent, var(--text-muted));
|
|
||||||
font-weight: 500;
|
|
||||||
}
|
|
||||||
|
|
||||||
.result_content {
|
|
||||||
font-size: 0.875rem;
|
|
||||||
color: var(--text-secondary);
|
|
||||||
line-height: 1.55;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
No Results
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.no-results {
|
|
||||||
text-align: center;
|
|
||||||
padding: 4rem 2rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.no-results-icon {
|
|
||||||
font-size: 3rem;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
opacity: 0.3;
|
|
||||||
}
|
|
||||||
|
|
||||||
.no-results h2 {
|
|
||||||
font-size: 1.25rem;
|
|
||||||
font-weight: 500;
|
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.no-results p {
|
|
||||||
color: var(--text-muted);
|
|
||||||
font-size: 0.95rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Pagination
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.pagination {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: center;
|
|
||||||
gap: 0.5rem;
|
|
||||||
padding: 2rem 0;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pagination button,
|
|
||||||
.pagination form {
|
|
||||||
display: inline-flex;
|
|
||||||
margin: 0;
|
|
||||||
padding: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pagination button {
|
|
||||||
min-width: 40px;
|
|
||||||
height: 40px;
|
|
||||||
padding: 0 0.75rem;
|
|
||||||
border: 1px solid var(--border);
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
background: var(--bg);
|
|
||||||
color: var(--text-primary);
|
|
||||||
font-size: 0.9rem;
|
|
||||||
font-family: inherit;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: background 0.15s, border-color 0.15s;
|
|
||||||
line-height: 1;
|
|
||||||
box-sizing: border-box;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pagination button:hover {
|
|
||||||
background: var(--bg-secondary);
|
|
||||||
border-color: var(--border-focus);
|
|
||||||
}
|
|
||||||
|
|
||||||
.page-current {
|
|
||||||
min-width: 40px;
|
|
||||||
width: auto;
|
|
||||||
height: 40px;
|
|
||||||
padding: 0 0.75rem;
|
|
||||||
border: 1px solid var(--accent);
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
background: var(--accent);
|
|
||||||
color: #fff;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
font-weight: 500;
|
|
||||||
display: inline-flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: center;
|
|
||||||
box-sizing: border-box;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Footer
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
footer {
|
|
||||||
text-align: center;
|
|
||||||
padding: 3rem 1.5rem 2rem;
|
|
||||||
color: var(--text-muted);
|
|
||||||
font-size: 0.85rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
footer a {
|
|
||||||
color: var(--accent);
|
|
||||||
text-decoration: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
footer a:hover {
|
|
||||||
text-decoration: underline;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Corrections
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.correction {
|
|
||||||
font-size: 0.9rem;
|
|
||||||
color: var(--text-secondary);
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
padding: 0.5rem 0.75rem;
|
|
||||||
background: var(--bg-secondary);
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Image Grid
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.image-grid {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(auto-fill, minmax(180px, 1fr));
|
|
||||||
gap: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.image-result {
|
|
||||||
display: block;
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
overflow: hidden;
|
|
||||||
background: var(--bg-secondary);
|
|
||||||
border: 1px solid var(--border);
|
|
||||||
text-decoration: none;
|
|
||||||
color: inherit;
|
|
||||||
}
|
|
||||||
|
|
||||||
.image-thumb {
|
|
||||||
aspect-ratio: 1;
|
|
||||||
overflow: hidden;
|
|
||||||
background: var(--bg-tertiary);
|
|
||||||
}
|
|
||||||
|
|
||||||
.image-thumb img {
|
|
||||||
width: 100%;
|
|
||||||
height: 100%;
|
|
||||||
object-fit: cover;
|
|
||||||
}
|
|
||||||
|
|
||||||
.image-meta {
|
|
||||||
padding: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.image-title {
|
|
||||||
font-size: 0.8rem;
|
|
||||||
font-weight: 500;
|
|
||||||
color: var(--text-primary);
|
|
||||||
overflow: hidden;
|
|
||||||
text-overflow: ellipsis;
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
.image-source {
|
|
||||||
font-size: 0.7rem;
|
|
||||||
color: var(--text-muted);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Video Results
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.video-result {
|
|
||||||
display: flex;
|
|
||||||
gap: 1rem;
|
|
||||||
padding: 0.85rem 0;
|
|
||||||
border-bottom: 1px solid var(--border);
|
|
||||||
}
|
|
||||||
|
|
||||||
.video-result .result_thumbnail {
|
|
||||||
flex-shrink: 0;
|
|
||||||
width: 160px;
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
overflow: hidden;
|
|
||||||
background: var(--bg-tertiary);
|
|
||||||
}
|
|
||||||
|
|
||||||
.video-result .result_content_wrapper {
|
|
||||||
flex: 1;
|
|
||||||
min-width: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Back to Top
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.back-to-top {
|
|
||||||
text-align: center;
|
|
||||||
padding: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.back-to-top a {
|
|
||||||
color: var(--accent);
|
|
||||||
text-decoration: none;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.back-to-top a:hover {
|
|
||||||
text-decoration: underline;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Responsive
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
@media (max-width: 600px) {
|
|
||||||
.results-header {
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 0.75rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.results-logo {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.category-tabs {
|
|
||||||
overflow-x: auto;
|
|
||||||
-webkit-overflow-scrolling: touch;
|
|
||||||
}
|
|
||||||
|
|
||||||
.category-tabs::-webkit-scrollbar {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.image-grid {
|
|
||||||
grid-template-columns: repeat(auto-fill, minmax(140px, 1fr));
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ============================================================
|
|
||||||
Preferences Page
|
|
||||||
============================================================ */
|
|
||||||
|
|
||||||
.preferences-container {
|
|
||||||
max-width: 640px;
|
|
||||||
margin: 2rem auto;
|
|
||||||
padding: 0 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.preferences-title {
|
|
||||||
font-size: 1.5rem;
|
|
||||||
font-weight: 600;
|
|
||||||
margin-bottom: 2rem;
|
|
||||||
color: var(--text-primary);
|
|
||||||
}
|
|
||||||
|
|
||||||
.preferences-form {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 2rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-section {
|
|
||||||
background: var(--bg);
|
|
||||||
border: 1px solid var(--border);
|
|
||||||
border-radius: var(--radius-md);
|
|
||||||
padding: 1.25rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-section-title {
|
|
||||||
font-size: 0.7rem;
|
|
||||||
font-weight: 600;
|
|
||||||
text-transform: uppercase;
|
|
||||||
letter-spacing: 0.06em;
|
|
||||||
color: var(--text-muted);
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
padding-bottom: 0.5rem;
|
|
||||||
border-bottom: 1px solid var(--border);
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-desc {
|
|
||||||
font-size: 0.8rem;
|
|
||||||
color: var(--text-muted);
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-row {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: space-between;
|
|
||||||
gap: 1rem;
|
|
||||||
padding: 0.75rem 0;
|
|
||||||
border-bottom: 1px solid var(--border);
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-row:last-child {
|
|
||||||
border-bottom: none;
|
|
||||||
padding-bottom: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-row label:first-child {
|
|
||||||
font-size: 0.9rem;
|
|
||||||
color: var(--text-primary);
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-row-info {
|
|
||||||
flex: 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-row-info label {
|
|
||||||
font-weight: 500;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-row select {
|
|
||||||
padding: 0.5rem 0.75rem;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
font-family: inherit;
|
|
||||||
border: 1px solid var(--border);
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
background: var(--bg);
|
|
||||||
color: var(--text-primary);
|
|
||||||
cursor: pointer;
|
|
||||||
min-width: 120px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-row select:focus {
|
|
||||||
outline: none;
|
|
||||||
border-color: var(--accent);
|
|
||||||
}
|
|
||||||
|
|
||||||
.theme-buttons {
|
|
||||||
display: flex;
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.theme-btn {
|
|
||||||
padding: 0.5rem 1rem;
|
|
||||||
border: 1px solid var(--border);
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
background: var(--bg);
|
|
||||||
color: var(--text-secondary);
|
|
||||||
font-size: 0.85rem;
|
|
||||||
font-family: inherit;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: all 0.15s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.theme-btn:hover {
|
|
||||||
background: var(--bg-secondary);
|
|
||||||
}
|
|
||||||
|
|
||||||
.theme-btn.active {
|
|
||||||
background: var(--accent-soft);
|
|
||||||
border-color: var(--accent);
|
|
||||||
color: var(--accent);
|
|
||||||
font-weight: 500;
|
|
||||||
}
|
|
||||||
|
|
||||||
.engine-grid {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(auto-fill, minmax(140px, 1fr));
|
|
||||||
gap: 0.5rem;
|
|
||||||
margin-top: 0.75rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.engine-toggle {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 0.5rem;
|
|
||||||
padding: 0.5rem 0.75rem;
|
|
||||||
background: var(--bg-secondary);
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
font-size: 0.85rem;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: background 0.15s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.engine-toggle:hover {
|
|
||||||
background: var(--bg-tertiary);
|
|
||||||
}
|
|
||||||
|
|
||||||
.engine-toggle input[type="checkbox"] {
|
|
||||||
width: 16px;
|
|
||||||
height: 16px;
|
|
||||||
accent-color: var(--accent);
|
|
||||||
cursor: pointer;
|
|
||||||
}
|
|
||||||
|
|
||||||
.engine-toggle span {
|
|
||||||
color: var(--text-secondary);
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-actions {
|
|
||||||
display: flex;
|
|
||||||
gap: 1rem;
|
|
||||||
justify-content: flex-end;
|
|
||||||
padding-top: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.btn-primary,
|
|
||||||
.btn-secondary {
|
|
||||||
padding: 0.65rem 1.25rem;
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
font-size: 0.9rem;
|
|
||||||
font-family: inherit;
|
|
||||||
cursor: pointer;
|
|
||||||
text-decoration: none;
|
|
||||||
display: inline-flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: center;
|
|
||||||
transition: all 0.15s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.btn-primary {
|
|
||||||
background: var(--accent);
|
|
||||||
color: #fff;
|
|
||||||
border: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.btn-primary:hover {
|
|
||||||
background: var(--accent-hover);
|
|
||||||
}
|
|
||||||
|
|
||||||
.btn-secondary {
|
|
||||||
background: var(--bg);
|
|
||||||
color: var(--text-secondary);
|
|
||||||
border: 1px solid var(--border);
|
|
||||||
}
|
|
||||||
|
|
||||||
.btn-secondary:hover {
|
|
||||||
background: var(--bg-secondary);
|
|
||||||
color: var(--text-primary);
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (max-width: 600px) {
|
|
||||||
.engine-grid {
|
|
||||||
grid-template-columns: 1fr 1fr;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-row {
|
|
||||||
flex-direction: column;
|
|
||||||
align-items: flex-start;
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-row select {
|
|
||||||
width: 100%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pref-actions {
|
|
||||||
flex-direction: column;
|
|
||||||
}
|
|
||||||
|
|
||||||
.btn-primary,
|
|
||||||
.btn-secondary {
|
|
||||||
width: 100%;
|
|
||||||
text-align: center;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Dark mode class-based fallback */
|
|
||||||
.dark,
|
|
||||||
.dark body {
|
|
||||||
background: #0f0f0f !important;
|
|
||||||
color: #e8eaed !important;
|
|
||||||
}
|
|
||||||
.dark .site-header {
|
|
||||||
background: #1a1a1a !important;
|
|
||||||
border-color: #2e2e2e !important;
|
|
||||||
}
|
|
||||||
.dark .site-header a,
|
|
||||||
.dark .site-header .site-name {
|
|
||||||
color: #e8eaed !important;
|
|
||||||
}
|
|
||||||
.dark input[type="text"],
|
|
||||||
.dark input[type="search"],
|
|
||||||
.dark textarea {
|
|
||||||
background: #1a1a1a !important;
|
|
||||||
border-color: #2e2e2e !important;
|
|
||||||
color: #e8eaed !important;
|
|
||||||
}
|
|
||||||
.dark .search-btn,
|
|
||||||
.dark button {
|
|
||||||
background: #14b8a6 !important;
|
|
||||||
color: #0f0f0f !important;
|
|
||||||
}
|
|
||||||
.dark .pref-section,
|
|
||||||
.dark .pref-row,
|
|
||||||
.dark .engine-toggle,
|
|
||||||
.dark select {
|
|
||||||
background: #1a1a1a !important;
|
|
||||||
border-color: #2e2e2e !important;
|
|
||||||
color: #e8eaed !important;
|
|
||||||
}
|
|
||||||
.dark a {
|
|
||||||
color: #14b8a6 !important;
|
|
||||||
}
|
|
||||||
.dark .result,
|
|
||||||
.dark .result:hover,
|
|
||||||
.dark .result-item,
|
|
||||||
.dark .result-item:hover {
|
|
||||||
background: #1a1a1a !important;
|
|
||||||
border-color: #2e2e2e !important;
|
|
||||||
}
|
|
||||||
.dark footer {
|
|
||||||
background: #0f0f0f !important;
|
|
||||||
color: #9aa0a6 !important;
|
|
||||||
}
|
|
||||||
.dark .results-container,
|
|
||||||
.dark .results-content,
|
|
||||||
.dark .results-header {
|
|
||||||
background: #0f0f0f !important;
|
|
||||||
}
|
|
||||||
.dark .search-box {
|
|
||||||
background: #1a1a1a !important;
|
|
||||||
border-color: #2e2e2e !important;
|
|
||||||
}
|
|
||||||
|
|
@ -1,342 +0,0 @@
|
||||||
var ALL_ENGINES = [
|
|
||||||
'wikipedia', 'arxiv', 'crossref', 'braveapi',
|
|
||||||
'qwant', 'duckduckgo', 'github', 'reddit', 'bing'
|
|
||||||
];
|
|
||||||
|
|
||||||
var DEFAULT_PREFS = {
|
|
||||||
theme: 'system',
|
|
||||||
engines: ALL_ENGINES.slice(),
|
|
||||||
safeSearch: 'moderate',
|
|
||||||
format: 'html',
|
|
||||||
favicon: 'none' // 'none' | 'google' | 'duckduckgo'
|
|
||||||
};
|
|
||||||
|
|
||||||
var STORAGE_KEY = 'kafka_prefs';
|
|
||||||
|
|
||||||
function loadPrefs() {
|
|
||||||
var stored = localStorage.getItem(STORAGE_KEY);
|
|
||||||
var prefs = DEFAULT_PREFS;
|
|
||||||
if (stored) {
|
|
||||||
try {
|
|
||||||
var parsed = JSON.parse(stored);
|
|
||||||
prefs = {
|
|
||||||
theme: parsed.theme || DEFAULT_PREFS.theme,
|
|
||||||
engines: parsed.engines || DEFAULT_PREFS.engines.slice(),
|
|
||||||
safeSearch: parsed.safeSearch || DEFAULT_PREFS.safeSearch,
|
|
||||||
format: parsed.format || DEFAULT_PREFS.format,
|
|
||||||
favicon: parsed.favicon || DEFAULT_PREFS.favicon
|
|
||||||
};
|
|
||||||
} catch (e) {
|
|
||||||
prefs = DEFAULT_PREFS;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return prefs;
|
|
||||||
}
|
|
||||||
|
|
||||||
function savePrefs(prefs) {
|
|
||||||
localStorage.setItem(STORAGE_KEY, JSON.stringify(prefs));
|
|
||||||
}
|
|
||||||
|
|
||||||
function applyTheme(theme) {
|
|
||||||
if (theme === 'system') {
|
|
||||||
document.documentElement.removeAttribute('data-theme');
|
|
||||||
} else {
|
|
||||||
document.documentElement.setAttribute('data-theme', theme);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function applyFavicon(service) {
|
|
||||||
var faviconMap = {
|
|
||||||
google: function(domain) { return 'https://www.google.com/s2/favicons?domain=' + encodeURIComponent(domain) + '&sz=32'; },
|
|
||||||
duckduckgo: function(domain) { return 'https://icons.duckduckgo.com/ip3/' + encodeURIComponent(domain) + '.ico'; },
|
|
||||||
self: function(domain) { return '/favicon/' + encodeURIComponent(domain); }
|
|
||||||
};
|
|
||||||
var imgs = document.querySelectorAll('.result-favicon');
|
|
||||||
imgs.forEach(function(img) {
|
|
||||||
var domain = img.getAttribute('data-domain');
|
|
||||||
if (!domain) return;
|
|
||||||
if (service === 'none') {
|
|
||||||
img.style.display = 'none';
|
|
||||||
} else if (faviconMap[service]) {
|
|
||||||
img.style.display = '';
|
|
||||||
img.src = faviconMap[service](domain);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function syncEngineInput(prefs) {
|
|
||||||
var input = document.getElementById('engines-input');
|
|
||||||
if (input) {
|
|
||||||
input.value = prefs.engines.join(',');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function closePanel() {
|
|
||||||
var popover = document.getElementById('settings-popover');
|
|
||||||
var trigger = document.getElementById('settings-trigger');
|
|
||||||
if (popover) {
|
|
||||||
popover.setAttribute('data-open', 'false');
|
|
||||||
}
|
|
||||||
if (trigger) {
|
|
||||||
trigger.setAttribute('aria-expanded', 'false');
|
|
||||||
}
|
|
||||||
if (trigger) {
|
|
||||||
trigger.focus();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function openPanel() {
|
|
||||||
var popover = document.getElementById('settings-popover');
|
|
||||||
var trigger = document.getElementById('settings-trigger');
|
|
||||||
if (popover) {
|
|
||||||
popover.setAttribute('data-open', 'true');
|
|
||||||
}
|
|
||||||
if (trigger) {
|
|
||||||
trigger.setAttribute('aria-expanded', 'true');
|
|
||||||
}
|
|
||||||
var firstFocusable = popover ? popover.querySelector('button, input, select, textarea, a[href], [tabindex]:not([tabindex="-1"])') : null;
|
|
||||||
if (firstFocusable) {
|
|
||||||
firstFocusable.focus();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function escapeHtml(str) {
|
|
||||||
return String(str).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderPanel(prefs) {
|
|
||||||
var panel = document.getElementById('settings-popover');
|
|
||||||
if (!panel) return;
|
|
||||||
var body = panel.querySelector('.settings-popover-body');
|
|
||||||
if (!body) return;
|
|
||||||
|
|
||||||
var themeBtns = '';
|
|
||||||
['light', 'dark', 'system'].forEach(function(t) {
|
|
||||||
var icons = { light: '\u2600', dark: '\u263D', system: '\u2318' };
|
|
||||||
var labels = { light: 'Light', dark: 'Dark', system: 'System' };
|
|
||||||
var active = prefs.theme === t ? ' active' : '';
|
|
||||||
themeBtns += '<button class="theme-btn' + active + '" data-theme="' + t + '">' + icons[t] + ' ' + labels[t] + '</button>';
|
|
||||||
});
|
|
||||||
|
|
||||||
var engineToggles = '';
|
|
||||||
ALL_ENGINES.forEach(function(name) {
|
|
||||||
var checked = prefs.engines.indexOf(name) !== -1 ? ' checked' : '';
|
|
||||||
engineToggles += '<label class="engine-toggle"><input type="checkbox" value="' + escapeHtml(name) + '"' + checked + '><span>' + escapeHtml(name) + '</span></label>';
|
|
||||||
});
|
|
||||||
|
|
||||||
|
|
||||||
var faviconOptions = '';
|
|
||||||
['none', 'google', 'duckduckgo', 'self'].forEach(function(src) {
|
|
||||||
var labels = { none: 'None', google: 'Google', duckduckgo: 'DuckDuckGo', self: 'Self (Kafka)' };
|
|
||||||
var selected = prefs.favicon === src ? ' selected' : '';
|
|
||||||
faviconOptions += '<option value="' + src + '"' + selected + '>' + labels[src] + '</option>';
|
|
||||||
});
|
|
||||||
|
|
||||||
body.innerHTML =
|
|
||||||
'<div class="settings-section">' +
|
|
||||||
'<div class="settings-section-title">Appearance</div>' +
|
|
||||||
'<div class="theme-buttons">' + themeBtns + '</div>' +
|
|
||||||
'</div>' +
|
|
||||||
'<div class="settings-section">' +
|
|
||||||
'<div class="settings-section-title">Engines</div>' +
|
|
||||||
'<div class="engine-grid">' + engineToggles + '</div>' +
|
|
||||||
'<p class="settings-notice">Engine changes apply to your next search.</p>' +
|
|
||||||
'</div>' +
|
|
||||||
'<div class="settings-section">' +
|
|
||||||
'<div class="settings-section-title">Search Defaults</div>' +
|
|
||||||
'<div class="setting-row">' +
|
|
||||||
'<label for="pref-safesearch">Safe search</label>' +
|
|
||||||
'<select id="pref-safesearch">' + ssOptionsHtml + '</select>' +
|
|
||||||
'</div>' +
|
|
||||||
'<div class="setting-row">' +
|
|
||||||
'<label for="pref-format">Default format</label>' +
|
|
||||||
'<select id="pref-format">' + fmtOptionsHtml + '</select>' +
|
|
||||||
'</div>' +
|
|
||||||
'<div class="setting-row">' +
|
|
||||||
'<label for="pref-favicon">Favicon service</label>' +
|
|
||||||
'<select id="pref-favicon">' + faviconOptions + '</select>' +
|
|
||||||
'</div>' +
|
|
||||||
'</div>';
|
|
||||||
|
|
||||||
// Theme buttons
|
|
||||||
var themeBtnEls = panel.querySelectorAll('.theme-btn');
|
|
||||||
for (var i = 0; i < themeBtnEls.length; i++) {
|
|
||||||
themeBtnEls[i].addEventListener('click', (function(btn) {
|
|
||||||
return function() {
|
|
||||||
prefs.theme = btn.getAttribute('data-theme');
|
|
||||||
savePrefs(prefs);
|
|
||||||
applyTheme(prefs.theme);
|
|
||||||
syncEngineInput(prefs);
|
|
||||||
renderPanel(prefs);
|
|
||||||
};
|
|
||||||
})(themeBtnEls[i]));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Engine checkboxes
|
|
||||||
var checkboxes = panel.querySelectorAll('.engine-toggle input[type="checkbox"]');
|
|
||||||
for (var j = 0; j < checkboxes.length; j++) {
|
|
||||||
checkboxes[j].addEventListener('change', (function(cb) {
|
|
||||||
return function() {
|
|
||||||
var checked = Array.prototype.slice.call(panel.querySelectorAll('.engine-toggle input[type="checkbox"]:checked')).map(function(el) { return el.value; });
|
|
||||||
if (checked.length === 0) { renderPanel(prefs); return; }
|
|
||||||
prefs.engines = checked;
|
|
||||||
savePrefs(prefs);
|
|
||||||
syncEngineInput(prefs);
|
|
||||||
};
|
|
||||||
})(checkboxes[j]));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close button
|
|
||||||
var closeBtn = panel.querySelector('.settings-popover-close');
|
|
||||||
if (closeBtn) closeBtn.addEventListener('click', closePanel);
|
|
||||||
}
|
|
||||||
|
|
||||||
function initSettings() {
|
|
||||||
var prefs = loadPrefs();
|
|
||||||
applyTheme(prefs.theme);
|
|
||||||
applyFavicon(prefs.favicon);
|
|
||||||
syncEngineInput(prefs);
|
|
||||||
renderPanel(prefs);
|
|
||||||
|
|
||||||
var trigger = document.getElementById('settings-trigger');
|
|
||||||
var triggerMobile = document.getElementById('settings-trigger-mobile');
|
|
||||||
|
|
||||||
function togglePanel() {
|
|
||||||
var popover = document.getElementById('settings-popover');
|
|
||||||
if (popover && popover.getAttribute('data-open') === 'true') {
|
|
||||||
closePanel();
|
|
||||||
} else {
|
|
||||||
openPanel();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (trigger) {
|
|
||||||
trigger.addEventListener('click', togglePanel);
|
|
||||||
}
|
|
||||||
if (triggerMobile) {
|
|
||||||
triggerMobile.addEventListener('click', togglePanel);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Escape key handler
|
|
||||||
document.addEventListener('keydown', function(e) {
|
|
||||||
if (e.key === 'Escape') {
|
|
||||||
var popover = document.getElementById('settings-popover');
|
|
||||||
if (popover && popover.getAttribute('data-open') === 'true') {
|
|
||||||
closePanel();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Click outside handler
|
|
||||||
document.addEventListener('click', function(e) {
|
|
||||||
var popover = document.getElementById('settings-popover');
|
|
||||||
var trigger = document.getElementById('settings-trigger');
|
|
||||||
if (popover && popover.getAttribute('data-open') === 'true') {
|
|
||||||
if (!popover.contains(e.target) && (!trigger || !trigger.contains(e.target))) {
|
|
||||||
closePanel();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Focus trap
|
|
||||||
document.addEventListener('keydown', function(e) {
|
|
||||||
if (e.key === 'Tab') {
|
|
||||||
var popover = document.getElementById('settings-popover');
|
|
||||||
if (popover && popover.getAttribute('data-open') === 'true') {
|
|
||||||
var focusableElements = popover.querySelectorAll('button, input, select, textarea, a[href], [tabindex]:not([tabindex="-1"])');
|
|
||||||
var firstEl = focusableElements[0];
|
|
||||||
var lastEl = focusableElements[focusableElements.length - 1];
|
|
||||||
if (e.shiftKey && document.activeElement === firstEl) {
|
|
||||||
e.preventDefault();
|
|
||||||
lastEl.focus();
|
|
||||||
} else if (!e.shiftKey && document.activeElement === lastEl) {
|
|
||||||
e.preventDefault();
|
|
||||||
firstEl.focus();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
if (document.readyState === 'loading') {
|
|
||||||
document.addEventListener('DOMContentLoaded', initSettings);
|
|
||||||
} else {
|
|
||||||
initSettings();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Preferences page navigation
|
|
||||||
function initPreferences() {
|
|
||||||
var nav = document.getElementById('preferences-nav');
|
|
||||||
if (!nav) return;
|
|
||||||
|
|
||||||
var sections = document.querySelectorAll('.pref-section');
|
|
||||||
var navItems = nav.querySelectorAll('.preferences-nav-item');
|
|
||||||
|
|
||||||
function showSection(id) {
|
|
||||||
sections.forEach(function(sec) {
|
|
||||||
sec.style.display = sec.id === 'section-' + id ? 'block' : 'none';
|
|
||||||
});
|
|
||||||
navItems.forEach(function(item) {
|
|
||||||
item.classList.toggle('active', item.getAttribute('data-section') === id);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
navItems.forEach(function(item) {
|
|
||||||
item.addEventListener('click', function() {
|
|
||||||
showSection(item.getAttribute('data-section'));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// Load saved preferences
|
|
||||||
var prefs = loadPrefs();
|
|
||||||
|
|
||||||
// Apply favicon settings immediately on preferences page
|
|
||||||
applyFavicon(prefs.favicon);
|
|
||||||
|
|
||||||
// Theme
|
|
||||||
var themeEl = document.getElementById('pref-theme');
|
|
||||||
if (themeEl) {
|
|
||||||
themeEl.value = prefs.theme || 'system';
|
|
||||||
themeEl.addEventListener('change', function() {
|
|
||||||
prefs.theme = themeEl.value;
|
|
||||||
savePrefs(prefs);
|
|
||||||
applyTheme(prefs.theme);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Safe search
|
|
||||||
var ssEl = document.getElementById('pref-safesearch');
|
|
||||||
if (ssEl) {
|
|
||||||
ssEl.value = prefs.safeSearch || 'moderate';
|
|
||||||
ssEl.addEventListener('change', function() {
|
|
||||||
prefs.safeSearch = ssEl.value;
|
|
||||||
savePrefs(prefs);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Format (if exists on page)
|
|
||||||
var fmtEl = document.getElementById('pref-format');
|
|
||||||
if (fmtEl) {
|
|
||||||
fmtEl.value = prefs.format || 'html';
|
|
||||||
fmtEl.addEventListener('change', function() {
|
|
||||||
prefs.format = fmtEl.value;
|
|
||||||
savePrefs(prefs);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Favicon service (if exists on page)
|
|
||||||
var faviconEl = document.getElementById('pref-favicon');
|
|
||||||
if (faviconEl) {
|
|
||||||
faviconEl.value = prefs.favicon || 'none';
|
|
||||||
faviconEl.addEventListener('change', function() {
|
|
||||||
prefs.favicon = faviconEl.value;
|
|
||||||
savePrefs(prefs);
|
|
||||||
applyFavicon(prefs.favicon);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Show first section by default
|
|
||||||
showSection('search');
|
|
||||||
}
|
|
||||||
|
|
||||||
document.addEventListener('DOMContentLoaded', initPreferences);
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,15 +0,0 @@
|
||||||
{{define "image_item"}}
|
|
||||||
<a class="image-result" href="{{.URL}}" target="_blank" rel="noopener noreferrer">
|
|
||||||
<div class="image-thumb">
|
|
||||||
{{if .Thumbnail}}
|
|
||||||
<img src="{{.Thumbnail}}" alt="{{.Title}}" loading="lazy">
|
|
||||||
{{else}}
|
|
||||||
<div class="image-placeholder" aria-hidden="true">🖼️</div>
|
|
||||||
{{end}}
|
|
||||||
</div>
|
|
||||||
<div class="image-meta">
|
|
||||||
<span class="image-title">{{.Title}}</span>
|
|
||||||
{{if .Content}}<span class="image-source">{{.Content}}</span>{{end}}
|
|
||||||
</div>
|
|
||||||
</a>
|
|
||||||
{{end}}
|
|
||||||
|
|
@ -1,25 +1,15 @@
|
||||||
{{define "title"}}{{end}}
|
{{define "title"}}{{end}}
|
||||||
{{define "content"}}
|
{{define "content"}}
|
||||||
<div class="home-container">
|
<div class="index">
|
||||||
<a href="/" class="home-logo">
|
<div class="title"><h1>kafka</h1></div>
|
||||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">
|
<div id="search">
|
||||||
<circle cx="11" cy="11" r="8"/>
|
<form method="GET" action="/search" role="search" id="search-form">
|
||||||
<path d="m21 21-4.35-4.35"/>
|
<input type="text" name="q" id="q" placeholder="Search…" autocomplete="off" autofocus
|
||||||
</svg>
|
hx-get="/search" hx-target="#results" hx-trigger="keyup changed delay:500ms" hx-include="this">
|
||||||
<span class="home-logo-text">samsa</span>
|
<button type="submit">Search</button>
|
||||||
</a>
|
|
||||||
<p class="home-tagline">Private meta-search, powered by open source.</p>
|
|
||||||
|
|
||||||
<form class="search-form" method="GET" action="/search" role="search">
|
|
||||||
<div class="search-box">
|
|
||||||
<input type="text" name="q" placeholder="Search the web…" autocomplete="off" autofocus>
|
|
||||||
<button type="submit" class="search-btn" aria-label="Search">
|
|
||||||
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round">
|
|
||||||
<circle cx="11" cy="11" r="8"/>
|
|
||||||
<path d="m21 21-4.35-4.35"/>
|
|
||||||
</svg>
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
</form>
|
</form>
|
||||||
|
<div id="autocomplete-dropdown"></div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div id="results"></div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,12 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
|
<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
|
||||||
<ShortName>samsa</ShortName>
|
<ShortName>kafka</ShortName>
|
||||||
<Description>A privacy-respecting, open metasearch engine</Description>
|
<Description>A privacy-respecting, open metasearch engine</Description>
|
||||||
<InputEncoding>UTF-8</InputEncoding>
|
<InputEncoding>UTF-8</InputEncoding>
|
||||||
<OutputEncoding>UTF-8</OutputEncoding>
|
<OutputEncoding>UTF-8</OutputEncoding>
|
||||||
<LongName>samsa — Privacy-respecting metasearch</LongName>
|
<LongName>kafka — Privacy-respecting metasearch</LongName>
|
||||||
<Image width="16" height="16" type="image/svg+xml">/static/img/favicon.svg</Image>
|
<Image width="16" height="16" type="image/svg+xml">/static/img/favicon.svg</Image>
|
||||||
<Contact>https://git.ashisgreat.xyz/penal-colony/samsa</Contact>
|
<Contact>https://git.ashisgreat.xyz/penal-colony/kafka</Contact>
|
||||||
<Url type="text/html" method="GET" template="{baseUrl}/search?q={searchTerms}&format=html">
|
<Url type="text/html" method="GET" template="{baseUrl}/search?q={searchTerms}&format=html">
|
||||||
<Param name="pageno" value="{startPage?}" />
|
<Param name="pageno" value="{startPage?}" />
|
||||||
<Param name="language" value="{language?}" />
|
<Param name="language" value="{language?}" />
|
||||||
|
|
|
||||||
|
|
@ -1,112 +0,0 @@
|
||||||
{{define "title"}}Preferences{{end}}
|
|
||||||
{{define "content"}}
|
|
||||||
<div class="preferences-container">
|
|
||||||
<h1 class="preferences-title">Preferences</h1>
|
|
||||||
|
|
||||||
<form class="preferences-form" method="POST" action="/preferences">
|
|
||||||
|
|
||||||
<section class="pref-section">
|
|
||||||
<h2 class="pref-section-title">Appearance</h2>
|
|
||||||
<div class="pref-row">
|
|
||||||
<label for="theme-select">Theme</label>
|
|
||||||
<select name="theme" id="theme-select">
|
|
||||||
<option value="light" {{if eq .Theme "light"}}selected{{end}}>Light</option>
|
|
||||||
<option value="dark" {{if eq .Theme "dark"}}selected{{end}}>Dark</option>
|
|
||||||
</select>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="pref-section">
|
|
||||||
<h2 class="pref-section-title">Search Engines</h2>
|
|
||||||
<p class="pref-desc">Select which engines to use for searches.</p>
|
|
||||||
<div class="engine-grid">
|
|
||||||
<label class="engine-toggle">
|
|
||||||
<input type="checkbox" name="engine" value="google" checked>
|
|
||||||
<span>Google</span>
|
|
||||||
</label>
|
|
||||||
<label class="engine-toggle">
|
|
||||||
<input type="checkbox" name="engine" value="duckduckgo" checked>
|
|
||||||
<span>DuckDuckGo</span>
|
|
||||||
</label>
|
|
||||||
<label class="engine-toggle">
|
|
||||||
<input type="checkbox" name="engine" value="bing" checked>
|
|
||||||
<span>Bing</span>
|
|
||||||
</label>
|
|
||||||
<label class="engine-toggle">
|
|
||||||
<input type="checkbox" name="engine" value="brave" checked>
|
|
||||||
<span>Brave</span>
|
|
||||||
</label>
|
|
||||||
<label class="engine-toggle">
|
|
||||||
<input type="checkbox" name="engine" value="wikipedia" checked>
|
|
||||||
<span>Wikipedia</span>
|
|
||||||
</label>
|
|
||||||
<label class="engine-toggle">
|
|
||||||
<input type="checkbox" name="engine" value="wikidata" checked>
|
|
||||||
<span>Wikidata</span>
|
|
||||||
</label>
|
|
||||||
<label class="engine-toggle">
|
|
||||||
<input type="checkbox" name="engine" value="github">
|
|
||||||
<span>GitHub</span>
|
|
||||||
</label>
|
|
||||||
<label class="engine-toggle">
|
|
||||||
<input type="checkbox" name="engine" value="reddit">
|
|
||||||
<span>Reddit</span>
|
|
||||||
</label>
|
|
||||||
<label class="engine-toggle">
|
|
||||||
<input type="checkbox" name="engine" value="youtube">
|
|
||||||
<span>YouTube</span>
|
|
||||||
</label>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="pref-section">
|
|
||||||
<h2 class="pref-section-title">Privacy</h2>
|
|
||||||
<div class="pref-row">
|
|
||||||
<div class="pref-row-info">
|
|
||||||
<label>Safe Search</label>
|
|
||||||
<p class="pref-desc">Filter explicit content from results</p>
|
|
||||||
</div>
|
|
||||||
<select name="safesearch">
|
|
||||||
<option value="0">Off</option>
|
|
||||||
<option value="1" selected>Moderate</option>
|
|
||||||
<option value="2">Strict</option>
|
|
||||||
</select>
|
|
||||||
</div>
|
|
||||||
<div class="pref-row">
|
|
||||||
<div class="pref-row-info">
|
|
||||||
<label for="pref-favicon">Favicon Service</label>
|
|
||||||
<p class="pref-desc">Fetch favicons for result URLs. "None" is most private.</p>
|
|
||||||
</div>
|
|
||||||
<select name="favicon" id="pref-favicon">
|
|
||||||
<option value="none" {{if eq .FaviconService "none"}}selected{{end}}>None</option>
|
|
||||||
<option value="google" {{if eq .FaviconService "google"}}selected{{end}}>Google</option>
|
|
||||||
<option value="duckduckgo" {{if eq .FaviconService "duckduckgo"}}selected{{end}}>DuckDuckGo</option>
|
|
||||||
<option value="self" {{if eq .FaviconService "self"}}selected{{end}}>Self (Kafka)</option>
|
|
||||||
</select>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="pref-section">
|
|
||||||
<h2 class="pref-section-title">Language</h2>
|
|
||||||
<div class="pref-row">
|
|
||||||
<label for="search-lang">Interface & Search Language</label>
|
|
||||||
<select name="language" id="search-lang">
|
|
||||||
<option value="all" selected>All languages</option>
|
|
||||||
<option value="en">English</option>
|
|
||||||
<option value="de">Deutsch</option>
|
|
||||||
<option value="fr">Français</option>
|
|
||||||
<option value="es">Español</option>
|
|
||||||
<option value="zh">中文</option>
|
|
||||||
<option value="ja">日本語</option>
|
|
||||||
<option value="ru">Русский</option>
|
|
||||||
</select>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<div class="pref-actions">
|
|
||||||
<a href="/" class="btn-secondary">Cancel</a>
|
|
||||||
<button type="submit" class="btn-primary">Save Preferences</button>
|
|
||||||
</div>
|
|
||||||
</form>
|
|
||||||
</div>
|
|
||||||
{{end}}
|
|
||||||
|
|
@ -1,17 +1,16 @@
|
||||||
{{define "result_item"}}
|
{{define "result_item"}}
|
||||||
<article class="result" data-engine="{{.Engine}}">
|
<article class="result">
|
||||||
<div class="result_header">
|
<h3 class="result_header">
|
||||||
<a href="{{.URL}}" target="_blank" rel="noopener noreferrer">{{.SafeTitle}}</a>
|
<a href="{{.URL}}" target="_blank" rel="noopener noreferrer">{{.Title}}</a>
|
||||||
</div>
|
</h3>
|
||||||
<div class="result_url">
|
<div class="result_url">
|
||||||
{{if .FaviconIconURL}}
|
|
||||||
<img class="result-favicon" src="{{.FaviconIconURL}}" alt="" loading="lazy" width="14" height="14">
|
|
||||||
{{end}}
|
|
||||||
<a href="{{.URL}}" target="_blank" rel="noopener noreferrer">{{.URL}}</a>
|
<a href="{{.URL}}" target="_blank" rel="noopener noreferrer">{{.URL}}</a>
|
||||||
<span class="engine-badge" data-engine="{{.Engine}}">{{.Engine}}</span>
|
|
||||||
</div>
|
</div>
|
||||||
{{if .Content}}
|
{{if .Content}}
|
||||||
<p class="result_content">{{.SafeContent}}</p>
|
<p class="result_content">{{.Content}}</p>
|
||||||
|
{{end}}
|
||||||
|
{{if .Engine}}
|
||||||
|
<div class="result_engine"><span class="engine">{{.Engine}}</span></div>
|
||||||
{{end}}
|
{{end}}
|
||||||
</article>
|
</article>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
|
||||||
|
|
@ -1,43 +1,14 @@
|
||||||
{{define "title"}}{{if .Query}}{{.Query}} — {{end}}{{end}}
|
{{define "title"}}{{if .Query}}{{.Query}} — {{end}}{{end}}
|
||||||
{{define "content"}}
|
{{define "content"}}
|
||||||
<div class="results-container">
|
<div id="search">
|
||||||
<div class="results-header">
|
<form method="GET" action="/search" role="search">
|
||||||
<div class="results-header-inner">
|
<input type="text" name="q" id="q" value="{{.Query}}" autocomplete="off"
|
||||||
<a href="/" class="results-logo">
|
hx-get="/search" hx-target="#results" hx-trigger="keyup changed delay:500ms" hx-include="this">
|
||||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">
|
<button type="submit">Search</button>
|
||||||
<circle cx="11" cy="11" r="8"/>
|
|
||||||
<path d="m21 21-4.35-4.35"/>
|
|
||||||
</svg>
|
|
||||||
<span>samsa</span>
|
|
||||||
</a>
|
|
||||||
|
|
||||||
<form class="header-search" method="GET" action="/search" role="search">
|
|
||||||
{{if and .ActiveCategory (ne .ActiveCategory "all")}}
|
|
||||||
<input type="hidden" name="category" value="{{.ActiveCategory}}">
|
|
||||||
{{end}}
|
|
||||||
<div class="search-box">
|
|
||||||
<input type="text" name="q" value="{{.Query}}" placeholder="Search…" autocomplete="off">
|
|
||||||
<button type="submit" class="search-btn" aria-label="Search">
|
|
||||||
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round">
|
|
||||||
<circle cx="11" cy="11" r="8"/>
|
|
||||||
<path d="m21 21-4.35-4.35"/>
|
|
||||||
</svg>
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="category-tabs" role="tablist">
|
<div id="results">
|
||||||
<a href="/search?q={{.Query | urlquery}}&category=" class="category-tab {{if or (eq .ActiveCategory "") (eq .ActiveCategory "all")}}active{{end}}">All</a>
|
|
||||||
<a href="/search?q={{.Query | urlquery}}&category=general" class="category-tab {{if eq .ActiveCategory "general"}}active{{end}}">General</a>
|
|
||||||
<a href="/search?q={{.Query | urlquery}}&category=it" class="category-tab {{if eq .ActiveCategory "it"}}active{{end}}">IT</a>
|
|
||||||
<a href="/search?q={{.Query | urlquery}}&category=news" class="category-tab {{if eq .ActiveCategory "news"}}active{{end}}">News</a>
|
|
||||||
<a href="/search?q={{.Query | urlquery}}&category=images" class="category-tab {{if eq .ActiveCategory "images"}}active{{end}}">Images</a>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="results-content">
|
|
||||||
{{template "results_inner" .}}
|
{{template "results_inner" .}}
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
|
||||||
|
|
@ -1,108 +1,107 @@
|
||||||
{{define "results_inner"}}
|
{{define "results_inner"}}
|
||||||
{{if .Corrections}}
|
{{if .Corrections}}
|
||||||
<div id="corrections" class="correction">{{range .Corrections}}{{.}} {{end}}</div>
|
<div class="corrections">
|
||||||
{{end}}
|
{{range .Corrections}}<span class="correction">{{.}}</span>{{end}}
|
||||||
|
|
||||||
{{if .Infoboxes}}
|
|
||||||
<div class="infobox-list" role="region" aria-label="Summary">
|
|
||||||
{{range .Infoboxes}}
|
|
||||||
<aside class="infobox-card">
|
|
||||||
{{if .ImgSrc}}
|
|
||||||
<div class="infobox-image-wrap">
|
|
||||||
<img src="{{.ImgSrc}}" alt="" class="infobox-img" loading="lazy" width="120" height="120">
|
|
||||||
</div>
|
|
||||||
{{end}}
|
|
||||||
<div class="infobox-main">
|
|
||||||
{{if .Title}}<h2 class="infobox-title">{{.Title}}</h2>{{end}}
|
|
||||||
{{if .Content}}<p class="infobox-content">{{.Content}}</p>{{end}}
|
|
||||||
{{if .URL}}<a href="{{.URL}}" class="infobox-link" target="_blank" rel="noopener noreferrer">Read article on Wikipedia</a>{{end}}
|
|
||||||
</div>
|
|
||||||
</aside>
|
|
||||||
{{end}}
|
|
||||||
</div>
|
|
||||||
{{end}}
|
|
||||||
|
|
||||||
{{if .UnresponsiveEngines}}
|
|
||||||
<div class="engine-errors-wrap" role="region" aria-label="Engine errors">
|
|
||||||
<details class="engine-errors">
|
|
||||||
<summary>Some search engines had errors</summary>
|
|
||||||
<ul class="engine-errors-list">
|
|
||||||
{{range .UnresponsiveEngines}}
|
|
||||||
<li class="engine-error-item">
|
|
||||||
<code class="engine-error-engine">{{index . 0}}</code>
|
|
||||||
<span class="engine-error-reason">{{index . 1}}</span>
|
|
||||||
</li>
|
|
||||||
{{end}}
|
|
||||||
</ul>
|
|
||||||
</details>
|
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
{{if .Answers}}
|
{{if .Answers}}
|
||||||
<div id="answers">
|
<div id="answers">
|
||||||
{{range .Answers}}
|
{{range .Answers}}
|
||||||
<div class="dialog-error">{{.}}</div>
|
<div class="answer">{{.}}</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
<div class="results-meta" id="results-meta">
|
<div id="sidebar">
|
||||||
{{if .NumberOfResults}}
|
{{if .NumberOfResults}}
|
||||||
<span>{{.NumberOfResults}} results</span>
|
<p id="result_count"><small>{{.NumberOfResults}} results</small></p>
|
||||||
|
{{end}}
|
||||||
|
|
||||||
|
{{if .Infoboxes}}
|
||||||
|
<div id="infoboxes">
|
||||||
|
{{range .Infoboxes}}
|
||||||
|
<div class="infobox">
|
||||||
|
{{if .title}}<div class="title">{{.title}}</div>{{end}}
|
||||||
|
{{if .content}}<div class="content">{{.content}}</div>{{end}}
|
||||||
|
{{if .img_src}}<img src="{{.img_src}}" alt="{{.title}}" loading="lazy">{{end}}
|
||||||
|
</div>
|
||||||
|
{{end}}
|
||||||
|
</div>
|
||||||
|
{{end}}
|
||||||
|
|
||||||
|
{{if .Suggestions}}
|
||||||
|
<div id="suggestions">
|
||||||
|
<small>Suggestions:</small>
|
||||||
|
<div>
|
||||||
|
{{range .Suggestions}}<span class="suggestion"><a href="/search?q={{.}}">{{.}}</a></span>{{end}}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{{end}}
|
||||||
|
|
||||||
|
{{if .UnresponsiveEngines}}
|
||||||
|
<div class="unresponsive_engines">
|
||||||
|
<small>Unresponsive engines:</small>
|
||||||
|
<ul>
|
||||||
|
{{range .UnresponsiveEngines}}<li>{{index . 0}}: {{index . 1}}</li>{{end}}
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div id="urls" role="main">
|
<div id="urls" role="main">
|
||||||
{{if .Results}}
|
{{if .Results}}
|
||||||
{{if .IsImageSearch}}
|
|
||||||
<div class="image-grid">
|
|
||||||
{{range .Results}}
|
{{range .Results}}
|
||||||
{{if eq .Template "images"}}
|
|
||||||
{{template "image_item" .}}
|
|
||||||
{{end}}
|
|
||||||
{{end}}
|
|
||||||
</div>
|
|
||||||
{{else}}
|
|
||||||
{{range .Results}}
|
|
||||||
{{if eq .Template "videos"}}
|
|
||||||
{{template "video_item" .}}
|
|
||||||
{{else if eq .Template "images"}}
|
|
||||||
{{template "image_item" .}}
|
|
||||||
{{else}}
|
|
||||||
{{template "result_item" .}}
|
{{template "result_item" .}}
|
||||||
{{end}}
|
{{end}}
|
||||||
{{end}}
|
{{else if not .Answers}}
|
||||||
{{end}}
|
<div class="no_results">
|
||||||
{{else if and (not .Answers) (not .Infoboxes)}}
|
<p>No results found.</p>
|
||||||
<div class="no-results">
|
{{if .Query}}<p>Try different keywords or check your spelling.</p>{{end}}
|
||||||
<div class="no-results-icon" aria-hidden="true">🔍</div>
|
|
||||||
<h2>No results found</h2>
|
|
||||||
<p>Try different keywords or check your spelling.</p>
|
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{{if .Pageno}}
|
{{if .Pageno}}
|
||||||
<nav class="pagination" role="navigation" aria-label="Pagination">
|
<nav id="pagination" role="navigation">
|
||||||
{{if gt .Pageno 1}}
|
{{if gt .Pageno 1}}
|
||||||
<a class="pag-link" href="/search?q={{.Query | urlquery}}&pageno={{.PrevPage}}{{if and .ActiveCategory (ne .ActiveCategory "all")}}&category={{.ActiveCategory | urlquery}}{{end}}">← Prev</a>
|
<form method="GET" action="/search" class="previous_page">
|
||||||
|
<input type="hidden" name="q" value="{{.Query}}">
|
||||||
|
<input type="hidden" name="pageno" value="{{.PrevPage}}">
|
||||||
|
<input type="hidden" name="format" value="html">
|
||||||
|
<button type="submit" role="link">← Previous</button>
|
||||||
|
</form>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
|
<div class="numbered_pagination">
|
||||||
{{range .PageNumbers}}
|
{{range .PageNumbers}}
|
||||||
{{if .IsCurrent}}
|
{{if .IsCurrent}}
|
||||||
<span class="page-current" aria-current="page">{{.Num}}</span>
|
<span class="page_number_current">{{.Num}}</span>
|
||||||
{{else}}
|
{{else}}
|
||||||
<a class="pag-link" href="/search?q={{$.Query | urlquery}}&pageno={{.Num}}{{if and $.ActiveCategory (ne $.ActiveCategory "all")}}&category={{$.ActiveCategory | urlquery}}{{end}}">{{.Num}}</a>
|
<form method="GET" action="/search" class="page_number">
|
||||||
|
<input type="hidden" name="q" value="{{$.Query}}">
|
||||||
|
<input type="hidden" name="pageno" value="{{.Num}}">
|
||||||
|
<input type="hidden" name="format" value="html">
|
||||||
|
<button type="submit" role="link">{{.Num}}</button>
|
||||||
|
</form>
|
||||||
{{end}}
|
{{end}}
|
||||||
{{end}}
|
{{end}}
|
||||||
|
</div>
|
||||||
|
|
||||||
{{if .HasNext}}
|
{{if .HasNext}}
|
||||||
<a class="pag-link" href="/search?q={{.Query | urlquery}}&pageno={{.NextPage}}{{if and .ActiveCategory (ne .ActiveCategory "all")}}&category={{.ActiveCategory | urlquery}}{{end}}">Next →</a>
|
<form method="GET" action="/search" class="next_page">
|
||||||
|
<input type="hidden" name="q" value="{{.Query}}">
|
||||||
|
<input type="hidden" name="pageno" value="{{.NextPage}}">
|
||||||
|
<input type="hidden" name="format" value="html">
|
||||||
|
<button type="submit" role="link">Next →</button>
|
||||||
|
</form>
|
||||||
{{end}}
|
{{end}}
|
||||||
</nav>
|
</nav>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
<div class="back-to-top">
|
<div id="backToTop">
|
||||||
<a href="#top">↑ Back to top</a>
|
<a href="#">↑ Back to top</a>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="htmx-indicator">Searching…</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
{{define "video_item"}}
|
|
||||||
<article class="result video-result" data-engine="{{.Engine}}">
|
|
||||||
{{if .Thumbnail}}
|
|
||||||
<div class="result_thumbnail">
|
|
||||||
<a href="{{.URL}}" target="_blank" rel="noopener noreferrer">
|
|
||||||
<img src="{{.Thumbnail}}" alt="{{.Title}}" loading="lazy">
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
{{end}}
|
|
||||||
<div class="result_content_wrapper">
|
|
||||||
<div class="result_header">
|
|
||||||
<a href="{{.URL}}" target="_blank" rel="noopener noreferrer">{{.SafeTitle}}</a>
|
|
||||||
</div>
|
|
||||||
<div class="result_url">
|
|
||||||
{{if .FaviconIconURL}}
|
|
||||||
<img class="result-favicon" src="{{.FaviconIconURL}}" alt="" loading="lazy" width="14" height="14">
|
|
||||||
{{end}}
|
|
||||||
{{if .URL}}
|
|
||||||
<a href="{{.URL}}" target="_blank" rel="noopener noreferrer">{{.URL}}</a>
|
|
||||||
{{end}}
|
|
||||||
<span class="engine-badge" data-engine="{{.Engine}}">{{.Engine}}</span>
|
|
||||||
</div>
|
|
||||||
{{if .Content}}
|
|
||||||
<p class="result_content">{{.SafeContent}}</p>
|
|
||||||
{{end}}
|
|
||||||
</div>
|
|
||||||
</article>
|
|
||||||
{{end}}
|
|
||||||
|
|
@ -1,34 +1,14 @@
|
||||||
// samsa — a privacy-respecting metasearch engine
|
|
||||||
// Copyright (C) 2026-present metamorphosis-dev
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package views
|
package views
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"embed"
|
"embed"
|
||||||
"encoding/xml"
|
|
||||||
"html"
|
|
||||||
"html/template"
|
"html/template"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
"github.com/metamorphosis-dev/samsa/internal/util"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed all:templates
|
//go:embed all:templates
|
||||||
|
|
@ -39,7 +19,6 @@ var staticFS embed.FS
|
||||||
|
|
||||||
// PageData holds all data passed to templates.
|
// PageData holds all data passed to templates.
|
||||||
type PageData struct {
|
type PageData struct {
|
||||||
SourceURL string
|
|
||||||
Query string
|
Query string
|
||||||
Pageno int
|
Pageno int
|
||||||
PrevPage int
|
PrevPage int
|
||||||
|
|
@ -53,37 +32,10 @@ type PageData struct {
|
||||||
Infoboxes []InfoboxView
|
Infoboxes []InfoboxView
|
||||||
UnresponsiveEngines [][2]string
|
UnresponsiveEngines [][2]string
|
||||||
PageNumbers []PageNumber
|
PageNumbers []PageNumber
|
||||||
ShowHeader bool
|
|
||||||
IsImageSearch bool
|
|
||||||
// Theme is the user's selected theme (light/dark) from cookie
|
|
||||||
Theme string
|
|
||||||
FaviconService string
|
|
||||||
// New fields for three-column layout
|
|
||||||
Categories []string
|
|
||||||
CategoryIcons map[string]string
|
|
||||||
DisabledCategories []string
|
|
||||||
ActiveCategory string
|
|
||||||
TimeFilters []FilterOption
|
|
||||||
TypeFilters []FilterOption
|
|
||||||
ActiveTime string
|
|
||||||
ActiveType string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ResultView is a template-friendly wrapper around a MainResult.
|
// ResultView is a template-friendly wrapper around a MainResult.
|
||||||
type ResultView struct {
|
type ResultView contracts.MainResult
|
||||||
contracts.MainResult
|
|
||||||
// TemplateName is the actual template to dispatch to, computed from Template.
|
|
||||||
// "videos" maps to "video_item", everything else maps to "result_item".
|
|
||||||
TemplateName string
|
|
||||||
// Domain is the hostname extracted from the result URL, used for favicon proxying.
|
|
||||||
Domain string
|
|
||||||
// FaviconIconURL is the resolved favicon image URL for the user's favicon preference (empty = hide).
|
|
||||||
FaviconIconURL string
|
|
||||||
// SafeTitle and SafeContent are HTML-unescaped versions for rendering.
|
|
||||||
// The API returns HTML entities which Go templates escape by default.
|
|
||||||
SafeTitle template.HTML
|
|
||||||
SafeContent template.HTML
|
|
||||||
}
|
|
||||||
|
|
||||||
// PageNumber represents a numbered pagination button.
|
// PageNumber represents a numbered pagination button.
|
||||||
type PageNumber struct {
|
type PageNumber struct {
|
||||||
|
|
@ -96,20 +48,12 @@ type InfoboxView struct {
|
||||||
Title string
|
Title string
|
||||||
Content string
|
Content string
|
||||||
ImgSrc string
|
ImgSrc string
|
||||||
URL string
|
|
||||||
}
|
|
||||||
|
|
||||||
// FilterOption represents a filter radio option for the sidebar.
|
|
||||||
type FilterOption struct {
|
|
||||||
Label string
|
|
||||||
Value string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
tmplFull *template.Template
|
tmplFull *template.Template
|
||||||
tmplIndex *template.Template
|
tmplIndex *template.Template
|
||||||
tmplFragment *template.Template
|
tmplFragment *template.Template
|
||||||
tmplPreferences *template.Template
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
|
@ -121,16 +65,13 @@ func init() {
|
||||||
}
|
}
|
||||||
|
|
||||||
tmplFull = template.Must(template.New("").Funcs(funcMap).ParseFS(tmplFS,
|
tmplFull = template.Must(template.New("").Funcs(funcMap).ParseFS(tmplFS,
|
||||||
"base.html", "results.html", "results_inner.html", "result_item.html", "video_item.html", "image_item.html",
|
"base.html", "results.html", "results_inner.html", "result_item.html",
|
||||||
))
|
))
|
||||||
tmplIndex = template.Must(template.New("").Funcs(funcMap).ParseFS(tmplFS,
|
tmplIndex = template.Must(template.New("").Funcs(funcMap).ParseFS(tmplFS,
|
||||||
"base.html", "index.html",
|
"base.html", "index.html",
|
||||||
))
|
))
|
||||||
tmplFragment = template.Must(template.New("").Funcs(funcMap).ParseFS(tmplFS,
|
tmplFragment = template.Must(template.New("").Funcs(funcMap).ParseFS(tmplFS,
|
||||||
"results.html", "results_inner.html", "result_item.html", "video_item.html", "image_item.html",
|
"results_inner.html", "result_item.html",
|
||||||
))
|
|
||||||
tmplPreferences = template.Must(template.New("").Funcs(funcMap).ParseFS(tmplFS,
|
|
||||||
"base.html", "preferences.html",
|
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -139,117 +80,31 @@ func StaticFS() (fs.FS, error) {
|
||||||
return fs.Sub(staticFS, "static")
|
return fs.Sub(staticFS, "static")
|
||||||
}
|
}
|
||||||
|
|
||||||
// OpenSearchXML returns the OpenSearch description XML with the base URL
|
// OpenSearchXML returns the OpenSearch description XML with {baseUrl}
|
||||||
// safely embedded via xml.EscapeText (no raw string interpolation).
|
// replaced by the provided base URL.
|
||||||
func OpenSearchXML(baseURL string) ([]byte, error) {
|
func OpenSearchXML(baseURL string) ([]byte, error) {
|
||||||
tmplFS, _ := fs.Sub(templatesFS, "templates")
|
tmplFS, _ := fs.Sub(templatesFS, "templates")
|
||||||
data, err := fs.ReadFile(tmplFS, "opensearch.xml")
|
data, err := fs.ReadFile(tmplFS, "opensearch.xml")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
result := strings.ReplaceAll(string(data), "{baseUrl}", baseURL)
|
||||||
var buf strings.Builder
|
|
||||||
xml.Escape(&buf, []byte(baseURL))
|
|
||||||
escapedBaseURL := buf.String()
|
|
||||||
|
|
||||||
result := strings.ReplaceAll(string(data), "{baseUrl}", escapedBaseURL)
|
|
||||||
return []byte(result), nil
|
return []byte(result), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// faviconIconURL returns a safe img src for the given service and hostname, or "" for none/invalid.
|
|
||||||
func faviconIconURL(service, domain string) string {
|
|
||||||
domain = strings.TrimSpace(domain)
|
|
||||||
if domain == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
switch service {
|
|
||||||
case "google":
|
|
||||||
return "https://www.google.com/s2/favicons?domain=" + url.QueryEscape(domain) + "&sz=32"
|
|
||||||
case "duckduckgo":
|
|
||||||
return "https://icons.duckduckgo.com/ip3/" + domain + ".ico"
|
|
||||||
case "self":
|
|
||||||
return "/favicon/" + domain
|
|
||||||
default:
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// FromResponse builds PageData from a search response and request params.
|
// FromResponse builds PageData from a search response and request params.
|
||||||
func FromResponse(resp contracts.SearchResponse, query string, pageno int, activeCategory, activeTime, activeType, faviconService string) PageData {
|
func FromResponse(resp contracts.SearchResponse, query string, pageno int) PageData {
|
||||||
// Set defaults
|
|
||||||
if activeCategory == "" {
|
|
||||||
activeCategory = "all"
|
|
||||||
}
|
|
||||||
|
|
||||||
pd := PageData{
|
pd := PageData{
|
||||||
Query: query,
|
Query: query,
|
||||||
Pageno: pageno,
|
Pageno: pageno,
|
||||||
NumberOfResults: resp.NumberOfResults,
|
NumberOfResults: resp.NumberOfResults,
|
||||||
UnresponsiveEngines: resp.UnresponsiveEngines,
|
UnresponsiveEngines: resp.UnresponsiveEngines,
|
||||||
FaviconService: faviconService,
|
|
||||||
|
|
||||||
// New: categories with icons
|
|
||||||
Categories: []string{"all", "news", "images", "videos", "maps"},
|
|
||||||
DisabledCategories: []string{"shopping", "music", "weather"},
|
|
||||||
CategoryIcons: map[string]string{
|
|
||||||
"all": "🌐",
|
|
||||||
"news": "📰",
|
|
||||||
"images": "🖼️",
|
|
||||||
"videos": "🎬",
|
|
||||||
"maps": "🗺️",
|
|
||||||
"shopping": "🛒",
|
|
||||||
"music": "🎵",
|
|
||||||
"weather": "🌤️",
|
|
||||||
},
|
|
||||||
ActiveCategory: activeCategory,
|
|
||||||
IsImageSearch: activeCategory == "images",
|
|
||||||
|
|
||||||
// Time filters
|
|
||||||
TimeFilters: []FilterOption{
|
|
||||||
{Label: "Any time", Value: ""},
|
|
||||||
{Label: "Past hour", Value: "h"},
|
|
||||||
{Label: "Past 24 hours", Value: "d"},
|
|
||||||
{Label: "Past week", Value: "w"},
|
|
||||||
{Label: "Past month", Value: "m"},
|
|
||||||
{Label: "Past year", Value: "y"},
|
|
||||||
},
|
|
||||||
ActiveTime: activeTime,
|
|
||||||
|
|
||||||
// Type filters
|
|
||||||
TypeFilters: []FilterOption{
|
|
||||||
{Label: "All results", Value: ""},
|
|
||||||
{Label: "News", Value: "news"},
|
|
||||||
{Label: "Videos", Value: "video"},
|
|
||||||
{Label: "Images", Value: "image"},
|
|
||||||
},
|
|
||||||
ActiveType: activeType,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert results.
|
// Convert results.
|
||||||
pd.Results = make([]ResultView, len(resp.Results))
|
pd.Results = make([]ResultView, len(resp.Results))
|
||||||
for i, r := range resp.Results {
|
for i, r := range resp.Results {
|
||||||
tmplName := "result_item"
|
pd.Results[i] = ResultView(r)
|
||||||
if r.Template == "videos" {
|
|
||||||
tmplName = "video_item"
|
|
||||||
}
|
|
||||||
// Sanitize URLs to prevent javascript:/data: scheme injection.
|
|
||||||
var domain string
|
|
||||||
if r.URL != nil {
|
|
||||||
safe := util.SanitizeResultURL(*r.URL)
|
|
||||||
r.URL = &safe
|
|
||||||
if u, err := url.Parse(safe); err == nil {
|
|
||||||
domain = u.Hostname()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
r.Thumbnail = util.SanitizeResultURL(r.Thumbnail)
|
|
||||||
pd.Results[i] = ResultView{
|
|
||||||
MainResult: r,
|
|
||||||
TemplateName: tmplName,
|
|
||||||
Domain: domain,
|
|
||||||
FaviconIconURL: faviconIconURL(faviconService, domain),
|
|
||||||
SafeTitle: template.HTML(html.UnescapeString(r.Title)),
|
|
||||||
SafeContent: template.HTML(html.UnescapeString(r.Content)),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert answers (they're map[string]any — extract string values).
|
// Convert answers (they're map[string]any — extract string values).
|
||||||
|
|
@ -272,12 +127,9 @@ func FromResponse(resp contracts.SearchResponse, query string, pageno int, activ
|
||||||
iv.Title = v
|
iv.Title = v
|
||||||
}
|
}
|
||||||
if v, ok := ib["img_src"].(string); ok {
|
if v, ok := ib["img_src"].(string); ok {
|
||||||
iv.ImgSrc = util.SanitizeResultURL(v)
|
iv.ImgSrc = v
|
||||||
}
|
}
|
||||||
if v, ok := ib["url"].(string); ok {
|
if iv.Title != "" || iv.Content != "" {
|
||||||
iv.URL = util.SanitizeResultURL(v)
|
|
||||||
}
|
|
||||||
if iv.Title != "" || iv.Content != "" || iv.ImgSrc != "" {
|
|
||||||
pd.Infoboxes = append(pd.Infoboxes, iv)
|
pd.Infoboxes = append(pd.Infoboxes, iv)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -309,15 +161,14 @@ func FromResponse(resp contracts.SearchResponse, query string, pageno int, activ
|
||||||
}
|
}
|
||||||
|
|
||||||
// RenderIndex renders the homepage (search box only).
|
// RenderIndex renders the homepage (search box only).
|
||||||
func RenderIndex(w http.ResponseWriter, sourceURL, theme string) error {
|
func RenderIndex(w http.ResponseWriter) error {
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
return tmplIndex.ExecuteTemplate(w, "base", PageData{ShowHeader: true, SourceURL: sourceURL, Theme: theme})
|
return tmplIndex.ExecuteTemplate(w, "base", PageData{})
|
||||||
}
|
}
|
||||||
|
|
||||||
// RenderSearch renders the full search results page (with base layout).
|
// RenderSearch renders the full search results page (with base layout).
|
||||||
func RenderSearch(w http.ResponseWriter, data PageData) error {
|
func RenderSearch(w http.ResponseWriter, data PageData) error {
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
data.ShowHeader = true
|
|
||||||
return tmplFull.ExecuteTemplate(w, "base", data)
|
return tmplFull.ExecuteTemplate(w, "base", data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -354,13 +205,4 @@ func RenderSearchAuto(w http.ResponseWriter, r *http.Request, data PageData) err
|
||||||
return RenderSearch(w, data)
|
return RenderSearch(w, data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// RenderPreferences renders the full preferences page.
|
var _ = strconv.Itoa
|
||||||
func RenderPreferences(w http.ResponseWriter, sourceURL, theme, faviconService string) error {
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
||||||
return tmplPreferences.ExecuteTemplate(w, "base", PageData{
|
|
||||||
ShowHeader: true,
|
|
||||||
SourceURL: sourceURL,
|
|
||||||
Theme: theme,
|
|
||||||
FaviconService: faviconService,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,9 @@
|
||||||
package views
|
package views
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/metamorphosis-dev/samsa/internal/contracts"
|
"github.com/metamorphosis-dev/kafka/internal/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func mockSearchResponse(query string, numResults int) contracts.SearchResponse {
|
func mockSearchResponse(query string, numResults int) contracts.SearchResponse {
|
||||||
|
|
@ -37,11 +36,11 @@ func mockEmptyResponse() contracts.SearchResponse {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFromResponse_Basic(t *testing.T) {
|
func TestFromResponse_Basic(t *testing.T) {
|
||||||
resp := mockSearchResponse("samsa trial", 42)
|
resp := mockSearchResponse("kafka trial", 42)
|
||||||
data := FromResponse(resp, "samsa trial", 1, "", "", "", "none")
|
data := FromResponse(resp, "kafka trial", 1)
|
||||||
|
|
||||||
if data.Query != "samsa trial" {
|
if data.Query != "kafka trial" {
|
||||||
t.Errorf("expected query 'samsa trial', got %q", data.Query)
|
t.Errorf("expected query 'kafka trial', got %q", data.Query)
|
||||||
}
|
}
|
||||||
if data.NumberOfResults != 42 {
|
if data.NumberOfResults != 42 {
|
||||||
t.Errorf("expected 42 results, got %d", data.NumberOfResults)
|
t.Errorf("expected 42 results, got %d", data.NumberOfResults)
|
||||||
|
|
@ -56,7 +55,7 @@ func TestFromResponse_Basic(t *testing.T) {
|
||||||
|
|
||||||
func TestFromResponse_Pagination(t *testing.T) {
|
func TestFromResponse_Pagination(t *testing.T) {
|
||||||
resp := mockSearchResponse("test", 100)
|
resp := mockSearchResponse("test", 100)
|
||||||
data := FromResponse(resp, "test", 3, "", "", "", "none")
|
data := FromResponse(resp, "test", 3)
|
||||||
|
|
||||||
if data.PrevPage != 2 {
|
if data.PrevPage != 2 {
|
||||||
t.Errorf("expected PrevPage 2, got %d", data.PrevPage)
|
t.Errorf("expected PrevPage 2, got %d", data.PrevPage)
|
||||||
|
|
@ -81,7 +80,7 @@ func TestFromResponse_Pagination(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFromResponse_Empty(t *testing.T) {
|
func TestFromResponse_Empty(t *testing.T) {
|
||||||
data := FromResponse(mockEmptyResponse(), "", 1, "", "", "", "none")
|
data := FromResponse(mockEmptyResponse(), "", 1)
|
||||||
|
|
||||||
if data.NumberOfResults != 0 {
|
if data.NumberOfResults != 0 {
|
||||||
t.Errorf("expected 0 results, got %d", data.NumberOfResults)
|
t.Errorf("expected 0 results, got %d", data.NumberOfResults)
|
||||||
|
|
@ -91,31 +90,6 @@ func TestFromResponse_Empty(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFromResponse_FaviconIconURL(t *testing.T) {
|
|
||||||
u := "https://example.com/path"
|
|
||||||
resp := contracts.SearchResponse{
|
|
||||||
Query: "q",
|
|
||||||
NumberOfResults: 1,
|
|
||||||
Results: []contracts.MainResult{{Title: "t", URL: &u, Engine: "bing"}},
|
|
||||||
Answers: []map[string]any{},
|
|
||||||
Corrections: []string{},
|
|
||||||
Infoboxes: []map[string]any{},
|
|
||||||
Suggestions: []string{},
|
|
||||||
UnresponsiveEngines: [][2]string{},
|
|
||||||
}
|
|
||||||
data := FromResponse(resp, "q", 1, "", "", "", "google")
|
|
||||||
if len(data.Results) != 1 {
|
|
||||||
t.Fatalf("expected 1 result, got %d", len(data.Results))
|
|
||||||
}
|
|
||||||
got := data.Results[0].FaviconIconURL
|
|
||||||
if got == "" || !strings.Contains(got, "google.com/s2/favicons") {
|
|
||||||
t.Fatalf("expected google favicon URL, got %q", got)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got, "example.com") {
|
|
||||||
t.Fatalf("expected domain in favicon URL, got %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsHTMXRequest(t *testing.T) {
|
func TestIsHTMXRequest(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue