feat(v3): add v3 stack — backend rewrite, renamed env vars, docs
- New Go backend binary (backend + runner) replacing old scraper/ - Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose - Rename scraperFetch → backendFetch across all 19 UI server files - Remove SCRAPER_PROXY env var and proxy transport from browser.Config - Add Meilisearch, Valkey, Caddy to docker-compose - Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
This commit is contained in:
Admin
257
v3/backend/internal/meili/client.go
Normal file
257
v3/backend/internal/meili/client.go
Normal file
@@ -0,0 +1,257 @@
|
||||
// Package meili provides a thin Meilisearch client for indexing and searching
|
||||
// locally scraped books.
|
||||
//
|
||||
// Index:
|
||||
// - Name: "books"
|
||||
// - Primary key: "slug"
|
||||
// - Searchable attributes: title, author, genres, summary
|
||||
// - Filterable attributes: status, genres
|
||||
// - Sortable attributes: rank, rating, total_chapters
|
||||
//
|
||||
// The client is intentionally simple: UpsertBook and Search only. All
|
||||
// Meilisearch-specific details (index management, attribute configuration)
|
||||
// are handled once in Configure(), called at startup.
|
||||
package meili
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/meilisearch/meilisearch-go"
|
||||
)
|
||||
|
||||
const indexName = "books"
|
||||
|
||||
// Client is the interface for Meilisearch operations used by runner and backend.
|
||||
type Client interface {
|
||||
// UpsertBook adds or updates a book document in the search index.
|
||||
UpsertBook(ctx context.Context, book domain.BookMeta) error
|
||||
// Search returns up to limit books matching query.
|
||||
Search(ctx context.Context, query string, limit int) ([]domain.BookMeta, error)
|
||||
// Catalogue queries books with optional filters, sort, and pagination.
|
||||
// Returns books and the total hit count for pagination.
|
||||
Catalogue(ctx context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, error)
|
||||
}
|
||||
|
||||
// CatalogueQuery holds parameters for the /api/catalogue endpoint.
|
||||
type CatalogueQuery struct {
|
||||
Q string // full-text query (may be empty for browse)
|
||||
Genre string // genre filter, e.g. "fantasy" or "all"
|
||||
Status string // status filter, e.g. "ongoing", "completed", or "all"
|
||||
Sort string // sort field: "popular", "new", "top-rated", "rank", ""
|
||||
Page int // 1-indexed
|
||||
Limit int // items per page, default 20
|
||||
}
|
||||
|
||||
// MeiliClient wraps the meilisearch-go SDK.
|
||||
type MeiliClient struct {
|
||||
idx meilisearch.IndexManager
|
||||
}
|
||||
|
||||
// New creates a MeiliClient. Call Configure() once at startup to ensure the
|
||||
// index exists and has the correct attribute settings.
|
||||
func New(host, apiKey string) *MeiliClient {
|
||||
cli := meilisearch.New(host, meilisearch.WithAPIKey(apiKey))
|
||||
return &MeiliClient{idx: cli.Index(indexName)}
|
||||
}
|
||||
|
||||
// Configure creates the index if absent and sets searchable/filterable
|
||||
// attributes. It is idempotent — safe to call on every startup.
|
||||
func Configure(host, apiKey string) error {
|
||||
cli := meilisearch.New(host, meilisearch.WithAPIKey(apiKey))
|
||||
|
||||
// Create index with primary key. Returns 202 if exists — ignore.
|
||||
task, err := cli.CreateIndex(&meilisearch.IndexConfig{
|
||||
Uid: indexName,
|
||||
PrimaryKey: "slug",
|
||||
})
|
||||
if err != nil {
|
||||
// 400 "index_already_exists" is not an error here; the SDK returns
|
||||
// an error with Code "index_already_exists" which we can ignore.
|
||||
// Any other error is fatal.
|
||||
if apiErr, ok := err.(*meilisearch.Error); ok && apiErr.MeilisearchApiError.Code == "index_already_exists" {
|
||||
// already exists — continue
|
||||
} else {
|
||||
return fmt.Errorf("meili: create index: %w", err)
|
||||
}
|
||||
} else {
|
||||
_ = task // task is async; we don't wait for it
|
||||
}
|
||||
|
||||
idx := cli.Index(indexName)
|
||||
|
||||
searchable := []string{"title", "author", "genres", "summary"}
|
||||
if _, err := idx.UpdateSearchableAttributes(&searchable); err != nil {
|
||||
return fmt.Errorf("meili: update searchable attributes: %w", err)
|
||||
}
|
||||
|
||||
filterable := []interface{}{"status", "genres"}
|
||||
if _, err := idx.UpdateFilterableAttributes(&filterable); err != nil {
|
||||
return fmt.Errorf("meili: update filterable attributes: %w", err)
|
||||
}
|
||||
|
||||
sortable := []string{"rank", "rating", "total_chapters"}
|
||||
if _, err := idx.UpdateSortableAttributes(&sortable); err != nil {
|
||||
return fmt.Errorf("meili: update sortable attributes: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// bookDoc is the Meilisearch document shape for a book.
|
||||
type bookDoc struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Cover string `json:"cover"`
|
||||
Status string `json:"status"`
|
||||
Genres []string `json:"genres"`
|
||||
Summary string `json:"summary"`
|
||||
TotalChapters int `json:"total_chapters"`
|
||||
SourceURL string `json:"source_url"`
|
||||
Rank int `json:"rank"`
|
||||
Rating float64 `json:"rating"`
|
||||
}
|
||||
|
||||
func toDoc(b domain.BookMeta) bookDoc {
|
||||
return bookDoc{
|
||||
Slug: b.Slug,
|
||||
Title: b.Title,
|
||||
Author: b.Author,
|
||||
Cover: b.Cover,
|
||||
Status: b.Status,
|
||||
Genres: b.Genres,
|
||||
Summary: b.Summary,
|
||||
TotalChapters: b.TotalChapters,
|
||||
SourceURL: b.SourceURL,
|
||||
Rank: b.Ranking,
|
||||
Rating: b.Rating,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDoc(d bookDoc) domain.BookMeta {
|
||||
return domain.BookMeta{
|
||||
Slug: d.Slug,
|
||||
Title: d.Title,
|
||||
Author: d.Author,
|
||||
Cover: d.Cover,
|
||||
Status: d.Status,
|
||||
Genres: d.Genres,
|
||||
Summary: d.Summary,
|
||||
TotalChapters: d.TotalChapters,
|
||||
SourceURL: d.SourceURL,
|
||||
Ranking: d.Rank,
|
||||
Rating: d.Rating,
|
||||
}
|
||||
}
|
||||
|
||||
// UpsertBook adds or replaces the book document in Meilisearch. The operation
|
||||
// is fire-and-forget (Meilisearch processes tasks asynchronously).
|
||||
func (c *MeiliClient) UpsertBook(_ context.Context, book domain.BookMeta) error {
|
||||
docs := []bookDoc{toDoc(book)}
|
||||
pk := "slug"
|
||||
if _, err := c.idx.AddDocuments(docs, &meilisearch.DocumentOptions{PrimaryKey: &pk}); err != nil {
|
||||
return fmt.Errorf("meili: upsert book %q: %w", book.Slug, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Search returns books matching query, up to limit results.
|
||||
func (c *MeiliClient) Search(_ context.Context, query string, limit int) ([]domain.BookMeta, error) {
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
res, err := c.idx.Search(query, &meilisearch.SearchRequest{
|
||||
Limit: int64(limit),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("meili: search %q: %w", query, err)
|
||||
}
|
||||
|
||||
books := make([]domain.BookMeta, 0, len(res.Hits))
|
||||
for _, hit := range res.Hits {
|
||||
// Hit is map[string]json.RawMessage — unmarshal directly into bookDoc.
|
||||
var doc bookDoc
|
||||
raw, err := json.Marshal(hit)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
continue
|
||||
}
|
||||
books = append(books, fromDoc(doc))
|
||||
}
|
||||
return books, nil
|
||||
}
|
||||
|
||||
// Catalogue queries books with optional full-text search, genre/status filters,
|
||||
// sort order, and pagination. Returns matching books and the total estimate.
|
||||
func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, error) {
|
||||
if q.Limit <= 0 {
|
||||
q.Limit = 20
|
||||
}
|
||||
if q.Page <= 0 {
|
||||
q.Page = 1
|
||||
}
|
||||
|
||||
req := &meilisearch.SearchRequest{
|
||||
Limit: int64(q.Limit),
|
||||
Offset: int64((q.Page - 1) * q.Limit),
|
||||
}
|
||||
|
||||
// Build filter
|
||||
var filters []string
|
||||
if q.Genre != "" && q.Genre != "all" {
|
||||
filters = append(filters, fmt.Sprintf("genres = %q", q.Genre))
|
||||
}
|
||||
if q.Status != "" && q.Status != "all" {
|
||||
filters = append(filters, fmt.Sprintf("status = %q", q.Status))
|
||||
}
|
||||
if len(filters) > 0 {
|
||||
req.Filter = strings.Join(filters, " AND ")
|
||||
}
|
||||
|
||||
// Map UI sort tokens to Meilisearch sort expressions
|
||||
switch q.Sort {
|
||||
case "rank":
|
||||
req.Sort = []string{"rank:asc"}
|
||||
case "top-rated":
|
||||
req.Sort = []string{"rating:desc"}
|
||||
case "new":
|
||||
req.Sort = []string{"total_chapters:desc"}
|
||||
// "popular" and "" → relevance (no explicit sort)
|
||||
}
|
||||
|
||||
res, err := c.idx.Search(q.Q, req)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("meili: catalogue query: %w", err)
|
||||
}
|
||||
|
||||
books := make([]domain.BookMeta, 0, len(res.Hits))
|
||||
for _, hit := range res.Hits {
|
||||
var doc bookDoc
|
||||
raw, err := json.Marshal(hit)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
continue
|
||||
}
|
||||
books = append(books, fromDoc(doc))
|
||||
}
|
||||
return books, res.EstimatedTotalHits, nil
|
||||
}
|
||||
|
||||
// NoopClient is a no-op Client used when Meilisearch is not configured.
|
||||
type NoopClient struct{}
|
||||
|
||||
func (NoopClient) UpsertBook(_ context.Context, _ domain.BookMeta) error { return nil }
|
||||
func (NoopClient) Search(_ context.Context, _ string, _ int) ([]domain.BookMeta, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (NoopClient) Catalogue(_ context.Context, _ CatalogueQuery) ([]domain.BookMeta, int64, error) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
Reference in New Issue
Block a user