feat(v3): add Dozzle/Uptime Kuma/Gotify; fix Meilisearch catalogue re-index churn

- docker-compose: add dozzle, uptime-kuma, gotify services with Caddy subdomains
  (logs/uptime/push.libnovel.cc); bind-mount dozzle/users.yml for simple auth
- Caddyfile: add site blocks for logs, uptime, push subdomains
- admin layout: add Logs, Uptime, Push tabs to external tools pill
- CatalogueEntry: add Slug field; populated by novelfire scraper
- meili.Client: add BookExists() to check if a book is already indexed
- catalogue_refresh: skip books already present in Meilisearch to prevent
  redundant re-scraping and re-indexing on every runner restart
- docker-compose runner: set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true
This commit is contained in:
Admin
2026-03-23 16:10:35 +05:00
parent 5b27d501af
commit b9b69cee44
8 changed files with 446 additions and 13 deletions

View File

@@ -28,6 +28,7 @@ type BookMeta struct {
// CatalogueEntry is a lightweight book reference returned by catalogue pages.
type CatalogueEntry struct {
Slug string `json:"slug"`
Title string `json:"title"`
URL string `json:"url"`
}

View File

@@ -29,6 +29,9 @@ const indexName = "books"
type Client interface {
// UpsertBook adds or updates a book document in the search index.
UpsertBook(ctx context.Context, book domain.BookMeta) error
// BookExists reports whether a book with the given slug is already in the
// index. Used by the catalogue refresh to skip re-indexing known books.
BookExists(ctx context.Context, slug string) bool
// Search returns up to limit books matching query.
Search(ctx context.Context, query string, limit int) ([]domain.BookMeta, error)
// Catalogue queries books with optional filters, sort, and pagination.
@@ -172,6 +175,15 @@ func (c *MeiliClient) UpsertBook(_ context.Context, book domain.BookMeta) error
return nil
}
// BookExists reports whether the slug is already present in the index.
// It fetches the document by primary key; a 404 or any error is treated as
// "not present" (safe default: re-index rather than silently skip).
func (c *MeiliClient) BookExists(_ context.Context, slug string) bool {
var doc bookDoc
err := c.idx.GetDocument(slug, nil, &doc)
return err == nil && doc.Slug != ""
}
// Search returns books matching query, up to limit results.
func (c *MeiliClient) Search(_ context.Context, query string, limit int) ([]domain.BookMeta, error) {
if limit <= 0 {
@@ -306,6 +318,7 @@ func sortStrings(s []string) {
type NoopClient struct{}
func (NoopClient) UpsertBook(_ context.Context, _ domain.BookMeta) error { return nil }
func (NoopClient) BookExists(_ context.Context, _ string) bool { return false }
func (NoopClient) Search(_ context.Context, _ string, _ int) ([]domain.BookMeta, error) {
return nil, nil
}

View File

@@ -111,7 +111,7 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
select {
case <-ctx.Done():
return
case entries <- domain.CatalogueEntry{Title: title, URL: bookURL}:
case entries <- domain.CatalogueEntry{Slug: slugFromURL(bookURL), Title: title, URL: bookURL}:
}
}

View File

@@ -51,6 +51,14 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) {
break
}
// Skip books already present in Meilisearch — they were indexed on a
// previous run. Re-indexing only happens when a scrape task is
// explicitly enqueued (e.g. via the admin UI or API).
if r.deps.SearchIndex.BookExists(ctx, entry.Slug) {
skipped++
continue
}
// Random jitter between books to avoid rate-limiting.
jitter := time.Duration(1000+rand.Intn(2000)) * time.Millisecond
select {