feat(v3): add Dozzle/Uptime Kuma/Gotify; fix Meilisearch catalogue re-index churn
- docker-compose: add dozzle, uptime-kuma, gotify services with Caddy subdomains (logs/uptime/push.libnovel.cc); bind-mount dozzle/users.yml for simple auth - Caddyfile: add site blocks for logs, uptime, push subdomains - admin layout: add Logs, Uptime, Push tabs to external tools pill - CatalogueEntry: add Slug field; populated by novelfire scraper - meili.Client: add BookExists() to check if a book is already indexed - catalogue_refresh: skip books already present in Meilisearch to prevent redundant re-scraping and re-indexing on every runner restart - docker-compose runner: set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true
This commit is contained in:
@@ -28,6 +28,7 @@ type BookMeta struct {
|
||||
|
||||
// CatalogueEntry is a lightweight book reference returned by catalogue pages.
|
||||
type CatalogueEntry struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
@@ -29,6 +29,9 @@ const indexName = "books"
|
||||
type Client interface {
|
||||
// UpsertBook adds or updates a book document in the search index.
|
||||
UpsertBook(ctx context.Context, book domain.BookMeta) error
|
||||
// BookExists reports whether a book with the given slug is already in the
|
||||
// index. Used by the catalogue refresh to skip re-indexing known books.
|
||||
BookExists(ctx context.Context, slug string) bool
|
||||
// Search returns up to limit books matching query.
|
||||
Search(ctx context.Context, query string, limit int) ([]domain.BookMeta, error)
|
||||
// Catalogue queries books with optional filters, sort, and pagination.
|
||||
@@ -172,6 +175,15 @@ func (c *MeiliClient) UpsertBook(_ context.Context, book domain.BookMeta) error
|
||||
return nil
|
||||
}
|
||||
|
||||
// BookExists reports whether the slug is already present in the index.
|
||||
// It fetches the document by primary key; a 404 or any error is treated as
|
||||
// "not present" (safe default: re-index rather than silently skip).
|
||||
func (c *MeiliClient) BookExists(_ context.Context, slug string) bool {
|
||||
var doc bookDoc
|
||||
err := c.idx.GetDocument(slug, nil, &doc)
|
||||
return err == nil && doc.Slug != ""
|
||||
}
|
||||
|
||||
// Search returns books matching query, up to limit results.
|
||||
func (c *MeiliClient) Search(_ context.Context, query string, limit int) ([]domain.BookMeta, error) {
|
||||
if limit <= 0 {
|
||||
@@ -306,6 +318,7 @@ func sortStrings(s []string) {
|
||||
type NoopClient struct{}
|
||||
|
||||
func (NoopClient) UpsertBook(_ context.Context, _ domain.BookMeta) error { return nil }
|
||||
func (NoopClient) BookExists(_ context.Context, _ string) bool { return false }
|
||||
func (NoopClient) Search(_ context.Context, _ string, _ int) ([]domain.BookMeta, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -111,7 +111,7 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case entries <- domain.CatalogueEntry{Title: title, URL: bookURL}:
|
||||
case entries <- domain.CatalogueEntry{Slug: slugFromURL(bookURL), Title: title, URL: bookURL}:
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,14 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) {
|
||||
break
|
||||
}
|
||||
|
||||
// Skip books already present in Meilisearch — they were indexed on a
|
||||
// previous run. Re-indexing only happens when a scrape task is
|
||||
// explicitly enqueued (e.g. via the admin UI or API).
|
||||
if r.deps.SearchIndex.BookExists(ctx, entry.Slug) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
// Random jitter between books to avoid rate-limiting.
|
||||
jitter := time.Duration(1000+rand.Intn(2000)) * time.Millisecond
|
||||
select {
|
||||
|
||||
Reference in New Issue
Block a user