chore: migrate to v3, Doppler secrets, clean up legacy code
Some checks failed
CI / v3 / Check ui (pull_request) Failing after 15s
CI / v3 / Test backend (pull_request) Failing after 16s
CI / v3 / Docker / backend (pull_request) Has been skipped
CI / v3 / Docker / runner (pull_request) Has been skipped
CI / v3 / Docker / ui (pull_request) Has been skipped

- Remove all pre-v3 code: scraper, ui-v2, backend v1, ios v1+v2, legacy CI workflows
- Flatten v3/ contents to repo root
- Add Doppler secrets management (project=libnovel, config=prd)
- Add justfile with doppler run wrappers for all docker compose commands
- Strip hardcoded env fallbacks from docker-compose.yml
- Add minimal README.md
- Clean up .gitignore
This commit is contained in:
Admin
2026-03-23 17:21:12 +05:00
parent 1118392811
commit 59e8cdb19a
522 changed files with 5259 additions and 80365 deletions

View File

@@ -5,6 +5,8 @@
// - RunBook scrapes one book (metadata + chapter list + chapter texts) end-to-end.
// - N worker goroutines pull chapter refs from a shared queue and call ScrapeChapterText.
// - The caller (runner poll loop) owns the outer task-claim / finish cycle.
// - An optional PostMetadata hook (set in Config) is called after WriteMetadata
// succeeds. The runner uses this to upsert books into Meilisearch.
package orchestrator
import (
@@ -25,14 +27,19 @@ type Config struct {
// Workers is the number of goroutines used to scrape chapters in parallel.
// Defaults to runtime.NumCPU() when 0.
Workers int
// PostMetadata is an optional hook called with the scraped BookMeta after
// WriteMetadata succeeds. Errors from the hook are logged but not fatal.
// Used by the runner to index books in Meilisearch.
PostMetadata func(ctx context.Context, meta domain.BookMeta)
}
// Orchestrator runs a single-book scrape pipeline.
type Orchestrator struct {
novel scraper.NovelScraper
store bookstore.BookWriter
log *slog.Logger
workers int
novel scraper.NovelScraper
store bookstore.BookWriter
log *slog.Logger
workers int
postMetadata func(ctx context.Context, meta domain.BookMeta)
}
// New returns a new Orchestrator.
@@ -44,7 +51,13 @@ func New(cfg Config, novel scraper.NovelScraper, store bookstore.BookWriter, log
if workers <= 0 {
workers = runtime.NumCPU()
}
return &Orchestrator{novel: novel, store: store, log: log, workers: workers}
return &Orchestrator{
novel: novel,
store: store,
log: log,
workers: workers,
postMetadata: cfg.PostMetadata,
}
}
// RunBook scrapes a single book described by task. It handles:
@@ -84,12 +97,16 @@ func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) doma
result.Errors++
} else {
result.BooksFound = 1
// Fire optional post-metadata hook (e.g. Meilisearch indexing).
if o.postMetadata != nil {
o.postMetadata(ctx, meta)
}
}
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
// ── Step 2: Chapter list ──────────────────────────────────────────────────
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL)
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL, task.ToChapter)
if err != nil {
o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err)
result.ErrorMessage = fmt.Sprintf("chapter list: %v", err)

View File

@@ -34,7 +34,7 @@ func (s *stubScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMe
return s.meta, s.metaErr
}
func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) {
return s.refs, s.refsErr
}