feat(v3): add v3 stack — backend rewrite, renamed env vars, docs
- New Go backend binary (backend + runner) replacing old scraper/ - Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose - Rename scraperFetch → backendFetch across all 19 UI server files - Remove SCRAPER_PROXY env var and proxy transport from browser.Config - Add Meilisearch, Valkey, Caddy to docker-compose - Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
This commit is contained in:
Admin
222
v3/backend/internal/orchestrator/orchestrator.go
Normal file
222
v3/backend/internal/orchestrator/orchestrator.go
Normal file
@@ -0,0 +1,222 @@
|
||||
// Package orchestrator coordinates metadata extraction, chapter-list fetching,
|
||||
// and parallel chapter scraping for a single book.
|
||||
//
|
||||
// Design:
|
||||
// - RunBook scrapes one book (metadata + chapter list + chapter texts) end-to-end.
|
||||
// - N worker goroutines pull chapter refs from a shared queue and call ScrapeChapterText.
|
||||
// - The caller (runner poll loop) owns the outer task-claim / finish cycle.
|
||||
// - An optional PostMetadata hook (set in Config) is called after WriteMetadata
|
||||
// succeeds. The runner uses this to upsert books into Meilisearch.
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/libnovel/backend/internal/bookstore"
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/libnovel/backend/internal/scraper"
|
||||
)
|
||||
|
||||
// Config holds tunable parameters for the orchestrator.
|
||||
type Config struct {
|
||||
// Workers is the number of goroutines used to scrape chapters in parallel.
|
||||
// Defaults to runtime.NumCPU() when 0.
|
||||
Workers int
|
||||
// PostMetadata is an optional hook called with the scraped BookMeta after
|
||||
// WriteMetadata succeeds. Errors from the hook are logged but not fatal.
|
||||
// Used by the runner to index books in Meilisearch.
|
||||
PostMetadata func(ctx context.Context, meta domain.BookMeta)
|
||||
}
|
||||
|
||||
// Orchestrator runs a single-book scrape pipeline.
|
||||
type Orchestrator struct {
|
||||
novel scraper.NovelScraper
|
||||
store bookstore.BookWriter
|
||||
log *slog.Logger
|
||||
workers int
|
||||
postMetadata func(ctx context.Context, meta domain.BookMeta)
|
||||
}
|
||||
|
||||
// New returns a new Orchestrator.
|
||||
func New(cfg Config, novel scraper.NovelScraper, store bookstore.BookWriter, log *slog.Logger) *Orchestrator {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
workers := cfg.Workers
|
||||
if workers <= 0 {
|
||||
workers = runtime.NumCPU()
|
||||
}
|
||||
return &Orchestrator{
|
||||
novel: novel,
|
||||
store: store,
|
||||
log: log,
|
||||
workers: workers,
|
||||
postMetadata: cfg.PostMetadata,
|
||||
}
|
||||
}
|
||||
|
||||
// RunBook scrapes a single book described by task. It handles:
|
||||
// 1. Metadata scrape + write
|
||||
// 2. Chapter list scrape + write
|
||||
// 3. Parallel chapter text scrape + write (worker pool)
|
||||
//
|
||||
// Returns a ScrapeResult with counters. The result's ErrorMessage is non-empty
|
||||
// if the run failed at the metadata or chapter-list level.
|
||||
func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) domain.ScrapeResult {
|
||||
o.log.Info("orchestrator: RunBook starting",
|
||||
"task_id", task.ID,
|
||||
"kind", task.Kind,
|
||||
"url", task.TargetURL,
|
||||
"workers", o.workers,
|
||||
)
|
||||
|
||||
var result domain.ScrapeResult
|
||||
|
||||
if task.TargetURL == "" {
|
||||
result.ErrorMessage = "task has no target URL"
|
||||
return result
|
||||
}
|
||||
|
||||
// ── Step 1: Metadata ──────────────────────────────────────────────────────
|
||||
meta, err := o.novel.ScrapeMetadata(ctx, task.TargetURL)
|
||||
if err != nil {
|
||||
o.log.Error("metadata scrape failed", "url", task.TargetURL, "err", err)
|
||||
result.ErrorMessage = fmt.Sprintf("metadata: %v", err)
|
||||
result.Errors++
|
||||
return result
|
||||
}
|
||||
|
||||
if err := o.store.WriteMetadata(ctx, meta); err != nil {
|
||||
o.log.Error("metadata write failed", "slug", meta.Slug, "err", err)
|
||||
// non-fatal: continue to chapters
|
||||
result.Errors++
|
||||
} else {
|
||||
result.BooksFound = 1
|
||||
// Fire optional post-metadata hook (e.g. Meilisearch indexing).
|
||||
if o.postMetadata != nil {
|
||||
o.postMetadata(ctx, meta)
|
||||
}
|
||||
}
|
||||
|
||||
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
|
||||
|
||||
// ── Step 2: Chapter list ──────────────────────────────────────────────────
|
||||
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL)
|
||||
if err != nil {
|
||||
o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err)
|
||||
result.ErrorMessage = fmt.Sprintf("chapter list: %v", err)
|
||||
result.Errors++
|
||||
return result
|
||||
}
|
||||
|
||||
o.log.Info("chapter list fetched", "slug", meta.Slug, "chapters", len(refs))
|
||||
|
||||
// Persist chapter refs (without text) so the index exists early.
|
||||
if wErr := o.store.WriteChapterRefs(ctx, meta.Slug, refs); wErr != nil {
|
||||
o.log.Warn("chapter refs write failed", "slug", meta.Slug, "err", wErr)
|
||||
}
|
||||
|
||||
// ── Step 3: Chapter texts (worker pool) ───────────────────────────────────
|
||||
type chapterJob struct {
|
||||
slug string
|
||||
ref domain.ChapterRef
|
||||
total int // total chapters to scrape (for progress logging)
|
||||
}
|
||||
work := make(chan chapterJob, o.workers*4)
|
||||
|
||||
var scraped, skipped, errors atomic.Int64
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 0; i < o.workers; i++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
for job := range work {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
if o.store.ChapterExists(ctx, job.slug, job.ref) {
|
||||
o.log.Debug("chapter already exists, skipping",
|
||||
"slug", job.slug, "chapter", job.ref.Number)
|
||||
skipped.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
ch, err := o.novel.ScrapeChapterText(ctx, job.ref)
|
||||
if err != nil {
|
||||
o.log.Error("chapter scrape failed",
|
||||
"slug", job.slug, "chapter", job.ref.Number, "err", err)
|
||||
errors.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := o.store.WriteChapter(ctx, job.slug, ch); err != nil {
|
||||
o.log.Error("chapter write failed",
|
||||
"slug", job.slug, "chapter", job.ref.Number, "err", err)
|
||||
errors.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
n := scraped.Add(1)
|
||||
// Log a progress summary every 25 chapters scraped.
|
||||
if n%25 == 0 {
|
||||
o.log.Info("scraping chapters",
|
||||
"slug", job.slug, "scraped", n, "total", job.total)
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
// Count how many chapters will actually be enqueued (for progress logging).
|
||||
toScrape := 0
|
||||
for _, ref := range refs {
|
||||
if task.FromChapter > 0 && ref.Number < task.FromChapter {
|
||||
continue
|
||||
}
|
||||
if task.ToChapter > 0 && ref.Number > task.ToChapter {
|
||||
continue
|
||||
}
|
||||
toScrape++
|
||||
}
|
||||
|
||||
// Enqueue chapter jobs respecting the optional range filter from the task.
|
||||
for _, ref := range refs {
|
||||
if task.FromChapter > 0 && ref.Number < task.FromChapter {
|
||||
skipped.Add(1)
|
||||
continue
|
||||
}
|
||||
if task.ToChapter > 0 && ref.Number > task.ToChapter {
|
||||
skipped.Add(1)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
goto drain
|
||||
case work <- chapterJob{slug: meta.Slug, ref: ref, total: toScrape}:
|
||||
}
|
||||
}
|
||||
|
||||
drain:
|
||||
close(work)
|
||||
wg.Wait()
|
||||
|
||||
result.ChaptersScraped = int(scraped.Load())
|
||||
result.ChaptersSkipped = int(skipped.Load())
|
||||
result.Errors += int(errors.Load())
|
||||
|
||||
o.log.Info("book scrape finished",
|
||||
"slug", meta.Slug,
|
||||
"scraped", result.ChaptersScraped,
|
||||
"skipped", result.ChaptersSkipped,
|
||||
"errors", result.Errors,
|
||||
)
|
||||
return result
|
||||
}
|
||||
210
v3/backend/internal/orchestrator/orchestrator_test.go
Normal file
210
v3/backend/internal/orchestrator/orchestrator_test.go
Normal file
@@ -0,0 +1,210 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
)
|
||||
|
||||
// ── stubs ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
type stubScraper struct {
|
||||
meta domain.BookMeta
|
||||
metaErr error
|
||||
refs []domain.ChapterRef
|
||||
refsErr error
|
||||
chapters map[int]domain.Chapter
|
||||
chapErr map[int]error
|
||||
}
|
||||
|
||||
func (s *stubScraper) SourceName() string { return "stub" }
|
||||
|
||||
func (s *stubScraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
|
||||
ch := make(chan domain.CatalogueEntry)
|
||||
errs := make(chan error)
|
||||
close(ch)
|
||||
close(errs)
|
||||
return ch, errs
|
||||
}
|
||||
|
||||
func (s *stubScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) {
|
||||
return s.meta, s.metaErr
|
||||
}
|
||||
|
||||
func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
|
||||
return s.refs, s.refsErr
|
||||
}
|
||||
|
||||
func (s *stubScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
|
||||
if s.chapErr != nil {
|
||||
if err, ok := s.chapErr[ref.Number]; ok {
|
||||
return domain.Chapter{}, err
|
||||
}
|
||||
}
|
||||
if s.chapters != nil {
|
||||
if ch, ok := s.chapters[ref.Number]; ok {
|
||||
return ch, nil
|
||||
}
|
||||
}
|
||||
return domain.Chapter{Ref: ref, Text: "text"}, nil
|
||||
}
|
||||
|
||||
func (s *stubScraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) {
|
||||
ch := make(chan domain.BookMeta)
|
||||
errs := make(chan error)
|
||||
close(ch)
|
||||
close(errs)
|
||||
return ch, errs
|
||||
}
|
||||
|
||||
type stubStore struct {
|
||||
mu sync.Mutex
|
||||
metaWritten []domain.BookMeta
|
||||
chaptersWritten []domain.Chapter
|
||||
existing map[string]bool // "slug:N" → exists
|
||||
writeMetaErr error
|
||||
}
|
||||
|
||||
func (s *stubStore) WriteMetadata(_ context.Context, meta domain.BookMeta) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.writeMetaErr != nil {
|
||||
return s.writeMetaErr
|
||||
}
|
||||
s.metaWritten = append(s.metaWritten, meta)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubStore) WriteChapter(_ context.Context, slug string, ch domain.Chapter) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.chaptersWritten = append(s.chaptersWritten, ch)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubStore) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubStore) ChapterExists(_ context.Context, slug string, ref domain.ChapterRef) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
key := slug + ":" + string(rune('0'+ref.Number))
|
||||
return s.existing[key]
|
||||
}
|
||||
|
||||
// ── tests ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestRunBook_HappyPath(t *testing.T) {
|
||||
sc := &stubScraper{
|
||||
meta: domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"},
|
||||
refs: []domain.ChapterRef{
|
||||
{Number: 1, Title: "Ch 1", URL: "https://example.com/book/test-book/chapter-1"},
|
||||
{Number: 2, Title: "Ch 2", URL: "https://example.com/book/test-book/chapter-2"},
|
||||
{Number: 3, Title: "Ch 3", URL: "https://example.com/book/test-book/chapter-3"},
|
||||
},
|
||||
}
|
||||
st := &stubStore{}
|
||||
o := New(Config{Workers: 2}, sc, st, nil)
|
||||
|
||||
task := domain.ScrapeTask{
|
||||
ID: "t1",
|
||||
Kind: "book",
|
||||
TargetURL: "https://example.com/book/test-book",
|
||||
}
|
||||
|
||||
result := o.RunBook(context.Background(), task)
|
||||
|
||||
if result.ErrorMessage != "" {
|
||||
t.Fatalf("unexpected error: %s", result.ErrorMessage)
|
||||
}
|
||||
if result.BooksFound != 1 {
|
||||
t.Errorf("BooksFound = %d, want 1", result.BooksFound)
|
||||
}
|
||||
if result.ChaptersScraped != 3 {
|
||||
t.Errorf("ChaptersScraped = %d, want 3", result.ChaptersScraped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBook_MetadataError(t *testing.T) {
|
||||
sc := &stubScraper{metaErr: errors.New("404 not found")}
|
||||
st := &stubStore{}
|
||||
o := New(Config{Workers: 1}, sc, st, nil)
|
||||
|
||||
result := o.RunBook(context.Background(), domain.ScrapeTask{
|
||||
ID: "t2",
|
||||
TargetURL: "https://example.com/book/missing",
|
||||
})
|
||||
|
||||
if result.ErrorMessage == "" {
|
||||
t.Fatal("expected ErrorMessage to be set")
|
||||
}
|
||||
if result.Errors != 1 {
|
||||
t.Errorf("Errors = %d, want 1", result.Errors)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBook_ChapterRange(t *testing.T) {
|
||||
sc := &stubScraper{
|
||||
meta: domain.BookMeta{Slug: "range-book", SourceURL: "https://example.com/book/range-book"},
|
||||
refs: func() []domain.ChapterRef {
|
||||
var refs []domain.ChapterRef
|
||||
for i := 1; i <= 10; i++ {
|
||||
refs = append(refs, domain.ChapterRef{Number: i, URL: "https://example.com/book/range-book/chapter-" + string(rune('0'+i))})
|
||||
}
|
||||
return refs
|
||||
}(),
|
||||
}
|
||||
st := &stubStore{}
|
||||
o := New(Config{Workers: 2}, sc, st, nil)
|
||||
|
||||
result := o.RunBook(context.Background(), domain.ScrapeTask{
|
||||
ID: "t3",
|
||||
TargetURL: "https://example.com/book/range-book",
|
||||
FromChapter: 3,
|
||||
ToChapter: 7,
|
||||
})
|
||||
|
||||
if result.ErrorMessage != "" {
|
||||
t.Fatalf("unexpected error: %s", result.ErrorMessage)
|
||||
}
|
||||
// chapters 3–7 = 5 scraped, chapters 1-2 and 8-10 = 5 skipped
|
||||
if result.ChaptersScraped != 5 {
|
||||
t.Errorf("ChaptersScraped = %d, want 5", result.ChaptersScraped)
|
||||
}
|
||||
if result.ChaptersSkipped != 5 {
|
||||
t.Errorf("ChaptersSkipped = %d, want 5", result.ChaptersSkipped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBook_ContextCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
sc := &stubScraper{
|
||||
meta: domain.BookMeta{Slug: "ctx-book", SourceURL: "https://example.com/book/ctx-book"},
|
||||
refs: []domain.ChapterRef{
|
||||
{Number: 1, URL: "https://example.com/book/ctx-book/chapter-1"},
|
||||
},
|
||||
}
|
||||
st := &stubStore{}
|
||||
o := New(Config{Workers: 1}, sc, st, nil)
|
||||
|
||||
// Should not panic; result may have errors or zero chapters.
|
||||
result := o.RunBook(ctx, domain.ScrapeTask{
|
||||
ID: "t4",
|
||||
TargetURL: "https://example.com/book/ctx-book",
|
||||
})
|
||||
_ = result
|
||||
}
|
||||
|
||||
func TestRunBook_EmptyTargetURL(t *testing.T) {
|
||||
o := New(Config{Workers: 1}, &stubScraper{}, &stubStore{}, nil)
|
||||
result := o.RunBook(context.Background(), domain.ScrapeTask{ID: "t5"})
|
||||
if result.ErrorMessage == "" {
|
||||
t.Fatal("expected ErrorMessage for empty target URL")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user