libnovel/scraper/internal/orchestrator/orchestrator_test.go

package orchestrator

import (
	"context"
	"sync"
	"testing"
	"time"

	"github.com/libnovel/scraper/internal/scraper"
	"github.com/libnovel/scraper/internal/storage"
	"io"
	"log/slog"
)

// ── mock NovelScraper ─────────────────────────────────────────────────────────

type mockScraper struct {
	catalogue     []scraper.CatalogueEntry
	meta          scraper.BookMeta
	metaErr       error
	chapters      []scraper.ChapterRef
	chapterTextFn func(ref scraper.ChapterRef) (scraper.Chapter, error)
}

func (m *mockScraper) SourceName() string { return "mock" }

func (m *mockScraper) ScrapeCatalogue(_ context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
	entries := make(chan scraper.CatalogueEntry, len(m.catalogue))
	errs := make(chan error, 1)
	for _, e := range m.catalogue {
		entries <- e
	}
	close(entries)
	close(errs)
	return entries, errs
}

func (m *mockScraper) ScrapeMetadata(_ context.Context, _ string) (scraper.BookMeta, error) {
	return m.meta, m.metaErr
}

func (m *mockScraper) ScrapeChapterList(_ context.Context, _ string) ([]scraper.ChapterRef, error) {
	return m.chapters, nil
}

func (m *mockScraper) ScrapeChapterText(_ context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
	if m.chapterTextFn != nil {
		return m.chapterTextFn(ref)
	}
	return scraper.Chapter{Ref: ref, Text: "stub text"}, nil
}

func (m *mockScraper) ScrapeRanking(_ context.Context, _ int) (<-chan scraper.BookMeta, <-chan error) {
	ch := make(chan scraper.BookMeta)
	errs := make(chan error)
	close(ch)
	close(errs)
	return ch, errs
}

// ── mock Store ────────────────────────────────────────────────────────────────

// mockStore records which methods were called; only implements what the
// orchestrator touches.  All other methods panic so unexpected calls surface
// as test failures rather than silent no-ops.
type mockStore struct {
	mu              sync.Mutex
	writtenMeta     []scraper.BookMeta
	writtenChapters []scraper.Chapter
	existingSlugs   map[string]map[int]bool // slug → chapterNum → exists
}

func newMockStore() *mockStore {
	return &mockStore{existingSlugs: make(map[string]map[int]bool)}
}

func (s *mockStore) ChapterExists(_ context.Context, slug string, ref scraper.ChapterRef) bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	if m, ok := s.existingSlugs[slug]; ok {
		return m[ref.Number]
	}
	return false
}

func (s *mockStore) WriteChapter(_ context.Context, slug string, ch scraper.Chapter) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.writtenChapters = append(s.writtenChapters, ch)
	return nil
}

func (s *mockStore) WriteChapterRefs(_ context.Context, _ string, _ []scraper.ChapterRef) error {
	return nil
}

func (s *mockStore) WriteMetadata(_ context.Context, meta scraper.BookMeta) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.writtenMeta = append(s.writtenMeta, meta)
	return nil
}

// Unimplemented Store methods — panic so accidental calls surface immediately.
func (s *mockStore) ReadMetadata(_ context.Context, _ string) (scraper.BookMeta, bool, error) {
	panic("ReadMetadata not expected")
}
func (s *mockStore) ListBooks(_ context.Context) ([]scraper.BookMeta, error) {
	panic("ListBooks not expected")
}
func (s *mockStore) LocalSlugs(_ context.Context) (map[string]bool, error) {
	panic("LocalSlugs not expected")
}
func (s *mockStore) MetadataMtime(_ context.Context, _ string) int64 { return 0 }
func (s *mockStore) ReadChapter(_ context.Context, _ string, _ int) (string, error) {
	panic("ReadChapter not expected")
}
func (s *mockStore) ListChapters(_ context.Context, _ string) ([]storage.ChapterInfo, error) {
	panic("ListChapters not expected")
}
func (s *mockStore) CountChapters(_ context.Context, _ string) int { return 0 }
func (s *mockStore) ReindexChapters(_ context.Context, _ string) (int, error) {
	panic("ReindexChapters not expected")
}
func (s *mockStore) WriteRankingItem(_ context.Context, _ storage.RankingItem) error { return nil }
func (s *mockStore) ReadRankingItems(_ context.Context) ([]storage.RankingItem, error) {
	return nil, nil
}
func (s *mockStore) RankingFreshEnough(_ context.Context, _ time.Duration) (bool, error) {
	return false, nil
}
func (s *mockStore) GetAudioCache(_ context.Context, _ string) (string, bool) { return "", false }
func (s *mockStore) SetAudioCache(_ context.Context, _, _ string) error       { return nil }
func (s *mockStore) PutAudio(_ context.Context, _ string, _ []byte) error     { return nil }
func (s *mockStore) GetProgress(_ context.Context, _, _ string) (storage.ReadingProgress, bool) {
	return storage.ReadingProgress{}, false
}
func (s *mockStore) SetProgress(_ context.Context, _ string, _ storage.ReadingProgress) error {
	return nil
}
func (s *mockStore) AllProgress(_ context.Context, _ string) ([]storage.ReadingProgress, error) {
	return nil, nil
}
func (s *mockStore) DeleteProgress(_ context.Context, _, _ string) error { return nil }
func (s *mockStore) AudioObjectKey(_ string, _ int, _ string) string     { return "" }

func (s *mockStore) AudioExists(_ context.Context, _ string) bool { return false }
func (s *mockStore) PresignChapter(_ context.Context, _ string, _ int, _ time.Duration) (string, error) {
	return "", nil
}
func (s *mockStore) PresignAudio(_ context.Context, _ string, _ time.Duration) (string, error) {
	return "", nil
}
func (s *mockStore) SaveBrowsePage(_ context.Context, _, _ string) error { return nil }
func (s *mockStore) GetBrowsePage(_ context.Context, _ string) (string, bool, error) {
	return "", false, nil
}
func (s *mockStore) BrowseHTMLKey(_ string, _ int) string                         { return "" }
func (s *mockStore) BrowseFilteredHTMLKey(_ string, _ int, _, _, _ string) string { return "" }
func (s *mockStore) BrowseCoverKey(_, _ string) string                            { return "" }
func (s *mockStore) SaveBrowseAsset(_ context.Context, _ string, _ []byte, _ string) error {
	return nil
}
func (s *mockStore) GetBrowseAsset(_ context.Context, _ string) ([]byte, string, bool, error) {
	return nil, "", false, nil
}
func (s *mockStore) CreateScrapeTask(_ context.Context, _, _ string) (string, error) {
	return "task-id", nil
}
func (s *mockStore) UpdateScrapeTask(_ context.Context, _ string, _ storage.ScrapeTaskUpdate) error {
	return nil
}
func (s *mockStore) ListScrapeTasks(_ context.Context) ([]storage.ScrapeTask, error) {
	return nil, nil
}

// ── helpers ───────────────────────────────────────────────────────────────────

func discardLogger() *slog.Logger {
	return slog.New(slog.NewTextHandler(io.Discard, nil))
}

// ── tests ─────────────────────────────────────────────────────────────────────

// TestRun_SingleBook verifies the happy-path single-book scrape: metadata is
// persisted and all chapters are written to the store.
func TestRun_SingleBook(t *testing.T) {
	novel := &mockScraper{
		meta: scraper.BookMeta{Slug: "the-iron-throne", Title: "The Iron Throne"},
		chapters: []scraper.ChapterRef{
			{Number: 1, Title: "Chapter 1", URL: "https://example.com/book/ch-1"},
			{Number: 2, Title: "Chapter 2", URL: "https://example.com/book/ch-2"},
			{Number: 3, Title: "Chapter 3", URL: "https://example.com/book/ch-3"},
		},
	}
	store := newMockStore()

	o := New(Config{Workers: 2, SingleBookURL: "https://example.com/book/the-iron-throne"}, novel, discardLogger(), store)
	if err := o.Run(context.Background()); err != nil {
		t.Fatalf("Run() returned error: %v", err)
	}

	store.mu.Lock()
	defer store.mu.Unlock()

	if len(store.writtenMeta) != 1 {
		t.Errorf("writtenMeta count = %d, want 1", len(store.writtenMeta))
	}
	if len(store.writtenChapters) != 3 {
		t.Errorf("writtenChapters count = %d, want 3", len(store.writtenChapters))
	}
}

// TestRun_SingleBook_SkipsExistingChapters verifies that chapters already in
// the store are not re-scraped.
func TestRun_SingleBook_SkipsExistingChapters(t *testing.T) {
	novel := &mockScraper{
		meta: scraper.BookMeta{Slug: "test-novel", Title: "Test Novel"},
		chapters: []scraper.ChapterRef{
			{Number: 1, Title: "Chapter 1"},
			{Number: 2, Title: "Chapter 2"},
		},
	}
	store := newMockStore()
	// Mark chapter 1 as already existing.
	store.existingSlugs["test-novel"] = map[int]bool{1: true}

	o := New(Config{Workers: 1, SingleBookURL: "https://example.com/book/test-novel"}, novel, discardLogger(), store)
	if err := o.Run(context.Background()); err != nil {
		t.Fatalf("Run() returned error: %v", err)
	}

	store.mu.Lock()
	defer store.mu.Unlock()

	// Only chapter 2 should have been written; chapter 1 was skipped.
	if len(store.writtenChapters) != 1 {
		t.Errorf("writtenChapters count = %d, want 1 (skipped ch1)", len(store.writtenChapters))
	}
	if store.writtenChapters[0].Ref.Number != 2 {
		t.Errorf("expected chapter 2 to be written, got chapter %d", store.writtenChapters[0].Ref.Number)
	}
}

// TestRun_CatalogueMode verifies that catalogue mode processes all books.
func TestRun_CatalogueMode(t *testing.T) {
	novel := &mockScraper{
		catalogue: []scraper.CatalogueEntry{
			{Title: "Book A", URL: "https://example.com/book/a"},
			{Title: "Book B", URL: "https://example.com/book/b"},
		},
		meta:     scraper.BookMeta{Slug: "book-slug", Title: "A Book"},
		chapters: []scraper.ChapterRef{{Number: 1, Title: "Chapter 1"}},
	}
	store := newMockStore()

	o := New(Config{Workers: 2}, novel, discardLogger(), store)
	if err := o.Run(context.Background()); err != nil {
		t.Fatalf("Run() returned error: %v", err)
	}

	store.mu.Lock()
	defer store.mu.Unlock()

	// 2 books → 2 metadata writes, 2 chapter writes (one chapter per book).
	if len(store.writtenMeta) != 2 {
		t.Errorf("writtenMeta count = %d, want 2", len(store.writtenMeta))
	}
	if len(store.writtenChapters) != 2 {
		t.Errorf("writtenChapters count = %d, want 2", len(store.writtenChapters))
	}
}

// TestRun_OnProgress_Called verifies that the OnProgress callback fires at
// least once upon completion.
func TestRun_OnProgress_Called(t *testing.T) {
	novel := &mockScraper{
		meta:     scraper.BookMeta{Slug: "progress-book", Title: "Progress Book"},
		chapters: []scraper.ChapterRef{{Number: 1, Title: "Chapter 1"}},
	}
	store := newMockStore()

	var callCount int
	o := New(Config{
		Workers:       1,
		SingleBookURL: "https://example.com/book/progress-book",
		OnProgress: func(_ Progress) {
			callCount++
		},
	}, novel, discardLogger(), store)

	if err := o.Run(context.Background()); err != nil {
		t.Fatalf("Run() returned error: %v", err)
	}
	if callCount == 0 {
		t.Error("OnProgress was never called")
	}
}

// TestRun_ContextCancelled verifies that Run returns a non-nil error when the
// context is cancelled before work completes.
func TestRun_ContextCancelled(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	cancel() // cancel immediately

	novel := &mockScraper{
		meta:     scraper.BookMeta{Slug: "cancel-book", Title: "Cancel Book"},
		chapters: []scraper.ChapterRef{{Number: 1}},
	}
	store := newMockStore()

	o := New(Config{Workers: 1, SingleBookURL: "https://example.com/book/cancel-book"}, novel, discardLogger(), store)
	err := o.Run(ctx)
	if err == nil {
		t.Error("expected non-nil error when context is cancelled, got nil")
	}
}