chore: migrate to v3, Doppler secrets, clean up legacy code
Some checks failed
CI / v3 / Check ui (pull_request) Failing after 15s
CI / v3 / Test backend (pull_request) Failing after 16s
CI / v3 / Docker / backend (pull_request) Has been skipped
CI / v3 / Docker / runner (pull_request) Has been skipped
CI / v3 / Docker / ui (pull_request) Has been skipped
Some checks failed
CI / v3 / Check ui (pull_request) Failing after 15s
CI / v3 / Test backend (pull_request) Failing after 16s
CI / v3 / Docker / backend (pull_request) Has been skipped
CI / v3 / Docker / runner (pull_request) Has been skipped
CI / v3 / Docker / ui (pull_request) Has been skipped
- Remove all pre-v3 code: scraper, ui-v2, backend v1, ios v1+v2, legacy CI workflows - Flatten v3/ contents to repo root - Add Doppler secrets management (project=libnovel, config=prd) - Add justfile with doppler run wrappers for all docker compose commands - Strip hardcoded env fallbacks from docker-compose.yml - Add minimal README.md - Clean up .gitignore
This commit is contained in:
@@ -111,7 +111,7 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case entries <- domain.CatalogueEntry{Title: title, URL: bookURL}:
|
||||
case entries <- domain.CatalogueEntry{Slug: slugFromURL(bookURL), Title: title, URL: bookURL}:
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,8 +194,12 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.Bo
|
||||
|
||||
// ── ChapterListProvider ───────────────────────────────────────────────────────
|
||||
|
||||
// ScrapeChapterList returns all chapter references for a book, ordered ascending.
|
||||
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) {
|
||||
// ScrapeChapterList returns chapter references for a book, ordered ascending.
|
||||
// upTo > 0 stops pagination as soon as at least upTo chapter numbers have been
|
||||
// collected — use this for range scrapes so we don't paginate 100 pages just
|
||||
// to discover refs we'll never scrape. upTo == 0 fetches all pages.
|
||||
// Each page fetch uses retryGet with 429-aware exponential backoff.
|
||||
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error) {
|
||||
var refs []domain.ChapterRef
|
||||
baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
|
||||
page := 1
|
||||
@@ -210,7 +214,7 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma
|
||||
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
|
||||
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
|
||||
|
||||
raw, err := s.client.GetContent(ctx, pageURL)
|
||||
raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second)
|
||||
if err != nil {
|
||||
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
|
||||
}
|
||||
@@ -255,6 +259,13 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma
|
||||
})
|
||||
}
|
||||
|
||||
// Early-stop: if we have seen at least upTo chapter numbers, we have
|
||||
// enough refs to cover the requested range — no need to paginate further.
|
||||
if upTo > 0 && len(refs) > 0 && refs[len(refs)-1].Number >= upTo {
|
||||
s.log.Debug("chapter list early-stop reached", "upTo", upTo, "collected", len(refs))
|
||||
break
|
||||
}
|
||||
|
||||
page++
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user