chore: migrate to v3, Doppler secrets, clean up legacy code
Some checks failed
CI / v3 / Check ui (pull_request) Failing after 15s
CI / v3 / Test backend (pull_request) Failing after 16s
CI / v3 / Docker / backend (pull_request) Has been skipped
CI / v3 / Docker / runner (pull_request) Has been skipped
CI / v3 / Docker / ui (pull_request) Has been skipped

- Remove all pre-v3 code: scraper, ui-v2, backend v1, ios v1+v2, legacy CI workflows
- Flatten v3/ contents to repo root
- Add Doppler secrets management (project=libnovel, config=prd)
- Add justfile with doppler run wrappers for all docker compose commands
- Strip hardcoded env fallbacks from docker-compose.yml
- Add minimal README.md
- Clean up .gitignore
This commit is contained in:
Admin
2026-03-23 17:21:12 +05:00
parent 1118392811
commit 59e8cdb19a
522 changed files with 5259 additions and 80365 deletions

View File

@@ -111,7 +111,7 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
select {
case <-ctx.Done():
return
case entries <- domain.CatalogueEntry{Title: title, URL: bookURL}:
case entries <- domain.CatalogueEntry{Slug: slugFromURL(bookURL), Title: title, URL: bookURL}:
}
}
@@ -194,8 +194,12 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.Bo
// ── ChapterListProvider ───────────────────────────────────────────────────────
// ScrapeChapterList returns all chapter references for a book, ordered ascending.
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) {
// ScrapeChapterList returns chapter references for a book, ordered ascending.
// upTo > 0 stops pagination as soon as at least upTo chapter numbers have been
// collected — use this for range scrapes so we don't paginate 100 pages just
// to discover refs we'll never scrape. upTo == 0 fetches all pages.
// Each page fetch uses retryGet with 429-aware exponential backoff.
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error) {
var refs []domain.ChapterRef
baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
page := 1
@@ -210,7 +214,7 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
raw, err := s.client.GetContent(ctx, pageURL)
raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second)
if err != nil {
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
}
@@ -255,6 +259,13 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma
})
}
// Early-stop: if we have seen at least upTo chapter numbers, we have
// enough refs to cover the requested range — no need to paginate further.
if upTo > 0 && len(refs) > 0 && refs[len(refs)-1].Number >= upTo {
s.log.Debug("chapter list early-stop reached", "upTo", upTo, "collected", len(refs))
break
}
page++
}