kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

feature/backend-rewrite #2

Open
kamil wants to merge 236 commits from feature/backend-rewrite into main
pull from: feature/backend-rewrite
merge into: kamil:main
Conversation 0 Commits 236 Files Changed 406 +8 -33
4 changed files with 8 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit cfd893d24b - Show all commits

4
.env.example
View File

@@ -39,10 +39,6 @@ ERROR_ALERT_URL=
# Which Browserless strategy the scraper uses: content | scrape | cdp | direct
BROWSERLESS_STRATEGY=direct
# Strategy for URL retrieval (chapter list). Uses browserless content strategy by default.
# Set to direct to use plain HTTP, or content/scrape/cdp for browserless.
BROWSERLESS_URL_STRATEGY=content
# ── Scraper ───────────────────────────────────────────────────────────────────
# Chapter worker goroutines (0 = NumCPU inside the container)
SCRAPER_WORKERS=0

5
docker-compose.yml
View File

@@ -119,10 +119,9 @@ services:
environment:
BROWSERLESS_URL: "http://browserless:${BROWSERLESS_PORT:-3030}"
BROWSERLESS_TOKEN: "${BROWSERLESS_TOKEN:-}"
# content | scrape | cdp | direct — swap to test different strategies.
# content | scrape | cdp | direct — used for catalogue, metadata, ranking.
# Chapter list and chapter text always use direct HTTP regardless of this setting.
BROWSERLESS_STRATEGY: "${BROWSERLESS_STRATEGY:-direct}"
# Strategy for URL retrieval (chapter list). Default: content (browserless)
BROWSERLESS_URL_STRATEGY: "${BROWSERLESS_URL_STRATEGY:-content}"
# 0 → defaults to NumCPU inside the container.
SCRAPER_WORKERS: "${SCRAPER_WORKERS:-0}"
SCRAPER_STATIC_ROOT: "/app/static/books"

13
scraper/cmd/scraper/main.go
View File

@@ -12,7 +12,7 @@
//
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapters always use direct HTTP)
// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapter list+text always use direct HTTP)
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
@@ -92,11 +92,9 @@ func run(log *slog.Logger) error {
}
strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect))))
urlStrategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_URL_STRATEGY", string(browser.StrategyContent))))
bc := newBrowserClient(strategy, browserCfg)
urlClient := newBrowserClient(urlStrategy, browserCfg)
// Chapter text is server-rendered on novelfire.net — direct HTTP is faster and avoids Browserless.
chapterClient := browser.NewDirectHTTPClient(browserCfg)
// Chapter list and chapter text are server-rendered on novelfire.net — direct HTTP for both.
directClient := browser.NewDirectHTTPClient(browserCfg)
// ── Storage backends ────────────────────────────────────────────────────
minioCfg := storage.MinioConfig{
@@ -121,7 +119,7 @@ func run(log *slog.Logger) error {
return fmt.Errorf("storage init failed: %w", err)
}
nf := novelfire.New(bc, log, urlClient, chapterClient, store)
nf := novelfire.New(bc, log, directClient, directClient, store)
workers := 0
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
@@ -228,8 +226,7 @@ Commands:
Environment variables:
BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
BROWSERLESS_TOKEN API token (default: "")
BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct)
BROWSERLESS_URL_STRATEGY Strategy for URL retrieval (default: content)
BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct; chapter list+text always use direct HTTP)
BROWSERLESS_MAX_CONCURRENT Max simultaneous sessions (default: 5)
BROWSERLESS_TIMEOUT HTTP request timeout sec (default: 90)
SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d)

19
scraper/internal/novelfire/scraper.go
View File

@@ -283,24 +283,11 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
s.log.Debug("chapter list fetch starting",
"page", page,
"payload_url", pageURL,
"payload_wait_selector", ".chapter-list",
"payload_wait_selector_timeout_ms", 15000,
"payload_wait_timeout_ms", 2000,
"strategy", s.urlClient.Strategy(),
)
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
URL: pageURL,
// Wait up to 15 s for the chapter list container to appear in the DOM.
WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000},
// After the selector is found, wait an additional 2 s for any
// deferred JS rendering (lazy-loaded links, infinite-scroll hydration).
WaitForTimeout: 2000,
RejectResourceTypes: rejectResourceTypes,
GotoOptions: &browser.GotoOptions{Timeout: 60000},
// Do NOT use BestAttempt — we want a complete page or a clear error,
// not silently partial HTML that looks like "no more chapters".
BestAttempt: false,
})
if err != nil {
s.log.Debug("chapter list fetch failed",
@@ -386,11 +373,7 @@ func (s *Scraper) ScrapeChapterListPage(ctx context.Context, pageURL string) ([]
s.log.Info("scraping chapter list page (single)", "url", pageURL)
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
URL: pageURL,
WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000},
WaitForTimeout: 2000,
RejectResourceTypes: rejectResourceTypes,
GotoOptions: &browser.GotoOptions{Timeout: 60000},
URL: pageURL,
})
if err != nil {
return nil, fmt.Errorf("chapter list page fetch: %w", err)