// Package browser provides a rate-limited HTTP client for web scraping. // The Client interface is the only thing the rest of the codebase depends on; // the concrete DirectClient can be swapped for any other implementation // (e.g. a Browserless-backed client) without touching callers. package browser import ( "context" "errors" "fmt" "io" "net/http" "net/url" "strconv" "sync" "time" ) // ErrRateLimit is returned by GetContent when the server responds with 429. // It carries the suggested retry delay (from Retry-After header, or a default). var ErrRateLimit = errors.New("rate limited (429)") // RateLimitError wraps ErrRateLimit and carries the suggested wait duration. type RateLimitError struct { // RetryAfter is how long the caller should wait before retrying. // Derived from the Retry-After response header when present; otherwise a default. RetryAfter time.Duration } func (e *RateLimitError) Error() string { return fmt.Sprintf("rate limited (429): retry after %s", e.RetryAfter) } func (e *RateLimitError) Is(target error) bool { return target == ErrRateLimit } // defaultRateLimitDelay is used when the server returns 429 with no Retry-After header. const defaultRateLimitDelay = 60 * time.Second // Client is the interface used by scrapers to fetch raw page HTML. // Implementations must be safe for concurrent use. type Client interface { // GetContent fetches the URL and returns the full response body as a string. // It should respect the provided context for cancellation and timeouts. GetContent(ctx context.Context, pageURL string) (string, error) } // Config holds tunable parameters for the direct HTTP client. type Config struct { // MaxConcurrent limits the number of simultaneous in-flight requests. // Defaults to 5 when 0. MaxConcurrent int // Timeout is the per-request deadline. Defaults to 90s when 0. Timeout time.Duration // ProxyURL is an optional outbound proxy, e.g. "http://user:pass@host:3128". // Falls back to HTTP_PROXY / HTTPS_PROXY environment variables when empty. ProxyURL string } // DirectClient is a plain net/http-based Client with a concurrency semaphore. type DirectClient struct { http *http.Client semaphore chan struct{} } // NewDirectClient returns a DirectClient configured by cfg. func NewDirectClient(cfg Config) *DirectClient { if cfg.MaxConcurrent <= 0 { cfg.MaxConcurrent = 5 } if cfg.Timeout <= 0 { cfg.Timeout = 90 * time.Second } transport := &http.Transport{ MaxIdleConnsPerHost: cfg.MaxConcurrent * 2, DisableCompression: false, } if cfg.ProxyURL != "" { proxyParsed, err := url.Parse(cfg.ProxyURL) if err == nil { transport.Proxy = http.ProxyURL(proxyParsed) } } else { transport.Proxy = http.ProxyFromEnvironment } return &DirectClient{ http: &http.Client{ Transport: transport, Timeout: cfg.Timeout, }, semaphore: make(chan struct{}, cfg.MaxConcurrent), } } // GetContent fetches pageURL respecting the concurrency limit. func (c *DirectClient) GetContent(ctx context.Context, pageURL string) (string, error) { // Acquire semaphore slot. select { case c.semaphore <- struct{}{}: case <-ctx.Done(): return "", ctx.Err() } defer func() { <-c.semaphore }() req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) if err != nil { return "", fmt.Errorf("browser: build request %s: %w", pageURL, err) } req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.5") resp, err := c.http.Do(req) if err != nil { return "", fmt.Errorf("browser: GET %s: %w", pageURL, err) } defer resp.Body.Close() if resp.StatusCode == http.StatusTooManyRequests { delay := defaultRateLimitDelay if ra := resp.Header.Get("Retry-After"); ra != "" { if secs, err := strconv.Atoi(ra); err == nil && secs > 0 { delay = time.Duration(secs) * time.Second } } return "", &RateLimitError{RetryAfter: delay} } if resp.StatusCode >= 400 { return "", fmt.Errorf("browser: GET %s returned %d", pageURL, resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("browser: read body %s: %w", pageURL, err) } return string(body), nil } // Do implements httputil.Client so DirectClient can be passed to RetryGet. func (c *DirectClient) Do(req *http.Request) (*http.Response, error) { select { case c.semaphore <- struct{}{}: case <-req.Context().Done(): return nil, req.Context().Err() } defer func() { <-c.semaphore }() return c.http.Do(req) } // ── Stub for testing ────────────────────────────────────────────────────────── // StubClient is a test double for Client. It returns pre-configured responses // keyed on URL. Calls to unknown URLs return an error. type StubClient struct { mu sync.Mutex pages map[string]string errors map[string]error callLog []string } // NewStub creates a StubClient with no pages pre-loaded. func NewStub() *StubClient { return &StubClient{ pages: make(map[string]string), errors: make(map[string]error), } } // SetPage registers a URL → HTML body mapping. func (s *StubClient) SetPage(u, html string) { s.mu.Lock() s.pages[u] = html s.mu.Unlock() } // SetError registers a URL → error mapping (returned instead of a body). func (s *StubClient) SetError(u string, err error) { s.mu.Lock() s.errors[u] = err s.mu.Unlock() } // CallLog returns the ordered list of URLs that were requested. func (s *StubClient) CallLog() []string { s.mu.Lock() defer s.mu.Unlock() out := make([]string, len(s.callLog)) copy(out, s.callLog) return out } // GetContent returns the registered page or an error for the URL. func (s *StubClient) GetContent(_ context.Context, pageURL string) (string, error) { s.mu.Lock() defer s.mu.Unlock() s.callLog = append(s.callLog, pageURL) if err, ok := s.errors[pageURL]; ok { return "", err } if html, ok := s.pages[pageURL]; ok { return html, nil } return "", fmt.Errorf("stub: no page registered for %q", pageURL) }