fb6b364382
- Remove dead code: browser cdp/content_scrape strategies, writer package, printUsage, downloadAndStoreCoverCLI in main.go - Fix bugs: defer-in-loop in pocketbase deleteWhere, listAll() pagination hard cap removed, splitChapterTitle off-by-one in date extraction - Split server.go (~1700 lines) into focused handler files: handlers_audio, handlers_browse, handlers_progress, handlers_ranking, handlers_scrape - Export htmlutil.AttrVal/TextContent/ResolveURL; add storage/coverutil.go to consolidate duplicate helpers - Flatten deeply nested conditionals: voices() early-return guards, ScrapeCatalogue next-link double attr scan, chapterNumberFromKey dead strings.Cut line, splitChapterTitle double-nested unit/suffix loop - Add unit tests: htmlutil (9 funcs), novelfire ScrapeMetadata (3 cases), orchestrator Run (5 cases), storage chapterNumberFromKey/splitChapterTitle (22 cases); all pass with go build/vet/test clean
128 lines
3.5 KiB
Go
128 lines
3.5 KiB
Go
package browser
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
// Config holds the connection parameters for a Browserless instance.
|
|
type Config struct {
|
|
// BaseURL is the HTTP base URL, e.g. "http://localhost:3030".
|
|
BaseURL string
|
|
// Token is the optional API token (BROWSERLESS_TOKEN env var).
|
|
Token string
|
|
// Timeout is the per-request HTTP timeout; defaults to 60 s.
|
|
Timeout time.Duration
|
|
// MaxConcurrent caps the number of simultaneous in-flight requests sent to
|
|
// Browserless. When all slots are occupied new calls block until one
|
|
// completes (or ctx is cancelled). 0 means no limit.
|
|
MaxConcurrent int
|
|
}
|
|
|
|
// makeSem returns a buffered channel used as a counting semaphore.
|
|
// If n <= 0 a nil channel is returned, which causes acquire/release to be no-ops.
|
|
func makeSem(n int) chan struct{} {
|
|
if n <= 0 {
|
|
return nil
|
|
}
|
|
return make(chan struct{}, n)
|
|
}
|
|
|
|
// acquire takes one slot from sem. It returns an error if ctx is cancelled
|
|
// before a slot becomes available. If sem is nil it returns immediately.
|
|
func acquire(ctx context.Context, sem chan struct{}) error {
|
|
if sem == nil {
|
|
return nil
|
|
}
|
|
select {
|
|
case sem <- struct{}{}:
|
|
return nil
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
// release frees the slot previously obtained by acquire.
|
|
// If sem is nil it is a no-op.
|
|
func release(sem chan struct{}) {
|
|
if sem != nil {
|
|
<-sem
|
|
}
|
|
}
|
|
|
|
// ─── /content client ──────────────────────────────────────────────────────────
|
|
|
|
// contentClient implements BrowserClient using the /content endpoint.
|
|
type contentClient struct {
|
|
cfg Config
|
|
http *http.Client
|
|
sem chan struct{}
|
|
}
|
|
|
|
// NewContentClient returns a BrowserClient that uses POST /content.
|
|
func NewContentClient(cfg Config) BrowserClient {
|
|
if cfg.Timeout == 0 {
|
|
cfg.Timeout = 90 * time.Second
|
|
}
|
|
return &contentClient{
|
|
cfg: cfg,
|
|
http: &http.Client{Timeout: cfg.Timeout},
|
|
sem: makeSem(cfg.MaxConcurrent),
|
|
}
|
|
}
|
|
|
|
func (c *contentClient) Strategy() Strategy { return StrategyContent }
|
|
|
|
func (c *contentClient) GetContent(ctx context.Context, req ContentRequest) (string, error) {
|
|
if err := acquire(ctx, c.sem); err != nil {
|
|
return "", fmt.Errorf("content: semaphore: %w", err)
|
|
}
|
|
defer release(c.sem)
|
|
|
|
body, err := json.Marshal(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("content: marshal request: %w", err)
|
|
}
|
|
|
|
url := c.cfg.BaseURL + "/content"
|
|
if c.cfg.Token != "" {
|
|
url += "?token=" + c.cfg.Token
|
|
}
|
|
|
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
|
if err != nil {
|
|
return "", fmt.Errorf("content: build request: %w", err)
|
|
}
|
|
httpReq.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := c.http.Do(httpReq)
|
|
if err != nil {
|
|
return "", fmt.Errorf("content: do request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
b, _ := io.ReadAll(resp.Body)
|
|
return "", fmt.Errorf("content: unexpected status %d: %s", resp.StatusCode, b)
|
|
}
|
|
|
|
raw, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return "", fmt.Errorf("content: read body: %w", err)
|
|
}
|
|
return string(raw), nil
|
|
}
|
|
|
|
func (c *contentClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
|
|
return ScrapeResponse{}, fmt.Errorf("content client does not support /scrape; use NewScrapeClient")
|
|
}
|
|
|
|
func (c *contentClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
|
|
return fmt.Errorf("content client does not support CDP")
|
|
}
|