refactor: audit, split server.go, add unit tests, and fix latent bugs
- Remove dead code: browser cdp/content_scrape strategies, writer package, printUsage, downloadAndStoreCoverCLI in main.go - Fix bugs: defer-in-loop in pocketbase deleteWhere, listAll() pagination hard cap removed, splitChapterTitle off-by-one in date extraction - Split server.go (~1700 lines) into focused handler files: handlers_audio, handlers_browse, handlers_progress, handlers_ranking, handlers_scrape - Export htmlutil.AttrVal/TextContent/ResolveURL; add storage/coverutil.go to consolidate duplicate helpers - Flatten deeply nested conditionals: voices() early-return guards, ScrapeCatalogue next-link double attr scan, chapterNumberFromKey dead strings.Cut line, splitChapterTitle double-nested unit/suffix loop - Add unit tests: htmlutil (9 funcs), novelfire ScrapeMetadata (3 cases), orchestrator Run (5 cases), storage chapterNumberFromKey/splitChapterTitle (22 cases); all pass with go build/vet/test clean
This commit is contained in:
Admin
@@ -1,137 +0,0 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// cdpClient implements BrowserClient using the CDP WebSocket endpoint.
|
||||
type cdpClient struct {
|
||||
cfg Config
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
// NewCDPClient returns a BrowserClient that uses CDP WebSocket sessions.
|
||||
func NewCDPClient(cfg Config) BrowserClient {
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = 60 * time.Second
|
||||
}
|
||||
return &cdpClient{cfg: cfg, sem: makeSem(cfg.MaxConcurrent)}
|
||||
}
|
||||
|
||||
func (c *cdpClient) Strategy() Strategy { return StrategyCDP }
|
||||
|
||||
func (c *cdpClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
|
||||
return "", fmt.Errorf("CDP client does not support /content; use NewContentClient")
|
||||
}
|
||||
|
||||
func (c *cdpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
|
||||
return ScrapeResponse{}, fmt.Errorf("CDP client does not support /scrape; use NewScrapeClient")
|
||||
}
|
||||
|
||||
// CDPSession opens a WebSocket to the Browserless /devtools/browser endpoint,
|
||||
// navigates to pageURL, and invokes fn with a live CDPConn.
|
||||
func (c *cdpClient) CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error {
|
||||
if err := acquire(ctx, c.sem); err != nil {
|
||||
return fmt.Errorf("cdp: semaphore: %w", err)
|
||||
}
|
||||
defer release(c.sem)
|
||||
|
||||
// Build WebSocket URL: ws://host:port/devtools/browser?token=...&url=...
|
||||
wsURL := strings.Replace(c.cfg.BaseURL, "http://", "ws://", 1)
|
||||
wsURL = strings.Replace(wsURL, "https://", "wss://", 1)
|
||||
wsURL += "/devtools/browser"
|
||||
sep := "?"
|
||||
if c.cfg.Token != "" {
|
||||
wsURL += sep + "token=" + c.cfg.Token
|
||||
sep = "&"
|
||||
}
|
||||
wsURL += sep + "url=" + pageURL
|
||||
|
||||
dialer := websocket.Dialer{
|
||||
HandshakeTimeout: 15 * time.Second,
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
}
|
||||
|
||||
conn, _, err := dialer.DialContext(ctx, wsURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cdp: dial %s: %w", wsURL, err)
|
||||
}
|
||||
|
||||
cdp := &cdpConn{ws: conn}
|
||||
defer cdp.Close()
|
||||
|
||||
return fn(ctx, cdp)
|
||||
}
|
||||
|
||||
// ─── cdpConn ─────────────────────────────────────────────────────────────────
|
||||
|
||||
type cdpConn struct {
|
||||
ws *websocket.Conn
|
||||
counter atomic.Int64
|
||||
}
|
||||
|
||||
type cdpRequest struct {
|
||||
ID int64 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params map[string]any `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
type cdpResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
Result map[string]any `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (c *cdpConn) Send(ctx context.Context, method string, params map[string]any) (map[string]any, error) {
|
||||
id := c.counter.Add(1)
|
||||
|
||||
req := cdpRequest{ID: id, Method: method, Params: params}
|
||||
data, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdp send: marshal: %w", err)
|
||||
}
|
||||
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.ws.SetWriteDeadline(dl)
|
||||
}
|
||||
if err := c.ws.WriteMessage(websocket.TextMessage, data); err != nil {
|
||||
return nil, fmt.Errorf("cdp send: write: %w", err)
|
||||
}
|
||||
|
||||
// Read messages until we find the response matching our id.
|
||||
for {
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.ws.SetReadDeadline(dl)
|
||||
}
|
||||
_, msg, err := c.ws.ReadMessage()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdp send: read: %w", err)
|
||||
}
|
||||
var resp cdpResponse
|
||||
if err := json.Unmarshal(msg, &resp); err != nil {
|
||||
continue // skip non-JSON frames (events etc.)
|
||||
}
|
||||
if resp.ID != id {
|
||||
continue // event or different command reply
|
||||
}
|
||||
if resp.Error != nil {
|
||||
return nil, fmt.Errorf("cdp error %d: %s", resp.Error.Code, resp.Error.Message)
|
||||
}
|
||||
return resp.Result, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cdpConn) Close() error {
|
||||
return c.ws.Close()
|
||||
}
|
||||
@@ -55,6 +55,8 @@ func release(sem chan struct{}) {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── /content client ──────────────────────────────────────────────────────────
|
||||
|
||||
// contentClient implements BrowserClient using the /content endpoint.
|
||||
type contentClient struct {
|
||||
cfg Config
|
||||
@@ -121,75 +123,5 @@ func (c *contentClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeRe
|
||||
}
|
||||
|
||||
func (c *contentClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
|
||||
return fmt.Errorf("content client does not support CDP; use NewCDPClient")
|
||||
}
|
||||
|
||||
// ─── /scrape client ───────────────────────────────────────────────────────────
|
||||
|
||||
type scrapeClient struct {
|
||||
cfg Config
|
||||
http *http.Client
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
// NewScrapeClient returns a BrowserClient that uses POST /scrape.
|
||||
func NewScrapeClient(cfg Config) BrowserClient {
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = 90 * time.Second
|
||||
}
|
||||
return &scrapeClient{
|
||||
cfg: cfg,
|
||||
http: &http.Client{Timeout: cfg.Timeout},
|
||||
sem: makeSem(cfg.MaxConcurrent),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *scrapeClient) Strategy() Strategy { return StrategyScrape }
|
||||
|
||||
func (c *scrapeClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
|
||||
return "", fmt.Errorf("scrape client does not support /content; use NewContentClient")
|
||||
}
|
||||
|
||||
func (c *scrapeClient) ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) {
|
||||
if err := acquire(ctx, c.sem); err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: semaphore: %w", err)
|
||||
}
|
||||
defer release(c.sem)
|
||||
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: marshal request: %w", err)
|
||||
}
|
||||
|
||||
url := c.cfg.BaseURL + "/scrape"
|
||||
if c.cfg.Token != "" {
|
||||
url += "?token=" + c.cfg.Token
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: build request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.http.Do(httpReq)
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: unexpected status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
|
||||
var result ScrapeResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: decode response: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *scrapeClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
|
||||
return fmt.Errorf("scrape client does not support CDP; use NewCDPClient")
|
||||
return fmt.Errorf("content client does not support CDP")
|
||||
}
|
||||
Reference in New Issue
Block a user