147 lines
4.2 KiB
Go
147 lines
4.2 KiB
Go
package browser
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
// Config holds the connection parameters for a Browserless instance.
|
|
type Config struct {
|
|
// BaseURL is the HTTP base URL, e.g. "http://localhost:3000".
|
|
BaseURL string
|
|
// Token is the optional API token (BROWSERLESS_TOKEN env var).
|
|
Token string
|
|
// Timeout is the per-request HTTP timeout; defaults to 60 s.
|
|
Timeout time.Duration
|
|
}
|
|
|
|
// contentClient implements BrowserClient using the /content endpoint.
|
|
type contentClient struct {
|
|
cfg Config
|
|
http *http.Client
|
|
}
|
|
|
|
// NewContentClient returns a BrowserClient that uses POST /content.
|
|
func NewContentClient(cfg Config) BrowserClient {
|
|
if cfg.Timeout == 0 {
|
|
cfg.Timeout = 60 * time.Second
|
|
}
|
|
return &contentClient{
|
|
cfg: cfg,
|
|
http: &http.Client{Timeout: cfg.Timeout},
|
|
}
|
|
}
|
|
|
|
func (c *contentClient) Strategy() Strategy { return StrategyContent }
|
|
|
|
func (c *contentClient) GetContent(ctx context.Context, req ContentRequest) (string, error) {
|
|
body, err := json.Marshal(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("content: marshal request: %w", err)
|
|
}
|
|
|
|
url := c.cfg.BaseURL + "/content"
|
|
if c.cfg.Token != "" {
|
|
url += "?token=" + c.cfg.Token
|
|
}
|
|
|
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
|
if err != nil {
|
|
return "", fmt.Errorf("content: build request: %w", err)
|
|
}
|
|
httpReq.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := c.http.Do(httpReq)
|
|
if err != nil {
|
|
return "", fmt.Errorf("content: do request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
b, _ := io.ReadAll(resp.Body)
|
|
return "", fmt.Errorf("content: unexpected status %d: %s", resp.StatusCode, b)
|
|
}
|
|
|
|
raw, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return "", fmt.Errorf("content: read body: %w", err)
|
|
}
|
|
return string(raw), nil
|
|
}
|
|
|
|
func (c *contentClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
|
|
return ScrapeResponse{}, fmt.Errorf("content client does not support /scrape; use NewScrapeClient")
|
|
}
|
|
|
|
func (c *contentClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
|
|
return fmt.Errorf("content client does not support CDP; use NewCDPClient")
|
|
}
|
|
|
|
// ─── /scrape client ───────────────────────────────────────────────────────────
|
|
|
|
type scrapeClient struct {
|
|
cfg Config
|
|
http *http.Client
|
|
}
|
|
|
|
// NewScrapeClient returns a BrowserClient that uses POST /scrape.
|
|
func NewScrapeClient(cfg Config) BrowserClient {
|
|
if cfg.Timeout == 0 {
|
|
cfg.Timeout = 60 * time.Second
|
|
}
|
|
return &scrapeClient{
|
|
cfg: cfg,
|
|
http: &http.Client{Timeout: cfg.Timeout},
|
|
}
|
|
}
|
|
|
|
func (c *scrapeClient) Strategy() Strategy { return StrategyScrape }
|
|
|
|
func (c *scrapeClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
|
|
return "", fmt.Errorf("scrape client does not support /content; use NewContentClient")
|
|
}
|
|
|
|
func (c *scrapeClient) ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) {
|
|
body, err := json.Marshal(req)
|
|
if err != nil {
|
|
return ScrapeResponse{}, fmt.Errorf("scrape: marshal request: %w", err)
|
|
}
|
|
|
|
url := c.cfg.BaseURL + "/scrape"
|
|
if c.cfg.Token != "" {
|
|
url += "?token=" + c.cfg.Token
|
|
}
|
|
|
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
|
if err != nil {
|
|
return ScrapeResponse{}, fmt.Errorf("scrape: build request: %w", err)
|
|
}
|
|
httpReq.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := c.http.Do(httpReq)
|
|
if err != nil {
|
|
return ScrapeResponse{}, fmt.Errorf("scrape: do request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
b, _ := io.ReadAll(resp.Body)
|
|
return ScrapeResponse{}, fmt.Errorf("scrape: unexpected status %d: %s", resp.StatusCode, b)
|
|
}
|
|
|
|
var result ScrapeResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
|
return ScrapeResponse{}, fmt.Errorf("scrape: decode response: %w", err)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (c *scrapeClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
|
|
return fmt.Errorf("scrape client does not support CDP; use NewCDPClient")
|
|
}
|