a85636d5db
- New Go backend binary (backend + runner) replacing old scraper/ - Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose - Rename scraperFetch → backendFetch across all 19 UI server files - Remove SCRAPER_PROXY env var and proxy transport from browser.Config - Add Meilisearch, Valkey, Caddy to docker-compose - Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
125 lines
3.7 KiB
Go
125 lines
3.7 KiB
Go
// Package httputil provides shared HTTP helpers used by both the runner and
|
|
// backend binaries. It has no imports from this module — only the standard
|
|
// library — so it is safe to import from anywhere in the dependency graph.
|
|
package httputil
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
// Client is the minimal interface for making HTTP GET requests.
|
|
// *http.Client satisfies this interface.
|
|
type Client interface {
|
|
Do(req *http.Request) (*http.Response, error)
|
|
}
|
|
|
|
// ErrMaxRetries is returned when RetryGet exhausts all attempts.
|
|
var ErrMaxRetries = errors.New("httputil: max retries exceeded")
|
|
|
|
// errClientError is returned by doGet for 4xx responses; it signals that the
|
|
// request should NOT be retried (the client is at fault).
|
|
var errClientError = errors.New("httputil: client error")
|
|
|
|
// RetryGet fetches url using client, retrying on network errors or 5xx
|
|
// responses with exponential backoff. It returns the full response body as a
|
|
// string on success.
|
|
//
|
|
// - maxAttempts: total number of attempts (must be >= 1)
|
|
// - baseDelay: initial wait before the second attempt; doubles each retry
|
|
func RetryGet(ctx context.Context, client Client, url string, maxAttempts int, baseDelay time.Duration) (string, error) {
|
|
if maxAttempts < 1 {
|
|
maxAttempts = 1
|
|
}
|
|
delay := baseDelay
|
|
|
|
var lastErr error
|
|
for attempt := 0; attempt < maxAttempts; attempt++ {
|
|
if attempt > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
return "", ctx.Err()
|
|
case <-time.After(delay):
|
|
}
|
|
delay *= 2
|
|
}
|
|
|
|
body, err := doGet(ctx, client, url)
|
|
if err == nil {
|
|
return body, nil
|
|
}
|
|
lastErr = err
|
|
|
|
// Do not retry on context cancellation.
|
|
if ctx.Err() != nil {
|
|
return "", ctx.Err()
|
|
}
|
|
// Do not retry on 4xx — the client is at fault.
|
|
if errors.Is(err, errClientError) {
|
|
return "", err
|
|
}
|
|
}
|
|
|
|
return "", fmt.Errorf("%w after %d attempts: %w", ErrMaxRetries, maxAttempts, lastErr)
|
|
}
|
|
|
|
func doGet(ctx context.Context, client Client, url string) (string, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return "", fmt.Errorf("build request: %w", err)
|
|
}
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("GET %s: %w", url, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode >= 500 {
|
|
return "", fmt.Errorf("GET %s: server error %d", url, resp.StatusCode)
|
|
}
|
|
if resp.StatusCode >= 400 {
|
|
return "", fmt.Errorf("%w: GET %s: client error %d", errClientError, url, resp.StatusCode)
|
|
}
|
|
|
|
raw, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return "", fmt.Errorf("read body %s: %w", url, err)
|
|
}
|
|
return string(raw), nil
|
|
}
|
|
|
|
// WriteJSON writes v as JSON to w with the given HTTP status code and sets the
|
|
// Content-Type header to application/json.
|
|
func WriteJSON(w http.ResponseWriter, status int, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(v)
|
|
}
|
|
|
|
// WriteError writes a JSON error object {"error": msg} with the given status.
|
|
func WriteError(w http.ResponseWriter, status int, msg string) {
|
|
WriteJSON(w, status, map[string]string{"error": msg})
|
|
}
|
|
|
|
// maxBodyBytes is the limit applied by DecodeJSON to prevent unbounded reads.
|
|
const maxBodyBytes = 1 << 20 // 1 MiB
|
|
|
|
// DecodeJSON decodes a JSON request body into v. It enforces a 1 MiB size
|
|
// limit and returns a descriptive error on any failure.
|
|
func DecodeJSON(r *http.Request, v any) error {
|
|
r.Body = http.MaxBytesReader(nil, r.Body, maxBodyBytes)
|
|
dec := json.NewDecoder(r.Body)
|
|
dec.DisallowUnknownFields()
|
|
if err := dec.Decode(v); err != nil {
|
|
return fmt.Errorf("decode JSON body: %w", err)
|
|
}
|
|
return nil
|
|
}
|