feat(v3): add v3 stack — backend rewrite, renamed env vars, docs

- New Go backend binary (backend + runner) replacing old scraper/ - Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose - Rename scraperFetch → backendFetch across all 19 UI server files - Remove SCRAPER_PROXY env var and proxy transport from browser.Config - Add Meilisearch, Valkey, Caddy to docker-compose - Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
2026-03-22 17:27:32 +05:00
parent 29d0eeb7e8
commit a85636d5db
178 changed files with 25951 additions and 0 deletions
--- a/v3/backend/internal/httputil/httputil.go
+++ b/v3/backend/internal/httputil/httputil.go
@@ -0,0 +1,124 @@
+// Package httputil provides shared HTTP helpers used by both the runner and
+// backend binaries. It has no imports from this module — only the standard
+// library — so it is safe to import from anywhere in the dependency graph.
+package httputil
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+// Client is the minimal interface for making HTTP GET requests.
+// *http.Client satisfies this interface.
+type Client interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+// ErrMaxRetries is returned when RetryGet exhausts all attempts.
+var ErrMaxRetries = errors.New("httputil: max retries exceeded")
+
+// errClientError is returned by doGet for 4xx responses; it signals that the
+// request should NOT be retried (the client is at fault).
+var errClientError = errors.New("httputil: client error")
+
+// RetryGet fetches url using client, retrying on network errors or 5xx
+// responses with exponential backoff. It returns the full response body as a
+// string on success.
+//
+//   - maxAttempts: total number of attempts (must be >= 1)
+//   - baseDelay:   initial wait before the second attempt; doubles each retry
+func RetryGet(ctx context.Context, client Client, url string, maxAttempts int, baseDelay time.Duration) (string, error) {
+	if maxAttempts < 1 {
+		maxAttempts = 1
+	}
+	delay := baseDelay
+
+	var lastErr error
+	for attempt := 0; attempt < maxAttempts; attempt++ {
+		if attempt > 0 {
+			select {
+			case <-ctx.Done():
+				return "", ctx.Err()
+			case <-time.After(delay):
+			}
+			delay *= 2
+		}
+
+		body, err := doGet(ctx, client, url)
+		if err == nil {
+			return body, nil
+		}
+		lastErr = err
+
+		// Do not retry on context cancellation.
+		if ctx.Err() != nil {
+			return "", ctx.Err()
+		}
+		// Do not retry on 4xx — the client is at fault.
+		if errors.Is(err, errClientError) {
+			return "", err
+		}
+	}
+
+	return "", fmt.Errorf("%w after %d attempts: %w", ErrMaxRetries, maxAttempts, lastErr)
+}
+
+func doGet(ctx context.Context, client Client, url string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return "", fmt.Errorf("build request: %w", err)
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("GET %s: %w", url, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 500 {
+		return "", fmt.Errorf("GET %s: server error %d", url, resp.StatusCode)
+	}
+	if resp.StatusCode >= 400 {
+		return "", fmt.Errorf("%w: GET %s: client error %d", errClientError, url, resp.StatusCode)
+	}
+
+	raw, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("read body %s: %w", url, err)
+	}
+	return string(raw), nil
+}
+
+// WriteJSON writes v as JSON to w with the given HTTP status code and sets the
+// Content-Type header to application/json.
+func WriteJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(v)
+}
+
+// WriteError writes a JSON error object {"error": msg} with the given status.
+func WriteError(w http.ResponseWriter, status int, msg string) {
+	WriteJSON(w, status, map[string]string{"error": msg})
+}
+
+// maxBodyBytes is the limit applied by DecodeJSON to prevent unbounded reads.
+const maxBodyBytes = 1 << 20 // 1 MiB
+
+// DecodeJSON decodes a JSON request body into v. It enforces a 1 MiB size
+// limit and returns a descriptive error on any failure.
+func DecodeJSON(r *http.Request, v any) error {
+	r.Body = http.MaxBytesReader(nil, r.Body, maxBodyBytes)
+	dec := json.NewDecoder(r.Body)
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(v); err != nil {
+		return fmt.Errorf("decode JSON body: %w", err)
+	}
+	return nil
+}
--- a/v3/backend/internal/httputil/httputil_test.go
+++ b/v3/backend/internal/httputil/httputil_test.go
@@ -0,0 +1,181 @@
+package httputil_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/libnovel/backend/internal/httputil"
+)
+
+// ── RetryGet ──────────────────────────────────────────────────────────────────
+
+func TestRetryGet_ImmediateSuccess(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("hello"))
+	}))
+	defer srv.Close()
+
+	body, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 3, time.Millisecond)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if body != "hello" {
+		t.Errorf("want hello, got %q", body)
+	}
+}
+
+func TestRetryGet_RetriesOn5xx(t *testing.T) {
+	calls := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls++
+		if calls < 3 {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
+		w.Write([]byte("ok"))
+	}))
+	defer srv.Close()
+
+	body, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 5, time.Millisecond)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if body != "ok" {
+		t.Errorf("want ok, got %q", body)
+	}
+	if calls != 3 {
+		t.Errorf("want 3 calls, got %d", calls)
+	}
+}
+
+func TestRetryGet_MaxAttemptsExceeded(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer srv.Close()
+
+	_, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 3, time.Millisecond)
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+}
+
+func TestRetryGet_ContextCancelDuringBackoff(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}))
+	defer srv.Close()
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Cancel after first failed attempt hits the backoff wait.
+	go func() { time.Sleep(5 * time.Millisecond); cancel() }()
+
+	_, err := httputil.RetryGet(ctx, srv.Client(), srv.URL, 10, 500*time.Millisecond)
+	if err == nil {
+		t.Fatal("expected context cancellation error")
+	}
+}
+
+func TestRetryGet_NoRetryOn4xx(t *testing.T) {
+	calls := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls++
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv.Close()
+
+	_, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 5, time.Millisecond)
+	if err == nil {
+		t.Fatal("expected error for 404")
+	}
+	// 4xx is NOT retried — should be exactly 1 call.
+	if calls != 1 {
+		t.Errorf("want 1 call for 4xx, got %d", calls)
+	}
+}
+
+// ── WriteJSON ─────────────────────────────────────────────────────────────────
+
+func TestWriteJSON_SetsHeadersAndStatus(t *testing.T) {
+	rr := httptest.NewRecorder()
+	httputil.WriteJSON(rr, http.StatusCreated, map[string]string{"key": "val"})
+
+	if rr.Code != http.StatusCreated {
+		t.Errorf("status: want 201, got %d", rr.Code)
+	}
+	if ct := rr.Header().Get("Content-Type"); ct != "application/json" {
+		t.Errorf("Content-Type: want application/json, got %q", ct)
+	}
+	var got map[string]string
+	if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	if got["key"] != "val" {
+		t.Errorf("body key: want val, got %q", got["key"])
+	}
+}
+
+// ── WriteError ────────────────────────────────────────────────────────────────
+
+func TestWriteError_Format(t *testing.T) {
+	rr := httptest.NewRecorder()
+	httputil.WriteError(rr, http.StatusBadRequest, "bad input")
+
+	if rr.Code != http.StatusBadRequest {
+		t.Errorf("status: want 400, got %d", rr.Code)
+	}
+	var got map[string]string
+	json.NewDecoder(rr.Body).Decode(&got)
+	if got["error"] != "bad input" {
+		t.Errorf("error field: want bad input, got %q", got["error"])
+	}
+}
+
+// ── DecodeJSON ────────────────────────────────────────────────────────────────
+
+func TestDecodeJSON_HappyPath(t *testing.T) {
+	body := `{"name":"test","value":42}`
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
+	req.Header.Set("Content-Type", "application/json")
+
+	var payload struct {
+		Name  string `json:"name"`
+		Value int    `json:"value"`
+	}
+	if err := httputil.DecodeJSON(req, &payload); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if payload.Name != "test" || payload.Value != 42 {
+		t.Errorf("unexpected payload: %+v", payload)
+	}
+}
+
+func TestDecodeJSON_UnknownFieldReturnsError(t *testing.T) {
+	body := `{"name":"test","unknown_field":"boom"}`
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
+
+	var payload struct {
+		Name string `json:"name"`
+	}
+	if err := httputil.DecodeJSON(req, &payload); err == nil {
+		t.Fatal("expected error for unknown field, got nil")
+	}
+}
+
+func TestDecodeJSON_BodyTooLarge(t *testing.T) {
+	// Build a body > 1 MiB.
+	big := bytes.Repeat([]byte("a"), 2<<20)
+	req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(big))
+
+	var payload map[string]any
+	if err := httputil.DecodeJSON(req, &payload); err == nil {
+		t.Fatal("expected error for oversized body, got nil")
+	}
+}