feat: add exponential backoff, some UI elements to see the resut of a scrape

This commit is contained in:
Admin
2026-02-26 18:51:32 +05:00
parent d68ea71239
commit e6e6f7dc4d
12 changed files with 462 additions and 153 deletions

View File

@@ -18,6 +18,7 @@ import (
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/writer"
)
// Server wraps an HTTP mux with the scraping endpoints.
@@ -26,6 +27,7 @@ type Server struct {
oCfg orchestrator.Config
novel scraper.NovelScraper
log *slog.Logger
writer *writer.Writer
mu sync.Mutex
running bool
}
@@ -33,10 +35,11 @@ type Server struct {
// New creates a new Server.
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger) *Server {
return &Server{
addr: addr,
oCfg: oCfg,
novel: novel,
log: log,
addr: addr,
oCfg: oCfg,
novel: novel,
log: log,
writer: writer.New(oCfg.StaticRoot),
}
}
@@ -47,6 +50,12 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /health", s.handleHealth)
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
// UI routes
mux.HandleFunc("GET /", s.handleHome)
mux.HandleFunc("GET /books/{slug}", s.handleBook)
mux.HandleFunc("GET /books/{slug}/chapters/{n}", s.handleChapter)
mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
srv := &http.Server{
Addr: s.addr,