feature/backend-rewrite #2

Open
kamil wants to merge 236 commits from feature/backend-rewrite into main
3 changed files with 167 additions and 13 deletions
Showing only changes of commit 041099598b - Show all commits

View File

@@ -237,6 +237,56 @@ func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time
return h.minio.PresignAudio(ctx, key, expires)
}
// ─── Scraping tasks ───────────────────────────────────────────────────────────
func (h *HybridStore) CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) {
return h.pb.CreateScrapingTask(ctx, kind, targetURL)
}
func (h *HybridStore) UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error {
data := map[string]interface{}{
"status": u.Status,
"books_found": u.BooksFound,
"chapters_scraped": u.ChaptersScraped,
"chapters_skipped": u.ChaptersSkipped,
"errors": u.Errors,
"error_message": u.ErrorMessage,
}
if !u.Finished.IsZero() {
data["finished"] = u.Finished.UTC().Format(time.RFC3339)
}
return h.pb.UpdateScrapingTask(ctx, id, data)
}
func (h *HybridStore) ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error) {
rows, err := h.pb.ListScrapingTasks(ctx)
if err != nil {
return nil, err
}
tasks := make([]ScrapeTask, 0, len(rows))
for _, r := range rows {
t := ScrapeTask{
ID: strVal(r, "id"),
Kind: strVal(r, "kind"),
TargetURL: strVal(r, "target_url"),
Status: strVal(r, "status"),
BooksFound: int(floatVal(r, "books_found")),
ChaptersScraped: int(floatVal(r, "chapters_scraped")),
ChaptersSkipped: int(floatVal(r, "chapters_skipped")),
Errors: int(floatVal(r, "errors")),
ErrorMessage: strVal(r, "error_message"),
}
if ts, ok := r["started"].(string); ok {
t.Started, _ = time.Parse(time.RFC3339, ts)
}
if ts, ok := r["finished"].(string); ok && ts != "" {
t.Finished, _ = time.Parse(time.RFC3339, ts)
}
tasks = append(tasks, t)
}
return tasks, nil
}
// ─── helpers ──────────────────────────────────────────────────────────────────
func recToBookMeta(rec map[string]interface{}) scraper.BookMeta {

View File

@@ -2,14 +2,18 @@
//
// Collections expected in PocketBase:
//
// books — slug(text,unique), title, author, cover, status, genres(json),
// summary, total_chapters(number), source_url, ranking(number), updated(date)
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
// ranking — rank(number), slug(text,unique), title, author, cover, status,
// genres(json), source_url, updated(date)
// progress — session_id(text), slug(text), chapter(number), updated(date)
// audio_cache — cache_key(text,unique), filename(text), updated(date)
// app_users — username(text,unique), password_hash(text), role(text), created(date)
// books — slug(text,unique), title, author, cover, status, genres(json),
// summary, total_chapters(number), source_url, ranking(number), updated(date)
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
// ranking — rank(number), slug(text,unique), title, author, cover, status,
// genres(json), source_url, updated(date)
// progress — session_id(text), slug(text), chapter(number), updated(date)
// audio_cache — cache_key(text,unique), filename(text), updated(date)
// app_users — username(text,unique), password_hash(text), role(text), created(date)
// scraping_tasks — id(auto), kind(text), target_url(text), status(text),
// books_found(number), chapters_scraped(number),
// chapters_skipped(number), errors(number),
// started(date), finished(date), error_message(text)
package storage
import (
@@ -345,6 +349,22 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
{"name": "created", "type": "date"},
},
},
{
"name": "scraping_tasks",
"type": "base",
"fields": []map[string]interface{}{
{"name": "kind", "type": "text", "required": true}, // "catalogue" | "book"
{"name": "target_url", "type": "text"}, // set for single-book scrapes
{"name": "status", "type": "text", "required": true}, // "running" | "done" | "failed" | "cancelled"
{"name": "books_found", "type": "number"},
{"name": "chapters_scraped", "type": "number"},
{"name": "chapters_skipped", "type": "number"},
{"name": "errors", "type": "number"},
{"name": "started", "type": "date"},
{"name": "finished", "type": "date"},
{"name": "error_message", "type": "text"},
},
},
}
for _, col := range collections {
name, _ := col["name"].(string)
@@ -590,6 +610,58 @@ func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (s
return filename, filename != "", nil
}
// ─── Scraping tasks ───────────────────────────────────────────────────────────
// CreateScrapingTask inserts a new scraping_tasks record with status="running"
// and returns the newly created record's ID.
func (s *PocketBaseStore) CreateScrapingTask(ctx context.Context, kind, targetURL string) (string, error) {
data := map[string]interface{}{
"kind": kind,
"target_url": targetURL,
"status": "running",
"books_found": 0,
"chapters_scraped": 0,
"chapters_skipped": 0,
"errors": 0,
"started": time.Now().UTC().Format(time.RFC3339),
}
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections/scraping_tasks/records", data)
if err != nil {
return "", err
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
return "", fmt.Errorf("pocketbase: CreateScrapingTask: status %d: %s", resp.StatusCode, b)
}
var rec map[string]interface{}
if err := json.Unmarshal(b, &rec); err != nil {
return "", fmt.Errorf("pocketbase: CreateScrapingTask: decode: %w", err)
}
id, _ := rec["id"].(string)
return id, nil
}
// UpdateScrapingTask patches counters on an existing scraping_tasks record.
func (s *PocketBaseStore) UpdateScrapingTask(ctx context.Context, id string, data map[string]interface{}) error {
resp, err := s.pb.do(ctx, http.MethodPatch,
fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), data)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pocketbase: UpdateScrapingTask id=%s: status %d: %s", id, resp.StatusCode, b)
}
return nil
}
// ListScrapingTasks returns all scraping_tasks sorted by started descending.
func (s *PocketBaseStore) ListScrapingTasks(ctx context.Context) ([]map[string]interface{}, error) {
return s.pb.listAll(ctx, "scraping_tasks", "", "-started")
}
// ─── helpers ──────────────────────────────────────────────────────────────────
// pbEsc escapes a string for use in a PocketBase filter expression.

View File

@@ -30,11 +30,33 @@ type ReadingProgress struct {
UpdatedAt time.Time `json:"updated_at"`
}
// AudioCacheEntry maps a (slug, chapter, voice, speed) tuple to a Kokoro
// download filename so audio is not re-generated after a server restart.
type AudioCacheEntry struct {
CacheKey string `json:"cache_key"`
Filename string `json:"filename"`
// ScrapeTask represents a single scraping job record from the scraping_tasks
// collection.
type ScrapeTask struct {
ID string `json:"id"`
Kind string `json:"kind"` // "catalogue" | "book"
TargetURL string `json:"target_url"` // non-empty for single-book scrapes
Status string `json:"status"` // "running" | "done" | "failed" | "cancelled"
BooksFound int `json:"books_found"`
ChaptersScraped int `json:"chapters_scraped"`
ChaptersSkipped int `json:"chapters_skipped"`
Errors int `json:"errors"`
Started time.Time `json:"started"`
Finished time.Time `json:"finished,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
}
// ScrapeTaskUpdate carries the fields that can be patched on a ScrapeTask.
// Zero-value fields are still sent; callers should only include keys they want
// to change via the map form used inside the store implementation.
type ScrapeTaskUpdate struct {
Status string
BooksFound int
ChaptersScraped int
ChaptersSkipped int
Errors int
Finished time.Time // zero = not finished yet
ErrorMessage string
}
// ─── Store interface ──────────────────────────────────────────────────────────
@@ -113,4 +135,14 @@ type Store interface {
// PresignAudio returns a presigned GET URL for an audio object.
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
// ── Scraping tasks ─────────────────────────────────────────────────────
// CreateScrapeTask inserts a new scraping_tasks record with status="running"
// and returns the assigned ID.
CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error)
// UpdateScrapeTask patches an existing task record.
UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error
// ListScrapeTasks returns all tasks sorted by started descending.
ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error)
}