feature/backend-rewrite #2
@@ -237,6 +237,56 @@ func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time
|
||||
return h.minio.PresignAudio(ctx, key, expires)
|
||||
}
|
||||
|
||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) {
|
||||
return h.pb.CreateScrapingTask(ctx, kind, targetURL)
|
||||
}
|
||||
|
||||
func (h *HybridStore) UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error {
|
||||
data := map[string]interface{}{
|
||||
"status": u.Status,
|
||||
"books_found": u.BooksFound,
|
||||
"chapters_scraped": u.ChaptersScraped,
|
||||
"chapters_skipped": u.ChaptersSkipped,
|
||||
"errors": u.Errors,
|
||||
"error_message": u.ErrorMessage,
|
||||
}
|
||||
if !u.Finished.IsZero() {
|
||||
data["finished"] = u.Finished.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return h.pb.UpdateScrapingTask(ctx, id, data)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error) {
|
||||
rows, err := h.pb.ListScrapingTasks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tasks := make([]ScrapeTask, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
t := ScrapeTask{
|
||||
ID: strVal(r, "id"),
|
||||
Kind: strVal(r, "kind"),
|
||||
TargetURL: strVal(r, "target_url"),
|
||||
Status: strVal(r, "status"),
|
||||
BooksFound: int(floatVal(r, "books_found")),
|
||||
ChaptersScraped: int(floatVal(r, "chapters_scraped")),
|
||||
ChaptersSkipped: int(floatVal(r, "chapters_skipped")),
|
||||
Errors: int(floatVal(r, "errors")),
|
||||
ErrorMessage: strVal(r, "error_message"),
|
||||
}
|
||||
if ts, ok := r["started"].(string); ok {
|
||||
t.Started, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
if ts, ok := r["finished"].(string); ok && ts != "" {
|
||||
t.Finished, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func recToBookMeta(rec map[string]interface{}) scraper.BookMeta {
|
||||
|
||||
@@ -2,14 +2,18 @@
|
||||
//
|
||||
// Collections expected in PocketBase:
|
||||
//
|
||||
// books — slug(text,unique), title, author, cover, status, genres(json),
|
||||
// summary, total_chapters(number), source_url, ranking(number), updated(date)
|
||||
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
|
||||
// ranking — rank(number), slug(text,unique), title, author, cover, status,
|
||||
// genres(json), source_url, updated(date)
|
||||
// progress — session_id(text), slug(text), chapter(number), updated(date)
|
||||
// audio_cache — cache_key(text,unique), filename(text), updated(date)
|
||||
// app_users — username(text,unique), password_hash(text), role(text), created(date)
|
||||
// books — slug(text,unique), title, author, cover, status, genres(json),
|
||||
// summary, total_chapters(number), source_url, ranking(number), updated(date)
|
||||
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
|
||||
// ranking — rank(number), slug(text,unique), title, author, cover, status,
|
||||
// genres(json), source_url, updated(date)
|
||||
// progress — session_id(text), slug(text), chapter(number), updated(date)
|
||||
// audio_cache — cache_key(text,unique), filename(text), updated(date)
|
||||
// app_users — username(text,unique), password_hash(text), role(text), created(date)
|
||||
// scraping_tasks — id(auto), kind(text), target_url(text), status(text),
|
||||
// books_found(number), chapters_scraped(number),
|
||||
// chapters_skipped(number), errors(number),
|
||||
// started(date), finished(date), error_message(text)
|
||||
package storage
|
||||
|
||||
import (
|
||||
@@ -345,6 +349,22 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
|
||||
{"name": "created", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "scraping_tasks",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "kind", "type": "text", "required": true}, // "catalogue" | "book"
|
||||
{"name": "target_url", "type": "text"}, // set for single-book scrapes
|
||||
{"name": "status", "type": "text", "required": true}, // "running" | "done" | "failed" | "cancelled"
|
||||
{"name": "books_found", "type": "number"},
|
||||
{"name": "chapters_scraped", "type": "number"},
|
||||
{"name": "chapters_skipped", "type": "number"},
|
||||
{"name": "errors", "type": "number"},
|
||||
{"name": "started", "type": "date"},
|
||||
{"name": "finished", "type": "date"},
|
||||
{"name": "error_message", "type": "text"},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, col := range collections {
|
||||
name, _ := col["name"].(string)
|
||||
@@ -590,6 +610,58 @@ func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (s
|
||||
return filename, filename != "", nil
|
||||
}
|
||||
|
||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
// CreateScrapingTask inserts a new scraping_tasks record with status="running"
|
||||
// and returns the newly created record's ID.
|
||||
func (s *PocketBaseStore) CreateScrapingTask(ctx context.Context, kind, targetURL string) (string, error) {
|
||||
data := map[string]interface{}{
|
||||
"kind": kind,
|
||||
"target_url": targetURL,
|
||||
"status": "running",
|
||||
"books_found": 0,
|
||||
"chapters_scraped": 0,
|
||||
"chapters_skipped": 0,
|
||||
"errors": 0,
|
||||
"started": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections/scraping_tasks/records", data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
||||
return "", fmt.Errorf("pocketbase: CreateScrapingTask: status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
var rec map[string]interface{}
|
||||
if err := json.Unmarshal(b, &rec); err != nil {
|
||||
return "", fmt.Errorf("pocketbase: CreateScrapingTask: decode: %w", err)
|
||||
}
|
||||
id, _ := rec["id"].(string)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// UpdateScrapingTask patches counters on an existing scraping_tasks record.
|
||||
func (s *PocketBaseStore) UpdateScrapingTask(ctx context.Context, id string, data map[string]interface{}) error {
|
||||
resp, err := s.pb.do(ctx, http.MethodPatch,
|
||||
fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pocketbase: UpdateScrapingTask id=%s: status %d: %s", id, resp.StatusCode, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListScrapingTasks returns all scraping_tasks sorted by started descending.
|
||||
func (s *PocketBaseStore) ListScrapingTasks(ctx context.Context) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "scraping_tasks", "", "-started")
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// pbEsc escapes a string for use in a PocketBase filter expression.
|
||||
|
||||
@@ -30,11 +30,33 @@ type ReadingProgress struct {
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// AudioCacheEntry maps a (slug, chapter, voice, speed) tuple to a Kokoro
|
||||
// download filename so audio is not re-generated after a server restart.
|
||||
type AudioCacheEntry struct {
|
||||
CacheKey string `json:"cache_key"`
|
||||
Filename string `json:"filename"`
|
||||
// ScrapeTask represents a single scraping job record from the scraping_tasks
|
||||
// collection.
|
||||
type ScrapeTask struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"` // "catalogue" | "book"
|
||||
TargetURL string `json:"target_url"` // non-empty for single-book scrapes
|
||||
Status string `json:"status"` // "running" | "done" | "failed" | "cancelled"
|
||||
BooksFound int `json:"books_found"`
|
||||
ChaptersScraped int `json:"chapters_scraped"`
|
||||
ChaptersSkipped int `json:"chapters_skipped"`
|
||||
Errors int `json:"errors"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished time.Time `json:"finished,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
}
|
||||
|
||||
// ScrapeTaskUpdate carries the fields that can be patched on a ScrapeTask.
|
||||
// Zero-value fields are still sent; callers should only include keys they want
|
||||
// to change via the map form used inside the store implementation.
|
||||
type ScrapeTaskUpdate struct {
|
||||
Status string
|
||||
BooksFound int
|
||||
ChaptersScraped int
|
||||
ChaptersSkipped int
|
||||
Errors int
|
||||
Finished time.Time // zero = not finished yet
|
||||
ErrorMessage string
|
||||
}
|
||||
|
||||
// ─── Store interface ──────────────────────────────────────────────────────────
|
||||
@@ -113,4 +135,14 @@ type Store interface {
|
||||
|
||||
// PresignAudio returns a presigned GET URL for an audio object.
|
||||
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
|
||||
|
||||
// ── Scraping tasks ─────────────────────────────────────────────────────
|
||||
|
||||
// CreateScrapeTask inserts a new scraping_tasks record with status="running"
|
||||
// and returns the assigned ID.
|
||||
CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error)
|
||||
// UpdateScrapeTask patches an existing task record.
|
||||
UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error
|
||||
// ListScrapeTasks returns all tasks sorted by started descending.
|
||||
ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user