Compare commits
210 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
23345e22e6 | ||
|
|
c7b3495a23 | ||
|
|
83a5910a59 | ||
|
|
0f6639aae7 | ||
|
|
88a25bc33e | ||
|
|
73ad4ece49 | ||
|
|
52f876d8e8 | ||
|
|
72eed89f59 | ||
|
|
12bb0db5f0 | ||
|
|
5ec1773768 | ||
|
|
fb8f1dfe25 | ||
|
|
3a2d113b1b | ||
|
|
0dcfdff65b | ||
|
|
1766011b47 | ||
|
|
a6f800b0d7 | ||
|
|
af9639af05 | ||
|
|
bfc08a2df2 | ||
|
|
dc3bc3ebf2 | ||
|
|
e9d7293d37 | ||
|
|
410af8f236 | ||
|
|
264c00c765 | ||
|
|
e4c72011eb | ||
|
|
6365b14ece | ||
|
|
7da5582075 | ||
|
|
dae841e317 | ||
|
|
16b2bfffa6 | ||
|
|
57be674f44 | ||
|
|
93390fab64 | ||
|
|
072517135f | ||
|
|
fe7c7acbb7 | ||
|
|
d4cce915d9 | ||
|
|
ac24e86f7d | ||
|
|
e9bb387f71 | ||
|
|
d7319b3f7c | ||
|
|
f380c85815 | ||
|
|
9d1b340b83 | ||
|
|
a307ddc9f5 | ||
|
|
004d1b6d9d | ||
|
|
7f20411f50 | ||
|
|
6e6c581904 | ||
|
|
cecedc8687 | ||
|
|
a88e98a436 | ||
|
|
d3ae86d55b | ||
|
|
5ad5c2dbce | ||
|
|
0de91dcc0c | ||
|
|
8e3e9ef31d | ||
|
|
3c5edd5742 | ||
|
|
2142e82fe4 | ||
|
|
88cde88f69 | ||
|
|
ffcc3981f2 | ||
|
|
a7b4694e60 | ||
|
|
8c895c6ba1 | ||
|
|
83059c8a9d | ||
|
|
b54ebf60b5 | ||
|
|
e027afe89d | ||
|
|
9fc2054e36 | ||
|
|
9a43b2190e | ||
|
|
5a7d7ce3b9 | ||
|
|
ce3eef1298 | ||
|
|
5d9b41bcf2 | ||
|
|
47268dea67 | ||
|
|
57591766f2 | ||
|
|
fa8fb96631 | ||
|
|
5ba84f7945 | ||
|
|
2793ad8cfa | ||
|
|
e43699747d | ||
|
|
1e85f1c0bc | ||
|
|
0c2349f259 | ||
|
|
c9252b5953 | ||
|
|
7efeee3fc2 | ||
|
|
9a05708019 | ||
|
|
24cb18e0fe | ||
|
|
71ba882858 | ||
|
|
c35f099f50 | ||
|
|
4df287ace4 | ||
|
|
0df45de2b6 | ||
|
|
825fb04c0d | ||
|
|
fc5cd30c93 | ||
|
|
37bd73651a | ||
|
|
466e289b68 | ||
|
|
bb604019fc | ||
|
|
0745178d9e | ||
|
|
603cd2bb02 | ||
|
|
228d4902bb | ||
|
|
884c82b2c3 | ||
|
|
c6536d5b9f | ||
|
|
460e7553bf | ||
|
|
89f0dfb113 | ||
|
|
88644341d8 | ||
|
|
992eb823f2 | ||
|
|
f51113a2f8 | ||
|
|
1eb70e9b9b | ||
|
|
70dd14e5c8 | ||
|
|
8096827c78 | ||
|
|
669fd765ee | ||
|
|
314af375d5 | ||
|
|
20c45e2676 | ||
|
|
09981a5f4d | ||
|
|
de9e0b4246 | ||
|
|
a72c1f6b52 | ||
|
|
5d3a1a09ef | ||
|
|
39ad0d6c11 | ||
|
|
765b37aea3 | ||
|
|
aff6de9b45 | ||
|
|
ec66e86a18 | ||
|
|
9b7cdad71a | ||
|
|
8f0a2f7e92 | ||
|
|
08d4718245 | ||
|
|
60a9540ef7 | ||
|
|
76d616a308 | ||
|
|
e723459507 | ||
|
|
b3358ac1d2 | ||
|
|
c0d33720e9 | ||
|
|
a5c603e7a6 | ||
|
|
219d4fb214 | ||
|
|
cec0dfe64a | ||
|
|
54616b82d7 | ||
|
|
ce5db37226 | ||
|
|
60bc8e5749 | ||
|
|
b4be0803aa | ||
|
|
12eca865ce | ||
|
|
589f39b49e | ||
|
|
53083429a0 | ||
|
|
70c8db28f9 | ||
|
|
1d00fd4e2e | ||
|
|
a54d8d43aa | ||
|
|
97e7a8dc02 | ||
|
|
fb6b364382 | ||
|
|
7b48707cd9 | ||
|
|
b0547c1b43 | ||
|
|
acbfafb8cd | ||
|
|
c8e0cf2813 | ||
|
|
3899a96576 | ||
|
|
1e7f396b2d | ||
|
|
0eee2eedf3 | ||
|
|
80da1bb3e2 | ||
|
|
9f3e895fa8 | ||
|
|
cf0c0dfaaf | ||
|
|
0402c408e4 | ||
|
|
d14644238f | ||
|
|
8de374cd35 | ||
|
|
82186cfd6d | ||
|
|
b87e758303 | ||
|
|
901b18ee13 | ||
|
|
034e670795 | ||
|
|
0d7b985469 | ||
|
|
53af7515a3 | ||
|
|
11a846d043 | ||
|
|
bf2ffa54db | ||
|
|
fe204598a2 | ||
|
|
9906c7d862 | ||
|
|
06feb91f4f | ||
|
|
5a7751e6d1 | ||
|
|
555973c053 | ||
|
|
c2d6ce1c5b | ||
|
|
8edad54b10 | ||
|
|
48d8fdb6b9 | ||
|
|
1b05b6ebc6 | ||
|
|
cabdd3ffdd | ||
|
|
f80b83309a | ||
|
|
49ba2c27c2 | ||
|
|
353d7397eb | ||
|
|
89ff90629f | ||
|
|
f6febfdb5e | ||
|
|
2c43907e34 | ||
|
|
0e868506ca | ||
|
|
1b234754e8 | ||
|
|
041099598b | ||
|
|
333c8ad868 | ||
|
|
d16ae00537 | ||
|
|
d16313bb6c | ||
|
|
1bab7028c6 | ||
|
|
6520fb9a50 | ||
|
|
7acf04fb9f | ||
|
|
c2bcb2b0a6 | ||
|
|
cfd893d24b | ||
|
|
cff0c78b4f | ||
|
|
d89cefe975 | ||
|
|
a0344b36d7 | ||
|
|
af3c487afb | ||
|
|
b8d4d94b18 | ||
|
|
56bf4dde22 | ||
|
|
2f0857be45 | ||
|
|
bf5774d8d0 | ||
|
|
5131ae0bc4 | ||
|
|
9fa0776258 | ||
|
|
f265d9d020 | ||
|
|
3c26dfe2c0 | ||
|
|
1820fa7303 | ||
|
|
38e400a4c7 | ||
|
|
cb90771248 | ||
|
|
59b1cfab1d | ||
|
|
f95ad3ed29 | ||
|
|
e4c4f8de66 | ||
|
|
4f84bd29c9 | ||
|
|
6bf79ab392 | ||
|
|
4ae6f0ab42 | ||
|
|
33e2a4dc01 | ||
|
|
cb4be0848f | ||
|
|
2f948f2a50 | ||
|
|
baab66823d | ||
|
|
11d2eaa0e5 | ||
|
|
9c115f00c4 | ||
|
|
5ac89da513 | ||
|
|
af86c6f96f | ||
|
|
da4a182f85 | ||
|
|
18e76c9668 | ||
|
|
9add9033b9 | ||
|
|
66d8481637 | ||
|
|
7f92a58fd7 |
50
.env.example
50
.env.example
@@ -1,6 +1,26 @@
|
||||
# libnovel scraper — environment overrides
|
||||
# Copy to .env and adjust values; do NOT commit this file with real secrets.
|
||||
|
||||
# ── Service ports (host-side) ─────────────────────────────────────────────────
|
||||
# Port the scraper HTTP API listens on (default 8080)
|
||||
SCRAPER_PORT=8080
|
||||
|
||||
# Port PocketBase listens on (default 8090)
|
||||
POCKETBASE_PORT=8090
|
||||
|
||||
# Port MinIO S3 API listens on (default 9000)
|
||||
MINIO_PORT=9000
|
||||
|
||||
# Port MinIO web console listens on (default 9001)
|
||||
MINIO_CONSOLE_PORT=9001
|
||||
|
||||
# Port Browserless Chrome listens on (default 3030)
|
||||
BROWSERLESS_PORT=3030
|
||||
|
||||
# Port the SvelteKit UI listens on (default 3000)
|
||||
UI_PORT=3000
|
||||
|
||||
# ── Browserless ───────────────────────────────────────────────────────────────
|
||||
# Browserless API token (leave empty to disable auth)
|
||||
BROWSERLESS_TOKEN=
|
||||
|
||||
@@ -19,10 +39,7 @@ ERROR_ALERT_URL=
|
||||
# Which Browserless strategy the scraper uses: content | scrape | cdp | direct
|
||||
BROWSERLESS_STRATEGY=direct
|
||||
|
||||
# Strategy for URL retrieval (chapter list). Uses browserless content strategy by default.
|
||||
# Set to direct to use plain HTTP, or content/scrape/cdp for browserless.
|
||||
BROWSERLESS_URL_STRATEGY=content
|
||||
|
||||
# ── Scraper ───────────────────────────────────────────────────────────────────
|
||||
# Chapter worker goroutines (0 = NumCPU inside the container)
|
||||
SCRAPER_WORKERS=0
|
||||
|
||||
@@ -39,3 +56,28 @@ KOKORO_URL=http://kokoro:8880
|
||||
# Single voices: af_bella, af_sky, af_heart, am_adam, …
|
||||
# Mixed voices: af_bella+af_sky or af_bella(2)+af_sky(1) (weighted blend)
|
||||
KOKORO_VOICE=af_bella
|
||||
|
||||
# ── MinIO / S3 object storage ─────────────────────────────────────────────────
|
||||
MINIO_ROOT_USER=admin
|
||||
MINIO_ROOT_PASSWORD=changeme123
|
||||
MINIO_BUCKET_CHAPTERS=libnovel-chapters
|
||||
MINIO_BUCKET_AUDIO=libnovel-audio
|
||||
|
||||
# ── PocketBase ────────────────────────────────────────────────────────────────
|
||||
# Admin credentials (used by scraper + UI server-side)
|
||||
POCKETBASE_ADMIN_EMAIL=admin@libnovel.local
|
||||
POCKETBASE_ADMIN_PASSWORD=changeme123
|
||||
|
||||
# ── SvelteKit UI ─────────────────────────────────────────────────────────────
|
||||
# Internal URL the SvelteKit server uses to reach the scraper API.
|
||||
# In docker-compose this is http://scraper:8080 (wired automatically).
|
||||
# Override here only if running the UI outside of docker-compose.
|
||||
SCRAPER_API_URL=http://localhost:8080
|
||||
|
||||
# Internal URL the SvelteKit server uses to reach PocketBase.
|
||||
# In docker-compose this is http://pocketbase:8090 (wired automatically).
|
||||
POCKETBASE_URL=http://localhost:8090
|
||||
|
||||
# Public MinIO URL reachable from the browser (for audio/presigned URLs).
|
||||
# In production, point this at your MinIO reverse-proxy or CDN domain.
|
||||
PUBLIC_MINIO_PUBLIC_URL=http://localhost:9000
|
||||
|
||||
76
.gitea/workflows/ci-scraper.yaml
Normal file
76
.gitea/workflows/ci-scraper.yaml
Normal file
@@ -0,0 +1,76 @@
|
||||
name: CI / Scraper
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main", "master", "v2"]
|
||||
paths:
|
||||
- "scraper/**"
|
||||
- ".gitea/workflows/ci-scraper.yaml"
|
||||
pull_request:
|
||||
branches: ["main", "master", "v2"]
|
||||
paths:
|
||||
- "scraper/**"
|
||||
- ".gitea/workflows/ci-scraper.yaml"
|
||||
|
||||
concurrency:
|
||||
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── lint & vet ───────────────────────────────────────────────────────────────
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: scraper/go.mod
|
||||
cache-dependency-path: scraper/go.sum
|
||||
|
||||
- name: go vet
|
||||
working-directory: scraper
|
||||
run: |
|
||||
go vet ./...
|
||||
go vet -tags integration ./...
|
||||
|
||||
# ── tests ────────────────────────────────────────────────────────────────────
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: scraper/go.mod
|
||||
cache-dependency-path: scraper/go.sum
|
||||
|
||||
- name: Run tests
|
||||
working-directory: scraper
|
||||
run: go test -short -race -count=1 -timeout=60s ./...
|
||||
|
||||
# ── push to Docker Hub ───────────────────────────────────────────────────────
|
||||
docker:
|
||||
name: Docker Push
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint, test]
|
||||
if: gitea.event_name == 'push'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: scraper
|
||||
push: true
|
||||
tags: |
|
||||
${{ secrets.DOCKER_USER }}/libnovel-scraper:latest
|
||||
${{ secrets.DOCKER_USER }}/libnovel-scraper:${{ gitea.sha }}
|
||||
67
.gitea/workflows/ci-ui.yaml
Normal file
67
.gitea/workflows/ci-ui.yaml
Normal file
@@ -0,0 +1,67 @@
|
||||
name: CI / UI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main", "master", "v2"]
|
||||
paths:
|
||||
- "ui/**"
|
||||
- ".gitea/workflows/ci-ui.yaml"
|
||||
pull_request:
|
||||
branches: ["main", "master", "v2"]
|
||||
paths:
|
||||
- "ui/**"
|
||||
- ".gitea/workflows/ci-ui.yaml"
|
||||
|
||||
concurrency:
|
||||
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── type-check & build ───────────────────────────────────────────────────────
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ui
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
cache-dependency-path: ui/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Type check
|
||||
run: npm run check
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
# ── push to Docker Hub ───────────────────────────────────────────────────────
|
||||
docker:
|
||||
name: Docker Push
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
if: gitea.event_name == 'push'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ui
|
||||
push: true
|
||||
tags: |
|
||||
${{ secrets.DOCKER_USER }}/libnovel-ui:latest
|
||||
${{ secrets.DOCKER_USER }}/libnovel-ui:${{ gitea.sha }}
|
||||
@@ -1,106 +0,0 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main", "master"]
|
||||
paths:
|
||||
- "scraper/**"
|
||||
- ".gitea/workflows/**"
|
||||
pull_request:
|
||||
branches: ["main", "master"]
|
||||
paths:
|
||||
- "scraper/**"
|
||||
- ".gitea/workflows/**"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: scraper
|
||||
|
||||
jobs:
|
||||
# ── lint & vet ───────────────────────────────────────────────────────────────
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: scraper/go.mod
|
||||
cache-dependency-path: scraper/go.sum
|
||||
|
||||
- name: go vet
|
||||
run: go vet ./...
|
||||
|
||||
- name: staticcheck
|
||||
run: |
|
||||
go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
staticcheck ./...
|
||||
|
||||
# ── tests ────────────────────────────────────────────────────────────────────
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: scraper/go.mod
|
||||
cache-dependency-path: scraper/go.sum
|
||||
|
||||
- name: Run tests
|
||||
run: go test -race -count=1 -timeout=60s ./...
|
||||
|
||||
# ── build binary ─────────────────────────────────────────────────────────────
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint, test]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: scraper/go.mod
|
||||
cache-dependency-path: scraper/go.sum
|
||||
|
||||
- name: Build binary
|
||||
run: |
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
||||
go build -ldflags="-s -w" -o bin/scraper ./cmd/scraper
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: scraper-linux-amd64
|
||||
path: scraper/bin/scraper
|
||||
retention-days: 7
|
||||
|
||||
# ── docker build (& push) ────────────────────────────────────────────────────
|
||||
# Uncomment once the runner has Docker available and a registry is configured.
|
||||
#
|
||||
# docker:
|
||||
# name: Docker
|
||||
# runs-on: ubuntu-latest
|
||||
# needs: [lint, test]
|
||||
# # Only push images on commits to the default branch, not on PRs.
|
||||
# # if: github.event_name == 'push'
|
||||
# steps:
|
||||
# - uses: actions/checkout@v4
|
||||
#
|
||||
# - name: Log in to Gitea registry
|
||||
# uses: docker/login-action@v3
|
||||
# with:
|
||||
# registry: gitea.kalekber.cc
|
||||
# username: ${{ secrets.REGISTRY_USER }}
|
||||
# password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
#
|
||||
# - name: Build and push
|
||||
# uses: docker/build-push-action@v5
|
||||
# with:
|
||||
# context: ./scraper
|
||||
# push: true
|
||||
# tags: |
|
||||
# gitea.kalekber.cc/kamil/libnovel:latest
|
||||
# gitea.kalekber.cc/kamil/libnovel:${{ gitea.sha }}
|
||||
63
.gitea/workflows/ios.yaml
Normal file
63
.gitea/workflows/ios.yaml
Normal file
@@ -0,0 +1,63 @@
|
||||
name: iOS CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["v2", "main"]
|
||||
paths:
|
||||
- "ios/**"
|
||||
- "justfile"
|
||||
- ".gitea/workflows/ios.yaml"
|
||||
- ".gitea/workflows/ios-release.yaml"
|
||||
pull_request:
|
||||
branches: ["v2", "main"]
|
||||
paths:
|
||||
- "ios/**"
|
||||
- "justfile"
|
||||
- ".gitea/workflows/ios.yaml"
|
||||
- ".gitea/workflows/ios-release.yaml"
|
||||
|
||||
concurrency:
|
||||
group: ios-macos-runner
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── build (simulator) ─────────────────────────────────────────────────────
|
||||
build:
|
||||
name: Build
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install just
|
||||
run: command -v just || brew install just
|
||||
|
||||
- name: Build (simulator)
|
||||
env:
|
||||
USER: runner
|
||||
run: just ios-build
|
||||
|
||||
# ── unit tests ────────────────────────────────────────────────────────────
|
||||
test:
|
||||
name: Test
|
||||
runs-on: macos-latest
|
||||
needs: build
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install just
|
||||
run: command -v just || brew install just
|
||||
|
||||
- name: Run unit tests
|
||||
env:
|
||||
USER: runner
|
||||
run: just ios-test
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-results
|
||||
path: ios/LibNovel/test-results.xml
|
||||
retention-days: 7
|
||||
65
.gitea/workflows/release-scraper.yaml
Normal file
65
.gitea/workflows/release-scraper.yaml
Normal file
@@ -0,0 +1,65 @@
|
||||
name: Release / Scraper
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
concurrency:
|
||||
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── lint & test ──────────────────────────────────────────────────────────────
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: scraper/go.mod
|
||||
cache-dependency-path: scraper/go.sum
|
||||
|
||||
- name: go vet
|
||||
working-directory: scraper
|
||||
run: |
|
||||
go vet ./...
|
||||
go vet -tags integration ./...
|
||||
|
||||
- name: Run tests
|
||||
working-directory: scraper
|
||||
run: go test -short -race -count=1 -timeout=60s ./...
|
||||
|
||||
# ── docker build & push ──────────────────────────────────────────────────────
|
||||
docker:
|
||||
name: Docker
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USER }}/libnovel-scraper
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: scraper
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
68
.gitea/workflows/release-ui.yaml
Normal file
68
.gitea/workflows/release-ui.yaml
Normal file
@@ -0,0 +1,68 @@
|
||||
name: Release / UI
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
concurrency:
|
||||
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── type-check & build ───────────────────────────────────────────────────────
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ui
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
cache-dependency-path: ui/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Type check
|
||||
run: npm run check
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
# ── docker build & push ──────────────────────────────────────────────────────
|
||||
docker:
|
||||
name: Docker
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USER }}/libnovel-ui
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ui
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -6,6 +6,7 @@
|
||||
|
||||
# ── Compiled binaries ──────────────────────────────────────────────────────────
|
||||
scraper/bin/
|
||||
scraper/scraper
|
||||
|
||||
# ── Scraped output (large, machine-generated) ──────────────────────────────────
|
||||
|
||||
|
||||
170
AGENTS.md
170
AGENTS.md
@@ -1,28 +1,43 @@
|
||||
# libnovel Project
|
||||
|
||||
Go web scraper for novelfire.net with TTS support via Kokoro-FastAPI.
|
||||
Go web scraper for novelfire.net with TTS support via Kokoro-FastAPI. Structured data in PocketBase, binary blobs (chapters, audio, browse snapshots) in MinIO. SvelteKit frontend.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
scraper/
|
||||
├── cmd/scraper/main.go # Entry point: 'run' (one-shot) and 'serve' (HTTP server)
|
||||
├── cmd/scraper/main.go # Entry point: run | refresh | serve | save-browse
|
||||
├── internal/
|
||||
│ ├── orchestrator/orchestrator.go # Coordinates catalogue walk, metadata extraction, chapter scraping
|
||||
│ ├── browser/ # Browser client (content/scrape/cdp strategies) via Browserless
|
||||
│ ├── novelfire/scraper.go # novelfire.net specific scraping logic
|
||||
│ ├── server/server.go # HTTP API (POST /scrape, POST /scrape/book)
|
||||
│ ├── writer/writer.go # File writer (metadata.yaml, chapter .md files)
|
||||
│ └── scraper/interfaces.go # NovelScraper interface definition
|
||||
└── static/books/ # Output directory for scraped content
|
||||
│ ├── orchestrator/orchestrator.go # Catalogue walk → per-book metadata goroutines → chapter worker pool
|
||||
│ ├── browser/ # BrowserClient interface + direct HTTP (production) + Browserless variants
|
||||
│ ├── novelfire/scraper.go # novelfire.net scraping (catalogue, metadata, chapters, ranking)
|
||||
│ ├── server/ # HTTP API server (server.go + 6 handler files)
|
||||
│ │ ├── server.go # Server struct, route registration, ListenAndServe
|
||||
│ │ ├── handlers_scrape.go # POST /scrape, /scrape/book, /scrape/book/range; job status/tasks
|
||||
│ │ ├── handlers_browse.go # GET /api/browse, /api/search, /api/cover — MinIO-cached browse pages
|
||||
│ │ ├── handlers_preview.go # GET /api/book-preview, /api/chapter-text-preview — live scrape, no store writes
|
||||
│ │ ├── handlers_audio.go # POST /api/audio, GET /api/audio-proxy, voice samples, presign
|
||||
│ │ ├── handlers_progress.go # GET/POST/DELETE /api/progress
|
||||
│ │ ├── handlers_ranking.go # GET /api/ranking, /api/cover
|
||||
│ │ └── helpers.go # stripMarkdown, hardcoded voice list fallback
|
||||
│ ├── storage/ # Persistence layer (PocketBase + MinIO)
|
||||
│ │ ├── store.go # Store interface — single abstraction for server + orchestrator
|
||||
│ │ ├── hybrid.go # HybridStore: routes structured data → PocketBase, blobs → MinIO
|
||||
│ │ ├── pocketbase.go # PocketBase REST admin client (7 collections, auth, schema bootstrap)
|
||||
│ │ ├── minio.go # MinIO client (3 buckets: chapters, audio, browse)
|
||||
│ │ └── coverutil.go # Best-effort cover image downloader → browse bucket
|
||||
│ └── scraper/
|
||||
│ ├── interfaces.go # NovelScraper interface + domain types (BookMeta, ChapterRef, etc.)
|
||||
│ └── htmlutil/htmlutil.go # HTML parsing helpers (NodeToMarkdown, ResolveURL, etc.)
|
||||
```
|
||||
|
||||
## Key Concepts
|
||||
|
||||
- **Orchestrator**: Manages concurrency - catalogue streaming → per-book metadata goroutines → chapter worker pool
|
||||
- **Browser Client**: 3 strategies (content/scrape/cdp) via Browserless Chrome container
|
||||
- **Writer**: Writes metadata.yaml and chapter markdown files to `static/books/{slug}/vol-0/1-50/`
|
||||
- **Server**: HTTP API with async scrape jobs, UI for browsing books/chapters, chapter-text endpoint for TTS
|
||||
- **Orchestrator**: Catalogue stream → per-book goroutines (metadata + chapter list) → shared chapter work channel → N worker goroutines (chapter text). Scrape jobs tracked in PocketBase `scraping_tasks`.
|
||||
- **Storage**: `HybridStore` implements the `Store` interface. PocketBase holds structured records (`books`, `chapters_idx`, `ranking`, `progress`, `audio_cache`, `app_users`, `scraping_tasks`). MinIO holds blobs (chapter markdown, audio MP3s, browse HTML snapshots, cover images).
|
||||
- **Browser Client**: Production uses `NewDirectHTTPClient` (plain HTTP, no Browserless). Browserless variants (content/scrape/cdp) exist in `browser/` but are only wired for the `save-browse` subcommand.
|
||||
- **Preview**: `GET /api/book-preview/{slug}` scrapes metadata + chapter list live without persisting anything — used when a book is not yet in the library. On first visit, metadata and chapter index are auto-saved to PocketBase in the background.
|
||||
- **Server**: 24 HTTP endpoints. Async scrape jobs (mutex, 409 on concurrent), in-flight dedup for audio generation, MinIO-backed browse page cache with mem-cache fallback.
|
||||
|
||||
## Commands
|
||||
|
||||
@@ -30,60 +45,127 @@ scraper/
|
||||
# Build
|
||||
cd scraper && go build -o bin/scraper ./cmd/scraper
|
||||
|
||||
# One-shot scrape (full catalogue)
|
||||
# Full catalogue scrape (one-shot)
|
||||
./bin/scraper run
|
||||
|
||||
# Single book
|
||||
./bin/scraper run --url https://novelfire.net/book/xxx
|
||||
|
||||
# Re-scrape a book already in the DB (uses stored source_url)
|
||||
./bin/scraper refresh <slug>
|
||||
|
||||
# HTTP server
|
||||
./bin/scraper serve
|
||||
|
||||
# Tests
|
||||
# Capture browse pages to MinIO via SingleFile CLI (requires SINGLEFILE_PATH + BROWSERLESS_URL)
|
||||
./bin/scraper save-browse
|
||||
|
||||
# Tests (unit only — integration tests require live services)
|
||||
cd scraper && go test ./... -short
|
||||
|
||||
# All tests (requires MinIO + PocketBase + Browserless)
|
||||
cd scraper && go test ./...
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
### Scraper (Go)
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| BROWSERLESS_URL | Browserless Chrome endpoint | http://localhost:3030 |
|
||||
| BROWSERLESS_STRATEGY | content \| scrape \| cdp | content |
|
||||
| SCRAPER_WORKERS | Chapter goroutines | NumCPU |
|
||||
| SCRAPER_STATIC_ROOT | Output directory | ./static/books |
|
||||
| SCRAPER_HTTP_ADDR | HTTP listen address | :8080 |
|
||||
| KOKORO_URL | Kokoro TTS endpoint | http://localhost:8880 |
|
||||
| KOKORO_VOICE | Default TTS voice | af_bella |
|
||||
| LOG_LEVEL | debug \| info \| warn \| error | info |
|
||||
| `LOG_LEVEL` | `debug\|info\|warn\|error` | `info` |
|
||||
| `SCRAPER_HTTP_ADDR` | HTTP listen address | `:8080` |
|
||||
| `SCRAPER_WORKERS` | Chapter goroutines | `NumCPU` |
|
||||
| `SCRAPER_TIMEOUT` | Per-request HTTP timeout (seconds) | `90` |
|
||||
| `KOKORO_URL` | Kokoro-FastAPI TTS base URL | `https://kokoro.kalekber.cc` |
|
||||
| `KOKORO_VOICE` | Default TTS voice | `af_bella` |
|
||||
| `MINIO_ENDPOINT` | MinIO S3 API host:port | `localhost:9000` |
|
||||
| `MINIO_PUBLIC_ENDPOINT` | Public MinIO endpoint for presigned URLs | `""` |
|
||||
| `MINIO_ACCESS_KEY` | MinIO access key | `admin` |
|
||||
| `MINIO_SECRET_KEY` | MinIO secret key | `changeme123` |
|
||||
| `MINIO_USE_SSL` | TLS for internal MinIO connection | `false` |
|
||||
| `MINIO_PUBLIC_USE_SSL` | TLS for public presigned URL endpoint | `true` |
|
||||
| `MINIO_BUCKET_CHAPTERS` | Chapter markdown bucket | `libnovel-chapters` |
|
||||
| `MINIO_BUCKET_AUDIO` | Audio MP3 bucket | `libnovel-audio` |
|
||||
| `MINIO_BUCKET_BROWSE` | Browse HTML + cover image bucket | `libnovel-browse` |
|
||||
| `POCKETBASE_URL` | PocketBase base URL | `http://localhost:8090` |
|
||||
| `POCKETBASE_ADMIN_EMAIL` | PocketBase admin email | `admin@libnovel.local` |
|
||||
| `POCKETBASE_ADMIN_PASSWORD` | PocketBase admin password | `changeme123` |
|
||||
| `BROWSERLESS_URL` | Browserless WS endpoint (save-browse only) | `http://localhost:3030` |
|
||||
| `SINGLEFILE_PATH` | SingleFile CLI binary path (save-browse only) | `single-file` |
|
||||
|
||||
### UI (SvelteKit)
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `AUTH_SECRET` | HMAC signing secret for auth tokens | `dev_secret_change_in_production` |
|
||||
| `SCRAPER_API_URL` | Internal URL of the Go scraper | `http://localhost:8080` |
|
||||
| `POCKETBASE_URL` | PocketBase base URL | `http://localhost:8090` |
|
||||
| `POCKETBASE_ADMIN_EMAIL` | PocketBase admin email | `admin@libnovel.local` |
|
||||
| `POCKETBASE_ADMIN_PASSWORD` | PocketBase admin password | `changeme123` |
|
||||
| `PUBLIC_MINIO_PUBLIC_URL` | Browser-visible MinIO URL (presigned links) | `http://localhost:9000` |
|
||||
|
||||
## Docker
|
||||
|
||||
```bash
|
||||
docker-compose up -d # Starts browserless, kokoro, scraper
|
||||
docker-compose up -d # Starts: minio, minio-init, pocketbase, pb-init, scraper, ui
|
||||
```
|
||||
|
||||
Services:
|
||||
|
||||
| Service | Port(s) | Role |
|
||||
|---------|---------|------|
|
||||
| `minio` | `9000` (S3 API), `9001` (console) | Object storage |
|
||||
| `minio-init` | — | One-shot bucket creation then exits |
|
||||
| `pocketbase` | `8090` | Structured data store |
|
||||
| `pb-init` | — | One-shot PocketBase collection bootstrap then exits |
|
||||
| `scraper` | `8080` | Go scraper HTTP API |
|
||||
| `ui` | `5252` → internal `3000` | SvelteKit frontend |
|
||||
|
||||
Kokoro and Browserless are **external services** — not in docker-compose.
|
||||
|
||||
## HTTP API Endpoints (Go scraper)
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| `GET` | `/health` | Liveness probe |
|
||||
| `POST` | `/scrape` | Enqueue full catalogue scrape |
|
||||
| `POST` | `/scrape/book` | Enqueue single-book scrape `{url}` |
|
||||
| `POST` | `/scrape/book/range` | Enqueue range scrape `{url, from, to?}` |
|
||||
| `GET` | `/api/scrape/status` | Current scrape job status |
|
||||
| `GET` | `/api/scrape/tasks` | All scrape task records |
|
||||
| `GET` | `/api/browse` | Browse novelfire catalogue (MinIO-cached) |
|
||||
| `GET` | `/api/search` | Search local + remote `?q=` |
|
||||
| `GET` | `/api/ranking` | Ranking list |
|
||||
| `GET` | `/api/cover/{domain}/{slug}` | Proxy cover image from MinIO |
|
||||
| `GET` | `/api/book-preview/{slug}` | Live metadata + chapter list (no store write) |
|
||||
| `GET` | `/api/chapter-text-preview/{slug}/{n}` | Live chapter text (no store write) |
|
||||
| `POST` | `/api/reindex/{slug}` | Rebuild chapters_idx from MinIO |
|
||||
| `GET` | `/api/chapter-text/{slug}/{n}` | Chapter text (markdown stripped) |
|
||||
| `POST` | `/api/audio/{slug}/{n}` | Trigger Kokoro TTS generation |
|
||||
| `GET` | `/api/audio-proxy/{slug}/{n}` | Proxy generated audio |
|
||||
| `POST` | `/api/audio/voice-samples` | Pre-generate voice samples |
|
||||
| `GET` | `/api/voices` | List available Kokoro voices |
|
||||
| `GET` | `/api/presign/chapter/{slug}/{n}` | Presigned MinIO URL for chapter |
|
||||
| `GET` | `/api/presign/audio/{slug}/{n}` | Presigned MinIO URL for audio |
|
||||
| `GET` | `/api/presign/voice-sample/{voice}` | Presigned MinIO URL for voice sample |
|
||||
| `GET` | `/api/progress` | Get reading progress (session-scoped) |
|
||||
| `POST` | `/api/progress/{slug}` | Set reading progress |
|
||||
| `DELETE` | `/api/progress/{slug}` | Delete reading progress |
|
||||
|
||||
## Code Patterns
|
||||
|
||||
- Uses `log/slog` for structured logging
|
||||
- Context-based cancellation throughout
|
||||
- Worker pool pattern in orchestrator (channel + goroutines)
|
||||
- Mutex for single async job (409 on concurrent scrape requests)
|
||||
- `log/slog` for structured logging throughout
|
||||
- Context-based cancellation on all network calls and goroutines
|
||||
- Worker pool pattern in orchestrator (buffered channel + WaitGroup)
|
||||
- Single async scrape job enforced by mutex; 409 on concurrent requests; job state persisted to `scraping_tasks` in PocketBase
|
||||
- `Store` interface decouples all persistence — pass it around, never touch MinIO/PocketBase clients directly outside `storage/`
|
||||
- Auth: custom HMAC-signed token (`userId:username:role.<sig>`) in `libnovel_auth` cookie; signed with `AUTH_SECRET`
|
||||
|
||||
## AI Context Tips
|
||||
|
||||
- Primary files to modify: `orchestrator.go`, `server.go`, `scraper.go`, `browser/*.go`
|
||||
- To add new source: implement `NovelScraper` interface from `internal/scraper/interfaces.go`
|
||||
- Skip `static/` directory - generated content, not source
|
||||
|
||||
## Speed Up AI Sessions (Optional)
|
||||
|
||||
For faster AI context loading, use **Context7** (free, local indexing):
|
||||
|
||||
```bash
|
||||
# Install and index once
|
||||
npx @context7/cli@latest index --path . --ignore .aiignore
|
||||
|
||||
# After first run, AI tools will query the index instead of re-scanning files
|
||||
```
|
||||
|
||||
VSCode extension: https://marketplace.visualstudio.com/items?itemName=context7.context7
|
||||
- **Primary files to modify**: `orchestrator.go`, `server/handlers_*.go`, `novelfire/scraper.go`, `storage/hybrid.go`, `storage/pocketbase.go`
|
||||
- **To add a new scrape source**: implement `NovelScraper` from `internal/scraper/interfaces.go`
|
||||
- **To add a new API endpoint**: add handler in the appropriate `handlers_*.go` file, register in `server.go` `ListenAndServe()`
|
||||
- **Storage changes**: update `Store` interface in `store.go`, implement on `HybridStore` (hybrid.go) and `PocketBaseStore`/`MinioClient` as needed; update mock in `orchestrator_test.go`
|
||||
- **Skip**: `scraper/bin/` (compiled binary), MinIO/PocketBase data volumes
|
||||
|
||||
@@ -1,82 +1,159 @@
|
||||
version: "3.9"
|
||||
|
||||
services:
|
||||
# ─── Browserless ────────────────────────────────────────────────────────────
|
||||
browserless:
|
||||
image: ghcr.io/browserless/chromium:latest
|
||||
container_name: libnovel-browserless
|
||||
# ─── MinIO (object storage for chapter .md files + audio cache) ─────────────
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
#container_name: libnovel-minio
|
||||
restart: unless-stopped
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
# Set a token to lock down the endpoint; the scraper reads it via
|
||||
# BROWSERLESS_TOKEN below.
|
||||
TOKEN: "${BROWSERLESS_TOKEN:-}"
|
||||
# Allow up to 10 concurrent browser sessions.
|
||||
CONCURRENT: "${BROWSERLESS_CONCURRENT:-10}"
|
||||
# Queue up to 100 requests before returning 429.
|
||||
QUEUED: "${BROWSERLESS_QUEUED:-100}"
|
||||
# Per-session timeout in ms.
|
||||
TIMEOUT: "${BROWSERLESS_TIMEOUT:-60000}"
|
||||
# Optional webhook URL for Browserless error alerts.
|
||||
ERROR_ALERT_URL: "${ERROR_ALERT_URL:-}"
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "3030:3000"
|
||||
# Shared memory is required for Chrome.
|
||||
shm_size: "2gb"
|
||||
- "${MINIO_PORT:-9000}:9000" # S3 API
|
||||
- "${MINIO_CONSOLE_PORT:-9001}:9001" # Web console
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:3000/json/version"]
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── Kokoro-FastAPI (TTS) ────────────────────────────────────────────────────
|
||||
# CPU image; swap for ghcr.io/remsky/kokoro-fastapi-gpu:latest on NVIDIA hosts.
|
||||
# Models are baked in — no volume mount required for the default voice set.
|
||||
kokoro:
|
||||
image: ghcr.io/remsky/kokoro-fastapi-cpu:latest
|
||||
container_name: libnovel-kokoro
|
||||
# ─── MinIO bucket initialisation ─────────────────────────────────────────────
|
||||
# Runs once to create the default buckets and then exits.
|
||||
minio-init:
|
||||
image: minio/mc:latest
|
||||
#container_name: libnovel-minio-init
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
entrypoint: >
|
||||
/bin/sh -c "
|
||||
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
|
||||
mc mb --ignore-existing local/libnovel-chapters;
|
||||
mc mb --ignore-existing local/libnovel-audio;
|
||||
mc mb --ignore-existing local/libnovel-browse;
|
||||
mc mb --ignore-existing local/libnovel-avatars;
|
||||
echo 'buckets ready';
|
||||
"
|
||||
environment:
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
|
||||
# ─── PocketBase (auth + structured data: books, chapters index, ranking, progress) ──
|
||||
pocketbase:
|
||||
image: ghcr.io/muchobien/pocketbase:latest
|
||||
#container_name: libnovel-pocketbase
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# Auto-create superuser on first boot (used by entrypoint.sh)
|
||||
PB_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
PB_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "8880:8880"
|
||||
- "${POCKETBASE_PORT:-8090}:8090"
|
||||
volumes:
|
||||
- pb_data:/pb_data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8880/health"]
|
||||
interval: 15s
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:8090/api/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── PocketBase collection bootstrap ────────────────────────────────────────
|
||||
# One-shot init container: creates all required collections via the admin API
|
||||
# and exits. Idempotent — safe to run on every `docker compose up`.
|
||||
pb-init:
|
||||
image: alpine:3.19
|
||||
depends_on:
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
volumes:
|
||||
- ./scripts/pb-init.sh:/pb-init.sh:ro
|
||||
entrypoint: ["sh", "/pb-init.sh"]
|
||||
|
||||
# ─── Scraper ─────────────────────────────────────────────────────────────────
|
||||
scraper:
|
||||
build:
|
||||
context: ./scraper
|
||||
dockerfile: Dockerfile
|
||||
container_name: libnovel-scraper
|
||||
#container_name: libnovel-scraper
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
kokoro:
|
||||
pb-init:
|
||||
condition: service_completed_successfully
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
BROWSERLESS_URL: "http://browserless:3000"
|
||||
BROWSERLESS_TOKEN: "${BROWSERLESS_TOKEN:-}"
|
||||
# content | scrape | cdp | direct — swap to test different strategies.
|
||||
BROWSERLESS_STRATEGY: "${BROWSERLESS_STRATEGY:-direct}"
|
||||
# Strategy for URL retrieval (chapter list). Default: content (browserless)
|
||||
BROWSERLESS_URL_STRATEGY: "${BROWSERLESS_URL_STRATEGY:-content}"
|
||||
# 0 → defaults to NumCPU inside the container.
|
||||
SCRAPER_WORKERS: "${SCRAPER_WORKERS:-0}"
|
||||
SCRAPER_STATIC_ROOT: "/app/static/books"
|
||||
SCRAPER_HTTP_ADDR: ":8080"
|
||||
LOG_LEVEL: "debug"
|
||||
# Kokoro-FastAPI TTS endpoint.
|
||||
KOKORO_URL: "${KOKORO_URL:-http://localhost:8880}"
|
||||
KOKORO_URL: "${KOKORO_URL:-https://kokoro.kalekber.cc}"
|
||||
KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}"
|
||||
# MinIO / S3 object storage
|
||||
MINIO_ENDPOINT: "minio:9000"
|
||||
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
MINIO_USE_SSL: "false"
|
||||
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
|
||||
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
|
||||
MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}"
|
||||
MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
|
||||
# Public endpoint used to sign presigned audio URLs so browsers can reach them.
|
||||
# Leave empty to use MINIO_ENDPOINT (fine for local dev).
|
||||
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-}"
|
||||
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-true}"
|
||||
# SingleFile CLI path for save-browse subcommand
|
||||
SINGLEFILE_PATH: "${SINGLEFILE_PATH:-single-file}"
|
||||
# PocketBase
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- static_books:/app/static/books
|
||||
- "${SCRAPER_PORT:-8080}:8080"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:8080/health"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# ─── SvelteKit UI ────────────────────────────────────────────────────────────
|
||||
ui:
|
||||
build:
|
||||
context: ./ui
|
||||
dockerfile: Dockerfile
|
||||
# container_name: libnovel-ui
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
pb-init:
|
||||
condition: service_completed_successfully
|
||||
scraper:
|
||||
condition: service_healthy
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
SCRAPER_API_URL: "http://scraper:8080"
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
|
||||
ports:
|
||||
- "${UI_PORT:-5252}:3000"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/health"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
static_books:
|
||||
minio_data:
|
||||
pb_data:
|
||||
|
||||
14
ios/.gitignore
vendored
Normal file
14
ios/.gitignore
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
# Xcode build artifacts — regenerate with: xcodegen generate --spec project.yml
|
||||
xcuserdata/
|
||||
*.xcuserstate
|
||||
*.xcworkspace/xcuserdata/
|
||||
DerivedData/
|
||||
build/
|
||||
|
||||
# Swift Package Manager — resolved by Xcode on first open
|
||||
LibNovel.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/
|
||||
.build/
|
||||
# Package.resolved is committed so SPM builds are reproducible
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
10
ios/LibNovel/.gitignore
vendored
Normal file
10
ios/LibNovel/.gitignore
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
# Fastlane
|
||||
fastlane/report.xml
|
||||
fastlane/Preview.html
|
||||
fastlane/screenshots/**/*.png
|
||||
fastlane/test_output
|
||||
fastlane/README.md
|
||||
|
||||
# Bundler
|
||||
.bundle
|
||||
vendor/bundle
|
||||
21
ios/LibNovel/ExportOptions.plist
Normal file
21
ios/LibNovel/ExportOptions.plist
Normal file
@@ -0,0 +1,21 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>method</key>
|
||||
<string>app-store</string>
|
||||
<key>teamID</key>
|
||||
<string>GHZXC6FVMU</string>
|
||||
<key>uploadBitcode</key>
|
||||
<false/>
|
||||
<key>uploadSymbols</key>
|
||||
<true/>
|
||||
<key>signingStyle</key>
|
||||
<string>manual</string>
|
||||
<key>provisioningProfiles</key>
|
||||
<dict>
|
||||
<key>com.kalekber.LibNovel</key>
|
||||
<string>LibNovel Distribution</string>
|
||||
</dict>
|
||||
</dict>
|
||||
</plist>
|
||||
3
ios/LibNovel/Gemfile
Normal file
3
ios/LibNovel/Gemfile
Normal file
@@ -0,0 +1,3 @@
|
||||
source "https://rubygems.org"
|
||||
|
||||
gem "fastlane"
|
||||
697
ios/LibNovel/LibNovel.xcodeproj/project.pbxproj
Normal file
697
ios/LibNovel/LibNovel.xcodeproj/project.pbxproj
Normal file
@@ -0,0 +1,697 @@
|
||||
// !$*UTF8*$!
|
||||
{
|
||||
archiveVersion = 1;
|
||||
classes = {
|
||||
};
|
||||
objectVersion = 77;
|
||||
objects = {
|
||||
|
||||
/* Begin PBXBuildFile section */
|
||||
032E049A4BB3CF0EA990C0CD /* LibNovelApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4F56C8E2BC3614530B81569D /* LibNovelApp.swift */; };
|
||||
08DFB5F626BA769556C8D145 /* BrowseView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1FA3F0FCA383180EE4C93BBA /* BrowseView.swift */; };
|
||||
0A52BC1CE71BED9E75D20D35 /* Models.swift in Sources */ = {isa = PBXBuildFile; fileRef = 762E378B9BC2161A7AA2CC36 /* Models.swift */; };
|
||||
2790B8C051BE389D83645047 /* BrowseViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9812F5FE30ED657FB40ABD7A /* BrowseViewModel.swift */; };
|
||||
2A15157AD2AE2271675C3485 /* ChapterReaderViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8995E667B3DD9CFCAD8A91D7 /* ChapterReaderViewModel.swift */; };
|
||||
3521DFD5FCBBED7B90368829 /* LibraryViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = FC338B05EA6DB22900712000 /* LibraryViewModel.swift */; };
|
||||
367C88FFC11701D2BAD8CCD0 /* RootTabView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2D5C115992F1CE2326236765 /* RootTabView.swift */; };
|
||||
41FB51553F1F1AEBFEA91C0A /* String+App.swift in Sources */ = {isa = PBXBuildFile; fileRef = FEC6F837FF2E902E334ED72E /* String+App.swift */; };
|
||||
4BB2C76262D5BD5DAD0D5D28 /* LibNovelTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4C918833E173D6B44D06955 /* LibNovelTests.swift */; };
|
||||
58E440CE4360D755401D1672 /* ProfileViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 937A589F84FD412BBB6FBC45 /* ProfileViewModel.swift */; };
|
||||
5D8D783259EF54C773788AAB /* AuthStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = F219788AE5ACBD6F240674F5 /* AuthStore.swift */; };
|
||||
64D80AACB8E1967B17921EE3 /* ProfileView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0B17D50389C6C98FC78BDBC /* ProfileView.swift */; };
|
||||
749292A18C57FA41EC88A30B /* BookDetailView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 39DE056C37FBC5EED8771821 /* BookDetailView.swift */; };
|
||||
7C74C10317D389121922A5E3 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 5A776719B77EDDB5E44743B0 /* Assets.xcassets */; };
|
||||
7D81DEB2EEFF9CA5079AEEF7 /* BookDetailViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 837F83AA12B59924FDF16617 /* BookDetailViewModel.swift */; };
|
||||
94D0C4B15734B4056BF3B127 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4B820081FA4817765A39939A /* ContentView.swift */; };
|
||||
9B2D6F241E707312AB80DC31 /* AuthView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CEF6782A2A28B2A485CBD48 /* AuthView.swift */; };
|
||||
A9B95BAD7CE2DCD1DDDABD4C /* AudioPlayerService.swift in Sources */ = {isa = PBXBuildFile; fileRef = DB13E89E50529E3081533A66 /* AudioPlayerService.swift */; };
|
||||
BE7805A4E78037A82B12AE56 /* PlayerViews.swift in Sources */ = {isa = PBXBuildFile; fileRef = DF49C3AEF9D010F9FEDAB1FC /* PlayerViews.swift */; };
|
||||
C807AD8D627CF6BED47D517C /* HomeViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AB2E843D93461074A89A171 /* HomeViewModel.swift */; };
|
||||
CFDAA4776344B075A1E3CD6B /* Kingfisher in Frameworks */ = {isa = PBXBuildFile; productRef = 09584EAB68A07B47F876A062 /* Kingfisher */; };
|
||||
E1F564399D1325F6A1B2B84F /* LibraryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C21107BECA55C07416E0CB8B /* LibraryView.swift */; };
|
||||
E2572692178FD17145FDAF77 /* Color+App.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9D83BB88C4306BE7A4F947CB /* Color+App.swift */; };
|
||||
EF3C57C400BF05CBEAC1F7FE /* HomeView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D6268D60803940CBD38FB921 /* HomeView.swift */; };
|
||||
F2AF05B9C8C23132A73ACDD3 /* CommonViews.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8E89FD8F46747CA653C5203D /* CommonViews.swift */; };
|
||||
F4FDA3C44752EB979235C042 /* NavDestination.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CAFB96D2500F34F0B0C860C /* NavDestination.swift */; };
|
||||
FB32F3772CA09684F00497F3 /* APIClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = B593F179EC3E9112126B540B /* APIClient.swift */; };
|
||||
FEFB5FDC2424D22914458001 /* ChapterReaderView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 81E3939152E23B4985FAF7E2 /* ChapterReaderView.swift */; };
|
||||
/* End PBXBuildFile section */
|
||||
|
||||
/* Begin PBXContainerItemProxy section */
|
||||
698AC3AA533BC05C985595D0 /* PBXContainerItemProxy */ = {
|
||||
isa = PBXContainerItemProxy;
|
||||
containerPortal = A10A669C0C8B43078C0FEE9F /* Project object */;
|
||||
proxyType = 1;
|
||||
remoteGlobalIDString = D039EDECDE3998D8534BB680;
|
||||
remoteInfo = LibNovel;
|
||||
};
|
||||
/* End PBXContainerItemProxy section */
|
||||
|
||||
/* Begin PBXFileReference section */
|
||||
1B8BF3DB582A658386E402C7 /* LibNovel.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = LibNovel.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
1FA3F0FCA383180EE4C93BBA /* BrowseView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BrowseView.swift; sourceTree = "<group>"; };
|
||||
235967A21B386BE13F56F3F8 /* LibNovelTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = LibNovelTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
2D5C115992F1CE2326236765 /* RootTabView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RootTabView.swift; sourceTree = "<group>"; };
|
||||
39DE056C37FBC5EED8771821 /* BookDetailView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookDetailView.swift; sourceTree = "<group>"; };
|
||||
3AB2E843D93461074A89A171 /* HomeViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HomeViewModel.swift; sourceTree = "<group>"; };
|
||||
4B820081FA4817765A39939A /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
|
||||
4F56C8E2BC3614530B81569D /* LibNovelApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LibNovelApp.swift; sourceTree = "<group>"; };
|
||||
5A776719B77EDDB5E44743B0 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
|
||||
762E378B9BC2161A7AA2CC36 /* Models.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Models.swift; sourceTree = "<group>"; };
|
||||
7CAFB96D2500F34F0B0C860C /* NavDestination.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NavDestination.swift; sourceTree = "<group>"; };
|
||||
7CEF6782A2A28B2A485CBD48 /* AuthView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuthView.swift; sourceTree = "<group>"; };
|
||||
81E3939152E23B4985FAF7E2 /* ChapterReaderView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChapterReaderView.swift; sourceTree = "<group>"; };
|
||||
837F83AA12B59924FDF16617 /* BookDetailViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookDetailViewModel.swift; sourceTree = "<group>"; };
|
||||
8995E667B3DD9CFCAD8A91D7 /* ChapterReaderViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChapterReaderViewModel.swift; sourceTree = "<group>"; };
|
||||
8E89FD8F46747CA653C5203D /* CommonViews.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CommonViews.swift; sourceTree = "<group>"; };
|
||||
937A589F84FD412BBB6FBC45 /* ProfileViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileViewModel.swift; sourceTree = "<group>"; };
|
||||
9812F5FE30ED657FB40ABD7A /* BrowseViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BrowseViewModel.swift; sourceTree = "<group>"; };
|
||||
9D83BB88C4306BE7A4F947CB /* Color+App.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Color+App.swift"; sourceTree = "<group>"; };
|
||||
B4C918833E173D6B44D06955 /* LibNovelTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LibNovelTests.swift; sourceTree = "<group>"; };
|
||||
B593F179EC3E9112126B540B /* APIClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = APIClient.swift; sourceTree = "<group>"; };
|
||||
C0B17D50389C6C98FC78BDBC /* ProfileView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileView.swift; sourceTree = "<group>"; };
|
||||
C21107BECA55C07416E0CB8B /* LibraryView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LibraryView.swift; sourceTree = "<group>"; };
|
||||
D6268D60803940CBD38FB921 /* HomeView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HomeView.swift; sourceTree = "<group>"; };
|
||||
DB13E89E50529E3081533A66 /* AudioPlayerService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioPlayerService.swift; sourceTree = "<group>"; };
|
||||
DF49C3AEF9D010F9FEDAB1FC /* PlayerViews.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PlayerViews.swift; sourceTree = "<group>"; };
|
||||
F219788AE5ACBD6F240674F5 /* AuthStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuthStore.swift; sourceTree = "<group>"; };
|
||||
FC338B05EA6DB22900712000 /* LibraryViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LibraryViewModel.swift; sourceTree = "<group>"; };
|
||||
FEC6F837FF2E902E334ED72E /* String+App.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "String+App.swift"; sourceTree = "<group>"; };
|
||||
/* End PBXFileReference section */
|
||||
|
||||
/* Begin PBXFrameworksBuildPhase section */
|
||||
EFE3211B202EDF04EB141EFB /* Frameworks */ = {
|
||||
isa = PBXFrameworksBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
CFDAA4776344B075A1E3CD6B /* Kingfisher in Frameworks */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXFrameworksBuildPhase section */
|
||||
|
||||
/* Begin PBXGroup section */
|
||||
2C0FB0EDFF9B3E24B97F4214 /* Resources */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
5A776719B77EDDB5E44743B0 /* Assets.xcassets */,
|
||||
);
|
||||
path = Resources;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
2C57B93EAF19A3B18E7B7E87 /* Views */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
2F18D1275D6022B9847E310E /* Auth */,
|
||||
FB5C0D4925633786D28C6DE3 /* BookDetail */,
|
||||
8E8AAA58A33084ADB8AEA80C /* Browse */,
|
||||
4EAB87A1ED4943A311F26F84 /* ChapterReader */,
|
||||
5D5809803A3D74FAE19DB218 /* Common */,
|
||||
811FC0F6B9C209D6EC8543BD /* Home */,
|
||||
FA994FD601E79EC811D822A4 /* Library */,
|
||||
89F2CB14192E7D7565A588E0 /* Player */,
|
||||
3DB66C5703A4CCAFFA1B7AFE /* Profile */,
|
||||
);
|
||||
path = Views;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
2F18D1275D6022B9847E310E /* Auth */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
7CEF6782A2A28B2A485CBD48 /* AuthView.swift */,
|
||||
);
|
||||
path = Auth;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
3DB66C5703A4CCAFFA1B7AFE /* Profile */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
C0B17D50389C6C98FC78BDBC /* ProfileView.swift */,
|
||||
);
|
||||
path = Profile;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
426F7C5465758645B93A1AB1 /* Networking */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
B593F179EC3E9112126B540B /* APIClient.swift */,
|
||||
);
|
||||
path = Networking;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
4EAB87A1ED4943A311F26F84 /* ChapterReader */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
81E3939152E23B4985FAF7E2 /* ChapterReaderView.swift */,
|
||||
);
|
||||
path = ChapterReader;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
5D5809803A3D74FAE19DB218 /* Common */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
8E89FD8F46747CA653C5203D /* CommonViews.swift */,
|
||||
);
|
||||
path = Common;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
6318D3C6F0DC6C8E2C377103 /* Products */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
1B8BF3DB582A658386E402C7 /* LibNovel.app */,
|
||||
235967A21B386BE13F56F3F8 /* LibNovelTests.xctest */,
|
||||
);
|
||||
name = Products;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
646952B9CE927F8038FF0A13 /* LibNovelTests */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
B4C918833E173D6B44D06955 /* LibNovelTests.swift */,
|
||||
);
|
||||
path = LibNovelTests;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
80148B5E27BD0A3DEDB3ADAA /* Models */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
762E378B9BC2161A7AA2CC36 /* Models.swift */,
|
||||
);
|
||||
path = Models;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
811FC0F6B9C209D6EC8543BD /* Home */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
D6268D60803940CBD38FB921 /* HomeView.swift */,
|
||||
);
|
||||
path = Home;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
89F2CB14192E7D7565A588E0 /* Player */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
DF49C3AEF9D010F9FEDAB1FC /* PlayerViews.swift */,
|
||||
);
|
||||
path = Player;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
8E8AAA58A33084ADB8AEA80C /* Browse */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
1FA3F0FCA383180EE4C93BBA /* BrowseView.swift */,
|
||||
);
|
||||
path = Browse;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
9AF55E5D62F980C72431782A = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
A28A184E73B15138A4D13F31 /* LibNovel */,
|
||||
646952B9CE927F8038FF0A13 /* LibNovelTests */,
|
||||
6318D3C6F0DC6C8E2C377103 /* Products */,
|
||||
);
|
||||
indentWidth = 4;
|
||||
sourceTree = "<group>";
|
||||
tabWidth = 4;
|
||||
usesTabs = 0;
|
||||
};
|
||||
A28A184E73B15138A4D13F31 /* LibNovel */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
FE92158CC5DA9AD446062724 /* App */,
|
||||
FD5EDEE9747643D45CA6423E /* Extensions */,
|
||||
80148B5E27BD0A3DEDB3ADAA /* Models */,
|
||||
426F7C5465758645B93A1AB1 /* Networking */,
|
||||
2C0FB0EDFF9B3E24B97F4214 /* Resources */,
|
||||
DA6F6F625578875F3E74F1D3 /* Services */,
|
||||
B6916C5C762A37AB1279DF44 /* ViewModels */,
|
||||
2C57B93EAF19A3B18E7B7E87 /* Views */,
|
||||
);
|
||||
path = LibNovel;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
B6916C5C762A37AB1279DF44 /* ViewModels */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
837F83AA12B59924FDF16617 /* BookDetailViewModel.swift */,
|
||||
9812F5FE30ED657FB40ABD7A /* BrowseViewModel.swift */,
|
||||
8995E667B3DD9CFCAD8A91D7 /* ChapterReaderViewModel.swift */,
|
||||
3AB2E843D93461074A89A171 /* HomeViewModel.swift */,
|
||||
FC338B05EA6DB22900712000 /* LibraryViewModel.swift */,
|
||||
937A589F84FD412BBB6FBC45 /* ProfileViewModel.swift */,
|
||||
);
|
||||
path = ViewModels;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
DA6F6F625578875F3E74F1D3 /* Services */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
DB13E89E50529E3081533A66 /* AudioPlayerService.swift */,
|
||||
F219788AE5ACBD6F240674F5 /* AuthStore.swift */,
|
||||
);
|
||||
path = Services;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
FA994FD601E79EC811D822A4 /* Library */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
C21107BECA55C07416E0CB8B /* LibraryView.swift */,
|
||||
);
|
||||
path = Library;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
FB5C0D4925633786D28C6DE3 /* BookDetail */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
39DE056C37FBC5EED8771821 /* BookDetailView.swift */,
|
||||
);
|
||||
path = BookDetail;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
FD5EDEE9747643D45CA6423E /* Extensions */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
9D83BB88C4306BE7A4F947CB /* Color+App.swift */,
|
||||
7CAFB96D2500F34F0B0C860C /* NavDestination.swift */,
|
||||
FEC6F837FF2E902E334ED72E /* String+App.swift */,
|
||||
);
|
||||
path = Extensions;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
FE92158CC5DA9AD446062724 /* App */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
4B820081FA4817765A39939A /* ContentView.swift */,
|
||||
4F56C8E2BC3614530B81569D /* LibNovelApp.swift */,
|
||||
2D5C115992F1CE2326236765 /* RootTabView.swift */,
|
||||
);
|
||||
path = App;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
/* End PBXGroup section */
|
||||
|
||||
/* Begin PBXNativeTarget section */
|
||||
5E6D3E8266BFCF0AAF5EC79D /* LibNovelTests */ = {
|
||||
isa = PBXNativeTarget;
|
||||
buildConfigurationList = 964FF85B62FA35E819BE7661 /* Build configuration list for PBXNativeTarget "LibNovelTests" */;
|
||||
buildPhases = (
|
||||
247D45B3DB26CAC41FA78A0B /* Sources */,
|
||||
);
|
||||
buildRules = (
|
||||
);
|
||||
dependencies = (
|
||||
9FD4A50EB175FC09D6BFD28D /* PBXTargetDependency */,
|
||||
);
|
||||
name = LibNovelTests;
|
||||
packageProductDependencies = (
|
||||
);
|
||||
productName = LibNovelTests;
|
||||
productReference = 235967A21B386BE13F56F3F8 /* LibNovelTests.xctest */;
|
||||
productType = "com.apple.product-type.bundle.unit-test";
|
||||
};
|
||||
D039EDECDE3998D8534BB680 /* LibNovel */ = {
|
||||
isa = PBXNativeTarget;
|
||||
buildConfigurationList = 29B2DE7267A3A4B2D89B32DA /* Build configuration list for PBXNativeTarget "LibNovel" */;
|
||||
buildPhases = (
|
||||
48661ADCA15B54E048CF694C /* Sources */,
|
||||
27446CA4728C022832398376 /* Resources */,
|
||||
EFE3211B202EDF04EB141EFB /* Frameworks */,
|
||||
);
|
||||
buildRules = (
|
||||
);
|
||||
dependencies = (
|
||||
);
|
||||
name = LibNovel;
|
||||
packageProductDependencies = (
|
||||
09584EAB68A07B47F876A062 /* Kingfisher */,
|
||||
);
|
||||
productName = LibNovel;
|
||||
productReference = 1B8BF3DB582A658386E402C7 /* LibNovel.app */;
|
||||
productType = "com.apple.product-type.application";
|
||||
};
|
||||
/* End PBXNativeTarget section */
|
||||
|
||||
/* Begin PBXProject section */
|
||||
A10A669C0C8B43078C0FEE9F /* Project object */ = {
|
||||
isa = PBXProject;
|
||||
attributes = {
|
||||
BuildIndependentTargetsInParallel = YES;
|
||||
LastUpgradeCheck = 2630;
|
||||
};
|
||||
buildConfigurationList = D27899EE96A9AFCBBE62EA3C /* Build configuration list for PBXProject "LibNovel" */;
|
||||
developmentRegion = en;
|
||||
hasScannedForEncodings = 0;
|
||||
knownRegions = (
|
||||
Base,
|
||||
en,
|
||||
);
|
||||
mainGroup = 9AF55E5D62F980C72431782A;
|
||||
minimizedProjectReferenceProxies = 1;
|
||||
packageReferences = (
|
||||
AFEF7128801A76181793EA9C /* XCRemoteSwiftPackageReference "Kingfisher" */,
|
||||
);
|
||||
preferredProjectObjectVersion = 77;
|
||||
productRefGroup = 6318D3C6F0DC6C8E2C377103 /* Products */;
|
||||
projectDirPath = "";
|
||||
projectRoot = "";
|
||||
targets = (
|
||||
D039EDECDE3998D8534BB680 /* LibNovel */,
|
||||
5E6D3E8266BFCF0AAF5EC79D /* LibNovelTests */,
|
||||
);
|
||||
};
|
||||
/* End PBXProject section */
|
||||
|
||||
/* Begin PBXResourcesBuildPhase section */
|
||||
27446CA4728C022832398376 /* Resources */ = {
|
||||
isa = PBXResourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
7C74C10317D389121922A5E3 /* Assets.xcassets in Resources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXResourcesBuildPhase section */
|
||||
|
||||
/* Begin PBXSourcesBuildPhase section */
|
||||
247D45B3DB26CAC41FA78A0B /* Sources */ = {
|
||||
isa = PBXSourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
4BB2C76262D5BD5DAD0D5D28 /* LibNovelTests.swift in Sources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
48661ADCA15B54E048CF694C /* Sources */ = {
|
||||
isa = PBXSourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
FB32F3772CA09684F00497F3 /* APIClient.swift in Sources */,
|
||||
A9B95BAD7CE2DCD1DDDABD4C /* AudioPlayerService.swift in Sources */,
|
||||
5D8D783259EF54C773788AAB /* AuthStore.swift in Sources */,
|
||||
9B2D6F241E707312AB80DC31 /* AuthView.swift in Sources */,
|
||||
749292A18C57FA41EC88A30B /* BookDetailView.swift in Sources */,
|
||||
7D81DEB2EEFF9CA5079AEEF7 /* BookDetailViewModel.swift in Sources */,
|
||||
08DFB5F626BA769556C8D145 /* BrowseView.swift in Sources */,
|
||||
2790B8C051BE389D83645047 /* BrowseViewModel.swift in Sources */,
|
||||
FEFB5FDC2424D22914458001 /* ChapterReaderView.swift in Sources */,
|
||||
2A15157AD2AE2271675C3485 /* ChapterReaderViewModel.swift in Sources */,
|
||||
E2572692178FD17145FDAF77 /* Color+App.swift in Sources */,
|
||||
F2AF05B9C8C23132A73ACDD3 /* CommonViews.swift in Sources */,
|
||||
94D0C4B15734B4056BF3B127 /* ContentView.swift in Sources */,
|
||||
EF3C57C400BF05CBEAC1F7FE /* HomeView.swift in Sources */,
|
||||
C807AD8D627CF6BED47D517C /* HomeViewModel.swift in Sources */,
|
||||
032E049A4BB3CF0EA990C0CD /* LibNovelApp.swift in Sources */,
|
||||
E1F564399D1325F6A1B2B84F /* LibraryView.swift in Sources */,
|
||||
3521DFD5FCBBED7B90368829 /* LibraryViewModel.swift in Sources */,
|
||||
0A52BC1CE71BED9E75D20D35 /* Models.swift in Sources */,
|
||||
F4FDA3C44752EB979235C042 /* NavDestination.swift in Sources */,
|
||||
BE7805A4E78037A82B12AE56 /* PlayerViews.swift in Sources */,
|
||||
64D80AACB8E1967B17921EE3 /* ProfileView.swift in Sources */,
|
||||
58E440CE4360D755401D1672 /* ProfileViewModel.swift in Sources */,
|
||||
367C88FFC11701D2BAD8CCD0 /* RootTabView.swift in Sources */,
|
||||
41FB51553F1F1AEBFEA91C0A /* String+App.swift in Sources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXSourcesBuildPhase section */
|
||||
|
||||
/* Begin PBXTargetDependency section */
|
||||
9FD4A50EB175FC09D6BFD28D /* PBXTargetDependency */ = {
|
||||
isa = PBXTargetDependency;
|
||||
target = D039EDECDE3998D8534BB680 /* LibNovel */;
|
||||
targetProxy = 698AC3AA533BC05C985595D0 /* PBXContainerItemProxy */;
|
||||
};
|
||||
/* End PBXTargetDependency section */
|
||||
|
||||
/* Begin XCBuildConfiguration section */
|
||||
428871329DC9E7B31FA1664B /* Release */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
BUNDLE_LOADER = "$(TEST_HOST)";
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
"@loader_path/Frameworks",
|
||||
);
|
||||
PRODUCT_BUNDLE_IDENTIFIER = com.kalekber.LibNovel.tests;
|
||||
SDKROOT = iphoneos;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
TEST_HOST = "$(BUILT_PRODUCTS_DIR)/LibNovel.app/LibNovel";
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
49CBF0D367E562629E002A4B /* Debug */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
BUNDLE_LOADER = "$(TEST_HOST)";
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
"@loader_path/Frameworks",
|
||||
);
|
||||
PRODUCT_BUNDLE_IDENTIFIER = com.kalekber.LibNovel.tests;
|
||||
SDKROOT = iphoneos;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
TEST_HOST = "$(BUILT_PRODUCTS_DIR)/LibNovel.app/LibNovel";
|
||||
};
|
||||
name = Debug;
|
||||
};
|
||||
8098D4A97F989064EC71E5A1 /* Debug */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
||||
CODE_SIGN_IDENTITY = "Apple Development";
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
CURRENT_PROJECT_VERSION = 1;
|
||||
DEVELOPMENT_TEAM = GHZXC6FVMU;
|
||||
GENERATE_INFOPLIST_FILE = NO;
|
||||
INFOPLIST_FILE = LibNovel/Resources/Info.plist;
|
||||
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.books";
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
);
|
||||
MARKETING_VERSION = 1.0.1;
|
||||
PRODUCT_BUNDLE_IDENTIFIER = com.kalekber.LibNovel;
|
||||
PROVISIONING_PROFILE_SPECIFIER = "";
|
||||
SDKROOT = iphoneos;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
};
|
||||
name = Debug;
|
||||
};
|
||||
9C182367114E72FF84D54A2F /* Debug */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ALWAYS_SEARCH_USER_PATHS = NO;
|
||||
CLANG_ANALYZER_NONNULL = YES;
|
||||
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
||||
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
||||
CLANG_CXX_LIBRARY = "libc++";
|
||||
CLANG_ENABLE_MODULES = YES;
|
||||
CLANG_ENABLE_OBJC_ARC = YES;
|
||||
CLANG_ENABLE_OBJC_WEAK = YES;
|
||||
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
||||
CLANG_WARN_BOOL_CONVERSION = YES;
|
||||
CLANG_WARN_COMMA = YES;
|
||||
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
||||
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
||||
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
||||
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
||||
CLANG_WARN_EMPTY_BODY = YES;
|
||||
CLANG_WARN_ENUM_CONVERSION = YES;
|
||||
CLANG_WARN_INFINITE_RECURSION = YES;
|
||||
CLANG_WARN_INT_CONVERSION = YES;
|
||||
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
||||
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
||||
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
||||
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
||||
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
||||
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
||||
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
||||
CLANG_WARN_UNREACHABLE_CODE = YES;
|
||||
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
||||
COPY_PHASE_STRIP = NO;
|
||||
CURRENT_PROJECT_VERSION = 1000;
|
||||
DEBUG_INFORMATION_FORMAT = dwarf;
|
||||
ENABLE_PREVIEWS = YES;
|
||||
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
||||
ENABLE_TESTABILITY = YES;
|
||||
ENABLE_USER_SCRIPT_SANDBOXING = YES;
|
||||
GCC_C_LANGUAGE_STANDARD = gnu11;
|
||||
GCC_DYNAMIC_NO_PIC = NO;
|
||||
GCC_NO_COMMON_BLOCKS = YES;
|
||||
GCC_OPTIMIZATION_LEVEL = 0;
|
||||
GCC_PREPROCESSOR_DEFINITIONS = (
|
||||
"$(inherited)",
|
||||
"DEBUG=1",
|
||||
);
|
||||
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
||||
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
||||
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
||||
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
||||
GCC_WARN_UNUSED_FUNCTION = YES;
|
||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LIBNOVEL_BASE_URL = "https://v2.libnovel.kalekber.cc";
|
||||
MARKETING_VERSION = 1.0.0;
|
||||
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
|
||||
MTL_FAST_MATH = YES;
|
||||
ONLY_ACTIVE_ARCH = YES;
|
||||
PRODUCT_NAME = "$(TARGET_NAME)";
|
||||
SDKROOT = iphoneos;
|
||||
STRING_CATALOG_GENERATE_SYMBOLS = YES;
|
||||
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
|
||||
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
|
||||
SWIFT_VERSION = 5.10;
|
||||
};
|
||||
name = Debug;
|
||||
};
|
||||
D9977A0FA70F052FD0C126D3 /* Release */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
||||
CODE_SIGN_IDENTITY = "Apple Distribution";
|
||||
"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution";
|
||||
CODE_SIGN_STYLE = Manual;
|
||||
CURRENT_PROJECT_VERSION = 1;
|
||||
DEVELOPMENT_TEAM = GHZXC6FVMU;
|
||||
"DEVELOPMENT_TEAM[sdk=iphoneos*]" = GHZXC6FVMU;
|
||||
GENERATE_INFOPLIST_FILE = NO;
|
||||
INFOPLIST_FILE = LibNovel/Resources/Info.plist;
|
||||
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.books";
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
);
|
||||
MARKETING_VERSION = 1.0.1;
|
||||
PRODUCT_BUNDLE_IDENTIFIER = com.kalekber.LibNovel;
|
||||
PROVISIONING_PROFILE = "af592c3a-f60b-4ac1-a14f-30b8a206017f";
|
||||
"PROVISIONING_PROFILE_SPECIFIER[sdk=iphoneos*]" = "LibNovel Distribution";
|
||||
SDKROOT = iphoneos;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
F9ED141CFB1E2EC6F5E9F089 /* Release */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ALWAYS_SEARCH_USER_PATHS = NO;
|
||||
CLANG_ANALYZER_NONNULL = YES;
|
||||
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
||||
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
||||
CLANG_CXX_LIBRARY = "libc++";
|
||||
CLANG_ENABLE_MODULES = YES;
|
||||
CLANG_ENABLE_OBJC_ARC = YES;
|
||||
CLANG_ENABLE_OBJC_WEAK = YES;
|
||||
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
||||
CLANG_WARN_BOOL_CONVERSION = YES;
|
||||
CLANG_WARN_COMMA = YES;
|
||||
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
||||
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
||||
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
||||
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
||||
CLANG_WARN_EMPTY_BODY = YES;
|
||||
CLANG_WARN_ENUM_CONVERSION = YES;
|
||||
CLANG_WARN_INFINITE_RECURSION = YES;
|
||||
CLANG_WARN_INT_CONVERSION = YES;
|
||||
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
||||
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
||||
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
||||
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
||||
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
||||
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
||||
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
||||
CLANG_WARN_UNREACHABLE_CODE = YES;
|
||||
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
||||
COPY_PHASE_STRIP = NO;
|
||||
CURRENT_PROJECT_VERSION = 1000;
|
||||
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
|
||||
ENABLE_NS_ASSERTIONS = NO;
|
||||
ENABLE_PREVIEWS = YES;
|
||||
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
||||
ENABLE_USER_SCRIPT_SANDBOXING = YES;
|
||||
GCC_C_LANGUAGE_STANDARD = gnu11;
|
||||
GCC_NO_COMMON_BLOCKS = YES;
|
||||
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
||||
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
||||
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
||||
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
||||
GCC_WARN_UNUSED_FUNCTION = YES;
|
||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LIBNOVEL_BASE_URL = "https://v2.libnovel.kalekber.cc";
|
||||
MARKETING_VERSION = 1.0.0;
|
||||
MTL_ENABLE_DEBUG_INFO = NO;
|
||||
MTL_FAST_MATH = YES;
|
||||
PRODUCT_NAME = "$(TARGET_NAME)";
|
||||
SDKROOT = iphoneos;
|
||||
STRING_CATALOG_GENERATE_SYMBOLS = YES;
|
||||
SWIFT_ACTIVE_COMPILATION_CONDITIONS = "";
|
||||
SWIFT_COMPILATION_MODE = wholemodule;
|
||||
SWIFT_OPTIMIZATION_LEVEL = "-O";
|
||||
SWIFT_VERSION = 5.10;
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
/* End XCBuildConfiguration section */
|
||||
|
||||
/* Begin XCConfigurationList section */
|
||||
29B2DE7267A3A4B2D89B32DA /* Build configuration list for PBXNativeTarget "LibNovel" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
8098D4A97F989064EC71E5A1 /* Debug */,
|
||||
D9977A0FA70F052FD0C126D3 /* Release */,
|
||||
);
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Debug;
|
||||
};
|
||||
964FF85B62FA35E819BE7661 /* Build configuration list for PBXNativeTarget "LibNovelTests" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
49CBF0D367E562629E002A4B /* Debug */,
|
||||
428871329DC9E7B31FA1664B /* Release */,
|
||||
);
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Debug;
|
||||
};
|
||||
D27899EE96A9AFCBBE62EA3C /* Build configuration list for PBXProject "LibNovel" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
9C182367114E72FF84D54A2F /* Debug */,
|
||||
F9ED141CFB1E2EC6F5E9F089 /* Release */,
|
||||
);
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Debug;
|
||||
};
|
||||
/* End XCConfigurationList section */
|
||||
|
||||
/* Begin XCRemoteSwiftPackageReference section */
|
||||
AFEF7128801A76181793EA9C /* XCRemoteSwiftPackageReference "Kingfisher" */ = {
|
||||
isa = XCRemoteSwiftPackageReference;
|
||||
repositoryURL = "https://github.com/onevcat/Kingfisher";
|
||||
requirement = {
|
||||
kind = upToNextMajorVersion;
|
||||
minimumVersion = 8.0.0;
|
||||
};
|
||||
};
|
||||
/* End XCRemoteSwiftPackageReference section */
|
||||
|
||||
/* Begin XCSwiftPackageProductDependency section */
|
||||
09584EAB68A07B47F876A062 /* Kingfisher */ = {
|
||||
isa = XCSwiftPackageProductDependency;
|
||||
package = AFEF7128801A76181793EA9C /* XCRemoteSwiftPackageReference "Kingfisher" */;
|
||||
productName = Kingfisher;
|
||||
};
|
||||
/* End XCSwiftPackageProductDependency section */
|
||||
};
|
||||
rootObject = A10A669C0C8B43078C0FEE9F /* Project object */;
|
||||
}
|
||||
7
ios/LibNovel/LibNovel.xcodeproj/project.xcworkspace/contents.xcworkspacedata
generated
Normal file
7
ios/LibNovel/LibNovel.xcodeproj/project.xcworkspace/contents.xcworkspacedata
generated
Normal file
@@ -0,0 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Workspace
|
||||
version = "1.0">
|
||||
<FileRef
|
||||
location = "self:">
|
||||
</FileRef>
|
||||
</Workspace>
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"originHash" : "ad75ae2d3b8d8b80d99635f65213a3c1092464aa54a86354f850b8317b6fa240",
|
||||
"pins" : [
|
||||
{
|
||||
"identity" : "kingfisher",
|
||||
"kind" : "remoteSourceControl",
|
||||
"location" : "https://github.com/onevcat/Kingfisher",
|
||||
"state" : {
|
||||
"revision" : "c92b84898e34ab46ff0dad86c02a0acbe2d87008",
|
||||
"version" : "8.8.0"
|
||||
}
|
||||
}
|
||||
],
|
||||
"version" : 3
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Scheme
|
||||
LastUpgradeVersion = "2630"
|
||||
version = "1.3">
|
||||
<BuildAction
|
||||
parallelizeBuildables = "YES"
|
||||
buildImplicitDependencies = "YES">
|
||||
<BuildActionEntries>
|
||||
<BuildActionEntry
|
||||
buildForTesting = "YES"
|
||||
buildForRunning = "YES"
|
||||
buildForProfiling = "YES"
|
||||
buildForArchiving = "YES"
|
||||
buildForAnalyzing = "YES">
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "D039EDECDE3998D8534BB680"
|
||||
BuildableName = "LibNovel.app"
|
||||
BlueprintName = "LibNovel"
|
||||
ReferencedContainer = "container:LibNovel.xcodeproj">
|
||||
</BuildableReference>
|
||||
</BuildActionEntry>
|
||||
</BuildActionEntries>
|
||||
</BuildAction>
|
||||
<TestAction
|
||||
buildConfiguration = "Debug"
|
||||
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
||||
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
||||
shouldUseLaunchSchemeArgsEnv = "YES">
|
||||
<MacroExpansion>
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "D039EDECDE3998D8534BB680"
|
||||
BuildableName = "LibNovel.app"
|
||||
BlueprintName = "LibNovel"
|
||||
ReferencedContainer = "container:LibNovel.xcodeproj">
|
||||
</BuildableReference>
|
||||
</MacroExpansion>
|
||||
<Testables>
|
||||
<TestableReference
|
||||
skipped = "NO"
|
||||
parallelizable = "NO">
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "5E6D3E8266BFCF0AAF5EC79D"
|
||||
BuildableName = "LibNovelTests.xctest"
|
||||
BlueprintName = "LibNovelTests"
|
||||
ReferencedContainer = "container:LibNovel.xcodeproj">
|
||||
</BuildableReference>
|
||||
</TestableReference>
|
||||
</Testables>
|
||||
</TestAction>
|
||||
<LaunchAction
|
||||
buildConfiguration = "Debug"
|
||||
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
||||
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
||||
launchStyle = "0"
|
||||
useCustomWorkingDirectory = "NO"
|
||||
ignoresPersistentStateOnLaunch = "NO"
|
||||
debugDocumentVersioning = "YES"
|
||||
debugServiceExtension = "internal"
|
||||
allowLocationSimulation = "YES">
|
||||
<BuildableProductRunnable
|
||||
runnableDebuggingMode = "0">
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "D039EDECDE3998D8534BB680"
|
||||
BuildableName = "LibNovel.app"
|
||||
BlueprintName = "LibNovel"
|
||||
ReferencedContainer = "container:LibNovel.xcodeproj">
|
||||
</BuildableReference>
|
||||
</BuildableProductRunnable>
|
||||
<EnvironmentVariables>
|
||||
<EnvironmentVariable
|
||||
key = "LIBNOVEL_BASE_URL"
|
||||
value = "["value": "https://v2.libnovel.kalekber.cc", "isEnabled": true]"
|
||||
isEnabled = "YES">
|
||||
</EnvironmentVariable>
|
||||
</EnvironmentVariables>
|
||||
</LaunchAction>
|
||||
<ProfileAction
|
||||
buildConfiguration = "Release"
|
||||
shouldUseLaunchSchemeArgsEnv = "YES"
|
||||
savedToolIdentifier = ""
|
||||
useCustomWorkingDirectory = "NO"
|
||||
debugDocumentVersioning = "YES">
|
||||
<BuildableProductRunnable
|
||||
runnableDebuggingMode = "0">
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "D039EDECDE3998D8534BB680"
|
||||
BuildableName = "LibNovel.app"
|
||||
BlueprintName = "LibNovel"
|
||||
ReferencedContainer = "container:LibNovel.xcodeproj">
|
||||
</BuildableReference>
|
||||
</BuildableProductRunnable>
|
||||
</ProfileAction>
|
||||
<AnalyzeAction
|
||||
buildConfiguration = "Debug">
|
||||
</AnalyzeAction>
|
||||
<ArchiveAction
|
||||
buildConfiguration = "Release"
|
||||
revealArchiveInOrganizer = "YES">
|
||||
</ArchiveAction>
|
||||
</Scheme>
|
||||
16
ios/LibNovel/LibNovel/App/ContentView.swift
Normal file
16
ios/LibNovel/LibNovel/App/ContentView.swift
Normal file
@@ -0,0 +1,16 @@
|
||||
import SwiftUI
|
||||
|
||||
struct ContentView: View {
|
||||
@EnvironmentObject var authStore: AuthStore
|
||||
@EnvironmentObject var audioPlayer: AudioPlayerService
|
||||
|
||||
var body: some View {
|
||||
Group {
|
||||
if authStore.isAuthenticated {
|
||||
RootTabView()
|
||||
} else {
|
||||
AuthView()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
15
ios/LibNovel/LibNovel/App/LibNovelApp.swift
Normal file
15
ios/LibNovel/LibNovel/App/LibNovelApp.swift
Normal file
@@ -0,0 +1,15 @@
|
||||
import SwiftUI
|
||||
|
||||
@main
|
||||
struct LibNovelApp: App {
|
||||
@StateObject private var authStore = AuthStore()
|
||||
@StateObject private var audioPlayer = AudioPlayerService()
|
||||
|
||||
var body: some Scene {
|
||||
WindowGroup {
|
||||
ContentView()
|
||||
.environmentObject(authStore)
|
||||
.environmentObject(audioPlayer)
|
||||
}
|
||||
}
|
||||
}
|
||||
105
ios/LibNovel/LibNovel/App/RootTabView.swift
Normal file
105
ios/LibNovel/LibNovel/App/RootTabView.swift
Normal file
@@ -0,0 +1,105 @@
|
||||
import SwiftUI
|
||||
|
||||
// MARK: - Root tab container with persistent mini-player overlay
|
||||
|
||||
struct RootTabView: View {
|
||||
@EnvironmentObject var authStore: AuthStore
|
||||
@EnvironmentObject var audioPlayer: AudioPlayerService
|
||||
|
||||
@State private var selectedTab: Tab = .home
|
||||
@State private var showFullPlayer: Bool = false
|
||||
|
||||
/// Live drag offset while the user is dragging the full player down.
|
||||
@State private var fullPlayerDragOffset: CGFloat = 0
|
||||
|
||||
enum Tab: Hashable {
|
||||
case home, library, browse, profile
|
||||
}
|
||||
|
||||
/// Height of the mini player bar (progress line 2pt + vertical padding 20pt + content ~44pt)
|
||||
private let miniPlayerBarHeight: CGFloat = AppLayout.miniPlayerBarHeight
|
||||
|
||||
var body: some View {
|
||||
ZStack(alignment: .bottom) {
|
||||
TabView(selection: $selectedTab) {
|
||||
HomeView()
|
||||
.tabItem { Label("Home", systemImage: "house.fill") }
|
||||
.tag(Tab.home)
|
||||
|
||||
LibraryView()
|
||||
.tabItem { Label("Library", systemImage: "book.pages.fill") }
|
||||
.tag(Tab.library)
|
||||
|
||||
BrowseView()
|
||||
.tabItem { Label("Discover", systemImage: "sparkles") }
|
||||
.tag(Tab.browse)
|
||||
|
||||
ProfileView()
|
||||
.tabItem { Label("Profile", systemImage: "gear") }
|
||||
.tag(Tab.profile)
|
||||
}
|
||||
// Reserve space for the mini-player above the tab bar so scroll content
|
||||
// never slides beneath it.
|
||||
.safeAreaInset(edge: .bottom) {
|
||||
if audioPlayer.isActive {
|
||||
Color.clear.frame(height: miniPlayerBarHeight)
|
||||
}
|
||||
}
|
||||
|
||||
// Mini-player pinned above the tab bar (hidden while full player is open)
|
||||
if audioPlayer.isActive && !showFullPlayer {
|
||||
MiniPlayerView(showFullPlayer: $showFullPlayer)
|
||||
.padding(.bottom, tabBarHeight)
|
||||
.transition(.move(edge: .bottom).combined(with: .opacity))
|
||||
.animation(.spring(response: 0.35, dampingFraction: 0.8), value: audioPlayer.isActive)
|
||||
}
|
||||
|
||||
// Full player — slides up from the bottom as a custom overlay (not a sheet)
|
||||
// so it feels physically connected to the mini player bar.
|
||||
if showFullPlayer {
|
||||
FullPlayerView(onDismiss: {
|
||||
withAnimation(.spring(response: 0.4, dampingFraction: 0.85)) {
|
||||
showFullPlayer = false
|
||||
fullPlayerDragOffset = 0
|
||||
}
|
||||
})
|
||||
.offset(y: max(fullPlayerDragOffset, 0))
|
||||
.gesture(
|
||||
DragGesture(minimumDistance: 10)
|
||||
.onChanged { value in
|
||||
if value.translation.height > 0 {
|
||||
// Rubberband slightly so it doesn't feel locked
|
||||
fullPlayerDragOffset = value.translation.height
|
||||
}
|
||||
}
|
||||
.onEnded { value in
|
||||
let velocity = value.predictedEndTranslation.height - value.translation.height
|
||||
if value.translation.height > 120 || velocity > 400 {
|
||||
withAnimation(.spring(response: 0.4, dampingFraction: 0.85)) {
|
||||
showFullPlayer = false
|
||||
fullPlayerDragOffset = 0
|
||||
}
|
||||
} else {
|
||||
withAnimation(.spring(response: 0.35, dampingFraction: 0.8)) {
|
||||
fullPlayerDragOffset = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
.transition(.move(edge: .bottom))
|
||||
.animation(.spring(response: 0.45, dampingFraction: 0.85), value: showFullPlayer)
|
||||
.ignoresSafeArea()
|
||||
}
|
||||
}
|
||||
.animation(.spring(response: 0.45, dampingFraction: 0.85), value: showFullPlayer)
|
||||
}
|
||||
|
||||
// Approximate safe-area-aware tab bar height
|
||||
private var tabBarHeight: CGFloat {
|
||||
let window = UIApplication.shared.connectedScenes
|
||||
.compactMap { $0 as? UIWindowScene }
|
||||
.first?.windows.first(where: \.isKeyWindow)
|
||||
let bottomInset = window?.safeAreaInsets.bottom ?? 0
|
||||
return 49 + bottomInset // 49pt is the standard iOS tab bar height
|
||||
}
|
||||
}
|
||||
10
ios/LibNovel/LibNovel/Extensions/Color+App.swift
Normal file
10
ios/LibNovel/LibNovel/Extensions/Color+App.swift
Normal file
@@ -0,0 +1,10 @@
|
||||
import SwiftUI
|
||||
|
||||
// MARK: - App accent color (amber — mirrors Tailwind amber-500 #f59e0b)
|
||||
extension Color {
|
||||
static let amber = Color(red: 0.96, green: 0.62, blue: 0.04)
|
||||
}
|
||||
|
||||
extension ShapeStyle where Self == Color {
|
||||
static var amber: Color { .amber }
|
||||
}
|
||||
100
ios/LibNovel/LibNovel/Extensions/NavDestination.swift
Normal file
100
ios/LibNovel/LibNovel/Extensions/NavDestination.swift
Normal file
@@ -0,0 +1,100 @@
|
||||
import SwiftUI
|
||||
|
||||
// MARK: - Navigation destination enum used across all tabs
|
||||
|
||||
enum NavDestination: Hashable {
|
||||
case book(String) // slug
|
||||
case chapter(String, Int) // slug + chapter number
|
||||
}
|
||||
|
||||
// MARK: - View extensions for shared navigation + error alert patterns
|
||||
|
||||
extension View {
|
||||
/// Registers the app-wide navigation destinations for NavDestination values.
|
||||
/// Apply once per NavigationStack instead of repeating the switch in every tab.
|
||||
func appNavigationDestination() -> some View {
|
||||
modifier(AppNavigationDestinationModifier())
|
||||
}
|
||||
|
||||
/// Presents a standard "Error" alert driven by an optional String binding.
|
||||
/// Dismissing the alert sets the binding back to nil.
|
||||
func errorAlert(_ error: Binding<String?>) -> some View {
|
||||
alert("Error", isPresented: Binding(
|
||||
get: { error.wrappedValue != nil },
|
||||
set: { if !$0 { error.wrappedValue = nil } }
|
||||
)) {
|
||||
Button("OK") { error.wrappedValue = nil }
|
||||
} message: {
|
||||
Text(error.wrappedValue ?? "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Navigation destination modifier
|
||||
|
||||
private struct AppNavigationDestinationModifier: ViewModifier {
|
||||
@Namespace private var zoomNamespace
|
||||
|
||||
func body(content: Content) -> some View {
|
||||
if #available(iOS 18.0, *) {
|
||||
content
|
||||
.navigationDestination(for: NavDestination.self) { dest in
|
||||
switch dest {
|
||||
case .book(let slug):
|
||||
BookDetailView(slug: slug)
|
||||
.navigationTransition(.zoom(sourceID: slug, in: zoomNamespace))
|
||||
case .chapter(let slug, let n):
|
||||
ChapterReaderView(slug: slug, chapterNumber: n)
|
||||
}
|
||||
}
|
||||
// Expose namespace to child views via environment
|
||||
.environment(\.bookZoomNamespace, zoomNamespace)
|
||||
} else {
|
||||
content
|
||||
.navigationDestination(for: NavDestination.self) { dest in
|
||||
switch dest {
|
||||
case .book(let slug): BookDetailView(slug: slug)
|
||||
case .chapter(let slug, let n): ChapterReaderView(slug: slug, chapterNumber: n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Environment key for zoom namespace
|
||||
|
||||
struct BookZoomNamespaceKey: EnvironmentKey {
|
||||
static var defaultValue: Namespace.ID? { nil }
|
||||
}
|
||||
|
||||
extension EnvironmentValues {
|
||||
var bookZoomNamespace: Namespace.ID? {
|
||||
get { self[BookZoomNamespaceKey.self] }
|
||||
set { self[BookZoomNamespaceKey.self] = newValue }
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Cover card zoom source modifier
|
||||
|
||||
/// Apply this to any cover image that should be a zoom source for book navigation.
|
||||
/// Falls back to a no-op on iOS 17 or when no namespace is available.
|
||||
struct BookCoverZoomSource: ViewModifier {
|
||||
let slug: String
|
||||
@Environment(\.bookZoomNamespace) private var namespace
|
||||
|
||||
func body(content: Content) -> some View {
|
||||
if #available(iOS 18.0, *), let ns = namespace {
|
||||
content.matchedTransitionSource(id: slug, in: ns)
|
||||
} else {
|
||||
content
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension View {
|
||||
/// Marks a cover image as the zoom source for a book's navigation transition.
|
||||
func bookCoverZoomSource(slug: String) -> some View {
|
||||
modifier(BookCoverZoomSource(slug: slug))
|
||||
}
|
||||
}
|
||||
|
||||
49
ios/LibNovel/LibNovel/Extensions/String+App.swift
Normal file
49
ios/LibNovel/LibNovel/Extensions/String+App.swift
Normal file
@@ -0,0 +1,49 @@
|
||||
import Foundation
|
||||
|
||||
// MARK: - String helpers for display purposes
|
||||
|
||||
extension String {
|
||||
/// Strips trailing relative-date suffixes (e.g. "2 years ago", "3 days ago",
|
||||
/// or "(One)4 years ago" where the number is attached without a preceding space).
|
||||
func strippingTrailingDate() -> String {
|
||||
let units = ["second", "minute", "hour", "day", "week", "month", "year"]
|
||||
let lower = self.lowercased()
|
||||
for unit in units {
|
||||
for suffix in [unit + "s ago", unit + " ago"] {
|
||||
guard let suffixRange = lower.range(of: suffix, options: .backwards) else { continue }
|
||||
// Everything before the suffix
|
||||
let before = String(self[self.startIndex ..< suffixRange.lowerBound])
|
||||
let trimmed = before.trimmingCharacters(in: .whitespaces)
|
||||
// Strip trailing digits (the numeric count, which may be attached without a space)
|
||||
var result = trimmed
|
||||
while let last = result.last, last.isNumber {
|
||||
result.removeLast()
|
||||
}
|
||||
result = result.trimmingCharacters(in: .whitespaces)
|
||||
if result != trimmed {
|
||||
// We actually stripped some digits — return cleaned result
|
||||
return result
|
||||
}
|
||||
// Fallback: number preceded by space
|
||||
if let spaceIdx = trimmed.lastIndex(of: " ") {
|
||||
let potentialNum = String(trimmed[trimmed.index(after: spaceIdx)...])
|
||||
if Int(potentialNum) != nil {
|
||||
return String(trimmed[trimmed.startIndex ..< spaceIdx])
|
||||
.trimmingCharacters(in: .whitespaces)
|
||||
}
|
||||
} else if Int(trimmed) != nil {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
}
|
||||
return self
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - App-wide layout constants
|
||||
|
||||
enum AppLayout {
|
||||
/// Height of the persistent mini-player bar:
|
||||
/// 12pt vertical padding (top) + 56pt cover height + 12pt vertical padding (bottom) + 12pt horizontal margin.
|
||||
static let miniPlayerBarHeight: CGFloat = 92
|
||||
}
|
||||
309
ios/LibNovel/LibNovel/Models/Models.swift
Normal file
309
ios/LibNovel/LibNovel/Models/Models.swift
Normal file
@@ -0,0 +1,309 @@
|
||||
import Foundation
|
||||
import SwiftUI
|
||||
|
||||
// MARK: - Book
|
||||
|
||||
struct Book: Identifiable, Codable, Hashable {
|
||||
let id: String
|
||||
let slug: String
|
||||
let title: String
|
||||
let author: String
|
||||
let cover: String
|
||||
let status: String
|
||||
let genres: [String]
|
||||
let summary: String
|
||||
let totalChapters: Int
|
||||
let sourceURL: String
|
||||
let ranking: Int
|
||||
let metaUpdated: String
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id, slug, title, author, cover, status, genres, summary
|
||||
case totalChapters = "total_chapters"
|
||||
case sourceURL = "source_url"
|
||||
case ranking
|
||||
case metaUpdated = "meta_updated"
|
||||
}
|
||||
|
||||
// PocketBase returns genres as either a JSON string array or a real array
|
||||
init(from decoder: Decoder) throws {
|
||||
let container = try decoder.container(keyedBy: CodingKeys.self)
|
||||
id = try container.decode(String.self, forKey: .id)
|
||||
slug = try container.decode(String.self, forKey: .slug)
|
||||
title = try container.decode(String.self, forKey: .title)
|
||||
author = try container.decode(String.self, forKey: .author)
|
||||
cover = try container.decodeIfPresent(String.self, forKey: .cover) ?? ""
|
||||
status = try container.decodeIfPresent(String.self, forKey: .status) ?? ""
|
||||
totalChapters = try container.decodeIfPresent(Int.self, forKey: .totalChapters) ?? 0
|
||||
sourceURL = try container.decodeIfPresent(String.self, forKey: .sourceURL) ?? ""
|
||||
ranking = try container.decodeIfPresent(Int.self, forKey: .ranking) ?? 0
|
||||
metaUpdated = try container.decodeIfPresent(String.self, forKey: .metaUpdated) ?? ""
|
||||
summary = try container.decodeIfPresent(String.self, forKey: .summary) ?? ""
|
||||
|
||||
// genres is sometimes a JSON-encoded string, sometimes a real array
|
||||
if let arr = try? container.decode([String].self, forKey: .genres) {
|
||||
genres = arr
|
||||
} else if let str = try? container.decode(String.self, forKey: .genres),
|
||||
let data = str.data(using: .utf8),
|
||||
let arr = try? JSONDecoder().decode([String].self, from: data) {
|
||||
genres = arr
|
||||
} else {
|
||||
genres = []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - ChapterIndex
|
||||
|
||||
struct ChapterIndex: Identifiable, Codable, Hashable {
|
||||
let id: String
|
||||
let slug: String
|
||||
let number: Int
|
||||
let title: String
|
||||
let dateLabel: String
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id, slug, number, title
|
||||
case dateLabel = "date_label"
|
||||
}
|
||||
}
|
||||
|
||||
struct ChapterIndexBrief: Codable, Hashable {
|
||||
let number: Int
|
||||
let title: String
|
||||
}
|
||||
|
||||
// MARK: - User Settings
|
||||
|
||||
struct UserSettings: Codable {
|
||||
var id: String?
|
||||
var autoNext: Bool
|
||||
var voice: String
|
||||
var speed: Double
|
||||
|
||||
// Server sends/expects camelCase: { autoNext, voice, speed }
|
||||
// (No CodingKeys needed — Swift synthesises the same names by default)
|
||||
|
||||
static let `default` = UserSettings(id: nil, autoNext: false, voice: "af_bella", speed: 1.0)
|
||||
}
|
||||
|
||||
// MARK: - Reading Display Settings (local only — stored in UserDefaults)
|
||||
|
||||
enum ReaderTheme: String, CaseIterable, Codable {
|
||||
case white, sepia, night
|
||||
|
||||
var backgroundColor: Color {
|
||||
switch self {
|
||||
case .white: return Color(.sRGB, white: 1.0, opacity: 1)
|
||||
case .sepia: return Color(red: 0.97, green: 0.93, blue: 0.82)
|
||||
case .night: return Color(red: 0.10, green: 0.10, blue: 0.12)
|
||||
}
|
||||
}
|
||||
|
||||
var textColor: Color {
|
||||
switch self {
|
||||
case .white: return Color(.sRGB, white: 0.1, opacity: 1)
|
||||
case .sepia: return Color(red: 0.25, green: 0.18, blue: 0.08)
|
||||
case .night: return Color(red: 0.85, green: 0.85, blue: 0.87)
|
||||
}
|
||||
}
|
||||
|
||||
var colorScheme: ColorScheme? {
|
||||
switch self {
|
||||
case .white: return nil // follows system
|
||||
case .sepia: return .light
|
||||
case .night: return .dark
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum ReaderFont: String, CaseIterable, Codable {
|
||||
case system = "System"
|
||||
case georgia = "Georgia"
|
||||
case newYork = "New York"
|
||||
|
||||
var fontName: String? {
|
||||
switch self {
|
||||
case .system: return nil
|
||||
case .georgia: return "Georgia"
|
||||
case .newYork: return "NewYorkMedium-Regular"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ReaderSettings: Codable, Equatable {
|
||||
var fontSize: CGFloat
|
||||
var lineSpacing: CGFloat
|
||||
var font: ReaderFont
|
||||
var theme: ReaderTheme
|
||||
var scrollMode: Bool
|
||||
|
||||
static let `default` = ReaderSettings(
|
||||
fontSize: 17,
|
||||
lineSpacing: 1.7,
|
||||
font: .system,
|
||||
theme: .white,
|
||||
scrollMode: false
|
||||
)
|
||||
|
||||
static let userDefaultsKey = "readerSettings"
|
||||
|
||||
static func load() -> ReaderSettings {
|
||||
guard let data = UserDefaults.standard.data(forKey: userDefaultsKey),
|
||||
let decoded = try? JSONDecoder().decode(ReaderSettings.self, from: data)
|
||||
else { return .default }
|
||||
return decoded
|
||||
}
|
||||
|
||||
func save() {
|
||||
if let data = try? JSONEncoder().encode(self) {
|
||||
UserDefaults.standard.set(data, forKey: ReaderSettings.userDefaultsKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - User
|
||||
|
||||
struct AppUser: Codable, Identifiable {
|
||||
let id: String
|
||||
let username: String
|
||||
let role: String
|
||||
let created: String
|
||||
let avatarURL: String?
|
||||
|
||||
var isAdmin: Bool { role == "admin" }
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id, username, role, created
|
||||
case avatarURL = "avatar_url"
|
||||
}
|
||||
|
||||
init(id: String, username: String, role: String, created: String, avatarURL: String?) {
|
||||
self.id = id
|
||||
self.username = username
|
||||
self.role = role
|
||||
self.created = created
|
||||
self.avatarURL = avatarURL
|
||||
}
|
||||
|
||||
init(from decoder: Decoder) throws {
|
||||
let c = try decoder.container(keyedBy: CodingKeys.self)
|
||||
id = try c.decode(String.self, forKey: .id)
|
||||
username = try c.decode(String.self, forKey: .username)
|
||||
role = try c.decodeIfPresent(String.self, forKey: .role) ?? "user"
|
||||
created = try c.decodeIfPresent(String.self, forKey: .created) ?? ""
|
||||
avatarURL = try c.decodeIfPresent(String.self, forKey: .avatarURL)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Ranking
|
||||
|
||||
struct RankingItem: Codable, Identifiable {
|
||||
var id: String { slug }
|
||||
let rank: Int
|
||||
let slug: String
|
||||
let title: String
|
||||
let author: String
|
||||
let cover: String
|
||||
let status: String
|
||||
let genres: [String]
|
||||
let sourceURL: String
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case rank, slug, title, author, cover, status, genres
|
||||
case sourceURL = "source_url"
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Home
|
||||
|
||||
struct ContinueReadingItem: Identifiable {
|
||||
var id: String { book.id }
|
||||
let book: Book
|
||||
let chapter: Int
|
||||
}
|
||||
|
||||
struct HomeStats: Codable {
|
||||
let totalBooks: Int
|
||||
let totalChapters: Int
|
||||
let booksInProgress: Int
|
||||
}
|
||||
|
||||
// MARK: - Session
|
||||
|
||||
struct UserSession: Codable, Identifiable {
|
||||
let id: String
|
||||
let userAgent: String
|
||||
let ip: String
|
||||
let createdAt: String
|
||||
let lastSeen: String
|
||||
var isCurrent: Bool
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id
|
||||
case userAgent = "user_agent"
|
||||
case ip
|
||||
case createdAt = "created_at"
|
||||
case lastSeen = "last_seen"
|
||||
case isCurrent = "is_current"
|
||||
}
|
||||
}
|
||||
|
||||
struct PreviewChapter: Codable, Identifiable {
|
||||
var id: Int { number }
|
||||
let number: Int
|
||||
let title: String
|
||||
let url: String
|
||||
}
|
||||
|
||||
struct BookBrief: Codable {
|
||||
let slug: String
|
||||
let title: String
|
||||
let cover: String
|
||||
}
|
||||
|
||||
// MARK: - Comments
|
||||
|
||||
struct BookComment: Identifiable, Codable, Hashable {
|
||||
let id: String
|
||||
let slug: String
|
||||
let userId: String
|
||||
let username: String
|
||||
let body: String
|
||||
var upvotes: Int
|
||||
var downvotes: Int
|
||||
let created: String
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id, slug, username, body, upvotes, downvotes, created
|
||||
case userId = "user_id"
|
||||
}
|
||||
|
||||
init(from decoder: Decoder) throws {
|
||||
let c = try decoder.container(keyedBy: CodingKeys.self)
|
||||
id = try c.decode(String.self, forKey: .id)
|
||||
slug = try c.decodeIfPresent(String.self, forKey: .slug) ?? ""
|
||||
userId = try c.decodeIfPresent(String.self, forKey: .userId) ?? ""
|
||||
username = try c.decodeIfPresent(String.self, forKey: .username) ?? ""
|
||||
body = try c.decodeIfPresent(String.self, forKey: .body) ?? ""
|
||||
upvotes = try c.decodeIfPresent(Int.self, forKey: .upvotes) ?? 0
|
||||
downvotes = try c.decodeIfPresent(Int.self, forKey: .downvotes) ?? 0
|
||||
created = try c.decodeIfPresent(String.self, forKey: .created) ?? ""
|
||||
}
|
||||
}
|
||||
|
||||
struct CommentsResponse: Decodable {
|
||||
let comments: [BookComment]
|
||||
let myVotes: [String: String]
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case comments
|
||||
case myVotes = "myVotes"
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Audio
|
||||
|
||||
enum NextPrefetchStatus {
|
||||
case none, prefetching, prefetched, failed
|
||||
}
|
||||
521
ios/LibNovel/LibNovel/Networking/APIClient.swift
Normal file
521
ios/LibNovel/LibNovel/Networking/APIClient.swift
Normal file
@@ -0,0 +1,521 @@
|
||||
import Foundation
|
||||
|
||||
// MARK: - API Client
|
||||
// Communicates with the SvelteKit UI server (not directly with the Go scraper).
|
||||
// The SvelteKit layer handles auth, PocketBase queries, and MinIO presigning.
|
||||
// For the iOS app we talk to the same /api/* endpoints the web UI uses,
|
||||
// so we reuse the exact same HMAC-cookie auth flow.
|
||||
|
||||
actor APIClient {
|
||||
static let shared = APIClient()
|
||||
|
||||
var baseURL: URL
|
||||
private var authCookie: String? // raw "libnovel_auth=<token>" header value
|
||||
|
||||
// URLSession with persistent cookie storage
|
||||
private let session: URLSession = {
|
||||
let config = URLSessionConfiguration.default
|
||||
config.httpCookieAcceptPolicy = .always
|
||||
config.httpShouldSetCookies = true
|
||||
config.httpCookieStorage = HTTPCookieStorage.shared
|
||||
return URLSession(configuration: config)
|
||||
}()
|
||||
|
||||
private init() {
|
||||
// Default: point at the UI server. Override via Settings bundle or compile flag.
|
||||
let urlString = Bundle.main.object(forInfoDictionaryKey: "LIBNOVEL_BASE_URL") as? String
|
||||
?? "https://v2.libnovel.kalekber.cc"
|
||||
baseURL = URL(string: urlString)!
|
||||
}
|
||||
|
||||
// MARK: - Auth cookie management
|
||||
|
||||
func setAuthCookie(_ value: String?) {
|
||||
authCookie = value
|
||||
if let value {
|
||||
// Also inject into shared cookie storage so redirects carry the cookie
|
||||
let cookieProps: [HTTPCookiePropertyKey: Any] = [
|
||||
.name: "libnovel_auth",
|
||||
.value: value,
|
||||
.domain: baseURL.host ?? "localhost",
|
||||
.path: "/"
|
||||
]
|
||||
if let cookie = HTTPCookie(properties: cookieProps) {
|
||||
HTTPCookieStorage.shared.setCookie(cookie)
|
||||
}
|
||||
} else {
|
||||
// Clear
|
||||
let cookieStorage = HTTPCookieStorage.shared
|
||||
cookieStorage.cookies(for: baseURL)?.forEach { cookieStorage.deleteCookie($0) }
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Low-level request builder
|
||||
|
||||
private func makeRequest(_ path: String, method: String = "GET", body: Encodable? = nil) throws -> URLRequest {
|
||||
// Build URL by appending the path string directly to the base URL string.
|
||||
// appendingPathComponent() percent-encodes slashes, which breaks multi-segment
|
||||
// paths like /api/chapter/slug/1. URL(string:) preserves slashes correctly.
|
||||
let urlString = baseURL.absoluteString.trimmingCharacters(in: CharacterSet(charactersIn: "/"))
|
||||
+ "/" + path.trimmingCharacters(in: CharacterSet(charactersIn: "/"))
|
||||
guard let url = URL(string: urlString) else {
|
||||
throw APIError.invalidResponse
|
||||
}
|
||||
var req = URLRequest(url: url)
|
||||
req.httpMethod = method
|
||||
req.setValue("application/json", forHTTPHeaderField: "Accept")
|
||||
if let body {
|
||||
req.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
req.httpBody = try JSONEncoder().encode(body)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// MARK: - Generic fetch
|
||||
|
||||
func fetch<T: Decodable>(_ path: String, method: String = "GET", body: Encodable? = nil) async throws -> T {
|
||||
let req = try makeRequest(path, method: method, body: body)
|
||||
let (data, response) = try await session.data(for: req)
|
||||
guard let http = response as? HTTPURLResponse else {
|
||||
throw APIError.invalidResponse
|
||||
}
|
||||
let rawBody = String(data: data, encoding: .utf8) ?? "<non-utf8 data, \(data.count) bytes>"
|
||||
guard (200..<300).contains(http.statusCode) else {
|
||||
throw APIError.httpError(http.statusCode, rawBody)
|
||||
}
|
||||
do {
|
||||
return try JSONDecoder.iso8601.decode(T.self, from: data)
|
||||
} catch {
|
||||
throw APIError.decodingError(error)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Auth
|
||||
|
||||
struct LoginRequest: Encodable {
|
||||
let username: String
|
||||
let password: String
|
||||
}
|
||||
|
||||
struct LoginResponse: Decodable {
|
||||
let token: String
|
||||
let user: AppUser
|
||||
}
|
||||
|
||||
func login(username: String, password: String) async throws -> LoginResponse {
|
||||
try await fetch("/api/auth/login", method: "POST",
|
||||
body: LoginRequest(username: username, password: password))
|
||||
}
|
||||
|
||||
func register(username: String, password: String) async throws -> LoginResponse {
|
||||
try await fetch("/api/auth/register", method: "POST",
|
||||
body: LoginRequest(username: username, password: password))
|
||||
}
|
||||
|
||||
func logout() async throws {
|
||||
let _: EmptyResponse = try await fetch("/api/auth/logout", method: "POST")
|
||||
await setAuthCookie(nil)
|
||||
}
|
||||
|
||||
// MARK: - Home
|
||||
|
||||
func homeData() async throws -> HomeDataResponse {
|
||||
try await fetch("/api/home")
|
||||
}
|
||||
|
||||
// MARK: - Library
|
||||
|
||||
func library() async throws -> [LibraryItem] {
|
||||
try await fetch("/api/library")
|
||||
}
|
||||
|
||||
func saveBook(slug: String) async throws {
|
||||
let _: EmptyResponse = try await fetch("/api/library/\(slug)", method: "POST")
|
||||
}
|
||||
|
||||
func unsaveBook(slug: String) async throws {
|
||||
let _: EmptyResponse = try await fetch("/api/library/\(slug)", method: "DELETE")
|
||||
}
|
||||
|
||||
// MARK: - Book Detail
|
||||
|
||||
func bookDetail(slug: String) async throws -> BookDetailResponse {
|
||||
try await fetch("/api/book/\(slug)")
|
||||
}
|
||||
|
||||
// MARK: - Chapter
|
||||
|
||||
func chapterContent(slug: String, chapter: Int) async throws -> ChapterResponse {
|
||||
try await fetch("/api/chapter/\(slug)/\(chapter)")
|
||||
}
|
||||
|
||||
// MARK: - Browse
|
||||
|
||||
func browse(page: Int, genre: String = "all", sort: String = "popular", status: String = "all") async throws -> BrowseResponse {
|
||||
let query = "?page=\(page)&genre=\(genre)&sort=\(sort)&status=\(status)"
|
||||
return try await fetch("/api/browse-page\(query)")
|
||||
}
|
||||
|
||||
func search(query: String) async throws -> SearchResponse {
|
||||
let encoded = query.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? query
|
||||
return try await fetch("/api/search?q=\(encoded)")
|
||||
}
|
||||
|
||||
func ranking() async throws -> [RankingItem] {
|
||||
try await fetch("/api/ranking")
|
||||
}
|
||||
|
||||
// MARK: - Progress
|
||||
|
||||
func progress() async throws -> [ProgressEntry] {
|
||||
try await fetch("/api/progress")
|
||||
}
|
||||
|
||||
func setProgress(slug: String, chapter: Int) async throws {
|
||||
struct Body: Encodable { let chapter: Int }
|
||||
let _: EmptyResponse = try await fetch("/api/progress/\(slug)", method: "POST", body: Body(chapter: chapter))
|
||||
}
|
||||
|
||||
func audioTime(slug: String, chapter: Int) async throws -> Double? {
|
||||
struct Response: Decodable { let audioTime: Double?; enum CodingKeys: String, CodingKey { case audioTime = "audio_time" } }
|
||||
let r: Response = try await fetch("/api/progress/audio-time?slug=\(slug)&chapter=\(chapter)")
|
||||
return r.audioTime
|
||||
}
|
||||
|
||||
func setAudioTime(slug: String, chapter: Int, time: Double) async throws {
|
||||
struct Body: Encodable { let slug: String; let chapter: Int; let audioTime: Double; enum CodingKeys: String, CodingKey { case slug, chapter; case audioTime = "audio_time" } }
|
||||
let _: EmptyResponse = try await fetch("/api/progress/audio-time", method: "PATCH", body: Body(slug: slug, chapter: chapter, audioTime: time))
|
||||
}
|
||||
|
||||
// MARK: - Audio
|
||||
|
||||
func triggerAudio(slug: String, chapter: Int, voice: String, speed: Double) async throws -> AudioTriggerResponse {
|
||||
struct Body: Encodable { let voice: String; let speed: Double }
|
||||
return try await fetch("/api/audio/\(slug)/\(chapter)", method: "POST", body: Body(voice: voice, speed: speed))
|
||||
}
|
||||
|
||||
/// Poll GET /api/audio/status/{slug}/{n}?voice=... until the job is done or failed.
|
||||
/// Returns the presigned/proxy URL on success, throws on failure or cancellation.
|
||||
func pollAudioStatus(slug: String, chapter: Int, voice: String) async throws -> String {
|
||||
let path = "/api/audio/status/\(slug)/\(chapter)?voice=\(voice)"
|
||||
struct StatusResponse: Decodable {
|
||||
let status: String
|
||||
let url: String?
|
||||
let error: String?
|
||||
}
|
||||
while true {
|
||||
try Task.checkCancellation()
|
||||
let r: StatusResponse = try await fetch(path)
|
||||
switch r.status {
|
||||
case "done":
|
||||
guard let url = r.url, !url.isEmpty else {
|
||||
throw URLError(.badServerResponse)
|
||||
}
|
||||
return url
|
||||
case "failed":
|
||||
throw NSError(
|
||||
domain: "AudioGeneration",
|
||||
code: 0,
|
||||
userInfo: [NSLocalizedDescriptionKey: r.error ?? "Audio generation failed"]
|
||||
)
|
||||
default:
|
||||
// pending / generating / idle — keep polling
|
||||
try await Task.sleep(nanoseconds: 2_000_000_000) // 2 s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func presignAudio(slug: String, chapter: Int, voice: String) async throws -> String {
|
||||
struct Response: Decodable { let url: String }
|
||||
let r: Response = try await fetch("/api/presign/audio?slug=\(slug)&chapter=\(chapter)&voice=\(voice)")
|
||||
return r.url
|
||||
}
|
||||
|
||||
func presignVoiceSample(voice: String) async throws -> String {
|
||||
struct Response: Decodable { let url: String }
|
||||
let r: Response = try await fetch("/api/presign/voice-sample?voice=\(voice)")
|
||||
return r.url
|
||||
}
|
||||
|
||||
func voices() async throws -> [String] {
|
||||
struct Response: Decodable { let voices: [String] }
|
||||
let r: Response = try await fetch("/api/voices")
|
||||
return r.voices
|
||||
}
|
||||
|
||||
// MARK: - Settings
|
||||
|
||||
func settings() async throws -> UserSettings {
|
||||
try await fetch("/api/settings")
|
||||
}
|
||||
|
||||
func updateSettings(_ settings: UserSettings) async throws {
|
||||
let _: EmptyResponse = try await fetch("/api/settings", method: "PUT", body: settings)
|
||||
}
|
||||
|
||||
// MARK: - Sessions
|
||||
|
||||
func sessions() async throws -> [UserSession] {
|
||||
struct Response: Decodable { let sessions: [UserSession] }
|
||||
let r: Response = try await fetch("/api/sessions")
|
||||
return r.sessions
|
||||
}
|
||||
|
||||
func revokeSession(id: String) async throws {
|
||||
let _: EmptyResponse = try await fetch("/api/sessions/\(id)", method: "DELETE")
|
||||
}
|
||||
|
||||
// MARK: - Avatar
|
||||
|
||||
struct AvatarPresignResponse: Decodable {
|
||||
let uploadURL: String
|
||||
let key: String
|
||||
enum CodingKeys: String, CodingKey { case uploadURL = "upload_url"; case key }
|
||||
}
|
||||
|
||||
struct AvatarResponse: Decodable {
|
||||
let avatarURL: String?
|
||||
enum CodingKeys: String, CodingKey { case avatarURL = "avatar_url" }
|
||||
}
|
||||
|
||||
/// Upload a profile avatar using a two-step presigned PUT flow:
|
||||
/// 1. POST /api/profile/avatar → get a presigned PUT URL + object key
|
||||
/// 2. PUT image bytes directly to MinIO via the presigned URL
|
||||
/// 3. PATCH /api/profile/avatar with the key to record it in PocketBase
|
||||
/// Returns the presigned GET URL for the uploaded avatar.
|
||||
func uploadAvatar(_ imageData: Data, mimeType: String = "image/jpeg") async throws -> String? {
|
||||
// Step 1: request a presigned PUT URL from the SvelteKit server
|
||||
let presign: AvatarPresignResponse = try await fetch(
|
||||
"/api/profile/avatar",
|
||||
method: "POST",
|
||||
body: ["mime_type": mimeType]
|
||||
)
|
||||
|
||||
// Step 2: PUT the image bytes directly to MinIO
|
||||
guard let putURL = URL(string: presign.uploadURL) else { throw APIError.invalidResponse }
|
||||
var putReq = URLRequest(url: putURL)
|
||||
putReq.httpMethod = "PUT"
|
||||
putReq.setValue(mimeType, forHTTPHeaderField: "Content-Type")
|
||||
putReq.httpBody = imageData
|
||||
|
||||
let (_, putResp) = try await session.data(for: putReq)
|
||||
guard let putHttp = putResp as? HTTPURLResponse,
|
||||
(200..<300).contains(putHttp.statusCode) else {
|
||||
let code = (putResp as? HTTPURLResponse)?.statusCode ?? 0
|
||||
throw APIError.httpError(code, "MinIO PUT failed")
|
||||
}
|
||||
|
||||
// Step 3: record the key in PocketBase and get back a presigned GET URL
|
||||
let result: AvatarResponse = try await fetch(
|
||||
"/api/profile/avatar",
|
||||
method: "PATCH",
|
||||
body: ["key": presign.key]
|
||||
)
|
||||
return result.avatarURL
|
||||
}
|
||||
|
||||
/// Fetches a fresh presigned GET URL for the current user's avatar.
|
||||
/// Returns nil if the user has no avatar set.
|
||||
/// Used on cold launch / session restore to convert the stored raw key into a viewable URL.
|
||||
func fetchAvatarPresignedURL() async throws -> String? {
|
||||
let result: AvatarResponse = try await fetch("/api/profile/avatar")
|
||||
return result.avatarURL
|
||||
}
|
||||
|
||||
// MARK: - Comments
|
||||
|
||||
func fetchComments(slug: String) async throws -> CommentsResponse {
|
||||
try await fetch("/api/comments/\(slug)")
|
||||
}
|
||||
|
||||
struct PostCommentBody: Encodable { let body: String }
|
||||
|
||||
func postComment(slug: String, body: String) async throws -> BookComment {
|
||||
try await fetch("/api/comments/\(slug)", method: "POST", body: PostCommentBody(body: body))
|
||||
}
|
||||
|
||||
struct VoteBody: Encodable { let vote: String }
|
||||
|
||||
/// Cast, change, or toggle-off a vote on a comment.
|
||||
/// Returns the updated BookComment (with refreshed upvotes/downvotes counts).
|
||||
func voteComment(commentId: String, vote: String) async throws -> BookComment {
|
||||
try await fetch("/api/comments/\(commentId)/vote", method: "POST", body: VoteBody(vote: vote))
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Response types
|
||||
|
||||
struct HomeDataResponse: Decodable {
|
||||
struct ContinueItem: Decodable {
|
||||
let book: Book
|
||||
let chapter: Int
|
||||
}
|
||||
let continueReading: [ContinueItem]
|
||||
let recentlyUpdated: [Book]
|
||||
let stats: HomeStats
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case continueReading = "continue_reading"
|
||||
case recentlyUpdated = "recently_updated"
|
||||
case stats
|
||||
}
|
||||
}
|
||||
|
||||
struct LibraryItem: Decodable, Identifiable {
|
||||
var id: String { book.id }
|
||||
let book: Book
|
||||
let savedAt: String
|
||||
let lastChapter: Int?
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case book
|
||||
case savedAt = "saved_at"
|
||||
case lastChapter = "last_chapter"
|
||||
}
|
||||
}
|
||||
|
||||
struct BookDetailResponse: Decodable {
|
||||
let book: Book
|
||||
let chapters: [ChapterIndex]
|
||||
let previewChapters: [PreviewChapter]?
|
||||
let inLib: Bool
|
||||
let saved: Bool
|
||||
let lastChapter: Int?
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case book, chapters
|
||||
case previewChapters = "preview_chapters"
|
||||
case inLib = "in_lib"
|
||||
case saved
|
||||
case lastChapter = "last_chapter"
|
||||
}
|
||||
}
|
||||
|
||||
struct ChapterResponse: Decodable {
|
||||
let book: BookBrief
|
||||
let chapter: ChapterIndex
|
||||
let html: String
|
||||
let voices: [String]
|
||||
let prev: Int?
|
||||
let next: Int?
|
||||
let chapters: [ChapterIndexBrief]
|
||||
let isPreview: Bool
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case book, chapter, html, voices, prev, next, chapters
|
||||
case isPreview = "is_preview"
|
||||
}
|
||||
}
|
||||
|
||||
struct BrowseResponse: Decodable {
|
||||
let novels: [BrowseNovel]
|
||||
let page: Int
|
||||
let hasNext: Bool
|
||||
}
|
||||
|
||||
struct BrowseNovel: Decodable, Identifiable, Hashable {
|
||||
var id: String { slug.isEmpty ? url : slug }
|
||||
let slug: String
|
||||
let title: String
|
||||
let cover: String
|
||||
let rank: String
|
||||
let rating: String
|
||||
let chapters: String
|
||||
let url: String
|
||||
let author: String
|
||||
let status: String
|
||||
let genres: [String]
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case slug, title, cover, rank, rating, chapters, url, author, status, genres
|
||||
}
|
||||
|
||||
init(from decoder: Decoder) throws {
|
||||
let c = try decoder.container(keyedBy: CodingKeys.self)
|
||||
slug = try c.decodeIfPresent(String.self, forKey: .slug) ?? ""
|
||||
title = try c.decode(String.self, forKey: .title)
|
||||
cover = try c.decodeIfPresent(String.self, forKey: .cover) ?? ""
|
||||
rank = try c.decodeIfPresent(String.self, forKey: .rank) ?? ""
|
||||
rating = try c.decodeIfPresent(String.self, forKey: .rating) ?? ""
|
||||
chapters = try c.decodeIfPresent(String.self, forKey: .chapters) ?? ""
|
||||
url = try c.decodeIfPresent(String.self, forKey: .url) ?? ""
|
||||
author = try c.decodeIfPresent(String.self, forKey: .author) ?? ""
|
||||
status = try c.decodeIfPresent(String.self, forKey: .status) ?? ""
|
||||
genres = try c.decodeIfPresent([String].self, forKey: .genres) ?? []
|
||||
}
|
||||
}
|
||||
|
||||
struct SearchResponse: Decodable {
|
||||
let results: [BrowseNovel]
|
||||
let localCount: Int
|
||||
let remoteCount: Int
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case results
|
||||
case localCount = "local_count"
|
||||
case remoteCount = "remote_count"
|
||||
}
|
||||
}
|
||||
|
||||
/// Returned by POST /api/audio/{slug}/{n}.
|
||||
/// - 202 Accepted: job enqueued → poll via pollAudioStatus()
|
||||
/// - 200 OK: audio already cached → url is ready to play
|
||||
struct AudioTriggerResponse: Decodable {
|
||||
let jobId: String? // present on 202
|
||||
let status: String? // present on 202: "pending" | "generating"
|
||||
let url: String? // present on 200: proxy URL ready to play
|
||||
let filename: String? // present on 200
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case jobId = "job_id"
|
||||
case status, url, filename
|
||||
}
|
||||
|
||||
/// True when the server accepted the request and created an async job.
|
||||
var isAsync: Bool { jobId != nil }
|
||||
}
|
||||
|
||||
struct ProgressEntry: Decodable, Identifiable {
|
||||
var id: String { slug }
|
||||
let slug: String
|
||||
let chapter: Int
|
||||
let audioTime: Double?
|
||||
let updated: String
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case slug, chapter, updated
|
||||
case audioTime = "audio_time"
|
||||
}
|
||||
}
|
||||
|
||||
struct EmptyResponse: Decodable {}
|
||||
|
||||
// MARK: - API Error
|
||||
|
||||
enum APIError: LocalizedError {
|
||||
case invalidResponse
|
||||
case httpError(Int, String)
|
||||
case decodingError(Error)
|
||||
case unauthorized
|
||||
case networkError(Error)
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .invalidResponse: return "Invalid server response"
|
||||
case .httpError(let code, let msg): return "HTTP \(code): \(msg)"
|
||||
case .decodingError(let e): return "Decode error: \(e.localizedDescription)"
|
||||
case .unauthorized: return "Not authenticated"
|
||||
case .networkError(let e): return e.localizedDescription
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - JSONDecoder helper
|
||||
|
||||
extension JSONDecoder {
|
||||
static let iso8601: JSONDecoder = {
|
||||
let d = JSONDecoder()
|
||||
d.dateDecodingStrategy = .iso8601
|
||||
return d
|
||||
}()
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"colors": [
|
||||
{
|
||||
"color": {
|
||||
"color-space": "srgb",
|
||||
"components": { "alpha": "1.000", "blue": "0.040", "green": "0.620", "red": "0.960" }
|
||||
},
|
||||
"idiom": "universal"
|
||||
}
|
||||
],
|
||||
"info": { "author": "xcode", "version": 1 }
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"images" : [
|
||||
{
|
||||
"filename" : "icon-1024.png",
|
||||
"idiom" : "universal",
|
||||
"platform" : "ios",
|
||||
"size" : "1024x1024"
|
||||
}
|
||||
],
|
||||
"info" : {
|
||||
"author" : "xcode",
|
||||
"version" : 1
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 6.6 KiB |
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"info": { "author": "xcode", "version": 1 }
|
||||
}
|
||||
43
ios/LibNovel/LibNovel/Resources/Info.plist
Normal file
43
ios/LibNovel/LibNovel/Resources/Info.plist
Normal file
@@ -0,0 +1,43 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>LibNovel</string>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>$(EXECUTABLE_NAME)</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>LibNovel</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>$(MARKETING_VERSION)</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>1000</string>
|
||||
<key>LIBNOVEL_BASE_URL</key>
|
||||
<string>$(LIBNOVEL_BASE_URL)</string>
|
||||
<key>LSRequiresIPhoneOS</key>
|
||||
<true/>
|
||||
<key>UIBackgroundModes</key>
|
||||
<array>
|
||||
<string>audio</string>
|
||||
</array>
|
||||
<key>UILaunchScreen</key>
|
||||
<dict/>
|
||||
<key>UISupportedInterfaceOrientations</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
<key>UISupportedInterfaceOrientations~ipad</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationPortraitUpsideDown</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
</dict>
|
||||
</plist>
|
||||
616
ios/LibNovel/LibNovel/Services/AudioPlayerService.swift
Normal file
616
ios/LibNovel/LibNovel/Services/AudioPlayerService.swift
Normal file
@@ -0,0 +1,616 @@
|
||||
import Foundation
|
||||
import AVFoundation
|
||||
import MediaPlayer
|
||||
import Combine
|
||||
import Kingfisher
|
||||
|
||||
// MARK: - PlaybackProgress
|
||||
// Isolated ObservableObject for high-frequency playback state (currentTime,
|
||||
// duration, isPlaying). Keeping these separate from AudioPlayerService means
|
||||
// the 0.5-second time-observer ticks only invalidate views that explicitly
|
||||
// observe PlaybackProgress — menus and other stable UI are unaffected.
|
||||
|
||||
@MainActor
|
||||
final class PlaybackProgress: ObservableObject {
|
||||
@Published var currentTime: Double = 0
|
||||
@Published var duration: Double = 0
|
||||
@Published var isPlaying: Bool = false
|
||||
}
|
||||
|
||||
// MARK: - AudioPlayerService
|
||||
// Central singleton that owns AVPlayer, drives audio state, handles lock-screen
|
||||
// controls (NowPlayingInfoCenter + MPRemoteCommandCenter), and pre-fetches the
|
||||
// next chapter audio.
|
||||
|
||||
@MainActor
|
||||
final class AudioPlayerService: ObservableObject {
|
||||
|
||||
// MARK: - Published state
|
||||
|
||||
@Published var slug: String = ""
|
||||
@Published var chapter: Int = 0
|
||||
@Published var chapterTitle: String = ""
|
||||
@Published var bookTitle: String = ""
|
||||
@Published var coverURL: String = ""
|
||||
@Published var voice: String = "af_bella"
|
||||
@Published var speed: Double = 1.0
|
||||
@Published var chapters: [ChapterIndexBrief] = []
|
||||
|
||||
@Published var status: AudioPlayerStatus = .idle
|
||||
@Published var audioURL: String = ""
|
||||
@Published var errorMessage: String = ""
|
||||
@Published var generationProgress: Double = 0
|
||||
|
||||
/// High-frequency playback state (currentTime / duration / isPlaying).
|
||||
/// Views that only need the seek bar or play-pause button should observe
|
||||
/// this directly so they don't trigger re-renders of menu-bearing parents.
|
||||
let progress = PlaybackProgress()
|
||||
|
||||
// Convenience forwarders so non-view call sites keep compiling unchanged.
|
||||
var currentTime: Double {
|
||||
get { progress.currentTime }
|
||||
set { progress.currentTime = newValue }
|
||||
}
|
||||
var duration: Double {
|
||||
get { progress.duration }
|
||||
set { progress.duration = newValue }
|
||||
}
|
||||
var isPlaying: Bool {
|
||||
get { progress.isPlaying }
|
||||
set { progress.isPlaying = newValue }
|
||||
}
|
||||
|
||||
@Published var autoNext: Bool = false
|
||||
@Published var nextChapter: Int? = nil
|
||||
@Published var prevChapter: Int? = nil
|
||||
|
||||
@Published var sleepTimer: SleepTimerOption? = nil
|
||||
/// Human-readable countdown string shown in the full player near the moon button.
|
||||
/// e.g. "38:12" for minute-based, "2 ch left" for chapter-based, "" when off.
|
||||
@Published var sleepTimerRemainingText: String = ""
|
||||
|
||||
@Published var nextPrefetchStatus: NextPrefetchStatus = .none
|
||||
@Published var nextAudioURL: String = ""
|
||||
@Published var nextPrefetchedChapter: Int? = nil
|
||||
|
||||
var isActive: Bool {
|
||||
switch status {
|
||||
case .idle: return false
|
||||
default: return true
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Private
|
||||
|
||||
private var player: AVPlayer?
|
||||
private var playerItem: AVPlayerItem?
|
||||
private var timeObserver: Any?
|
||||
private var statusObserver: AnyCancellable?
|
||||
private var durationObserver: AnyCancellable?
|
||||
private var finishObserver: AnyCancellable?
|
||||
private var generationTask: Task<Void, Never>?
|
||||
private var prefetchTask: Task<Void, Never>?
|
||||
|
||||
// Cached cover image — downloaded once per chapter load, reused on every
|
||||
// updateNowPlaying() call so we don't re-download on every play/pause/seek.
|
||||
private var cachedCoverArtwork: MPMediaItemArtwork?
|
||||
private var cachedCoverURL: String = ""
|
||||
|
||||
// Sleep timer tracking
|
||||
private var sleepTimerTask: Task<Void, Never>?
|
||||
private var sleepTimerStartChapter: Int = 0
|
||||
/// Absolute deadline for minute-based timers (nil when not active or chapter-based).
|
||||
private var sleepTimerDeadline: Date? = nil
|
||||
/// 1-second tick task that keeps sleepTimerRemainingText up-to-date.
|
||||
private var sleepTimerCountdownTask: Task<Void, Never>? = nil
|
||||
|
||||
// MARK: - Init
|
||||
|
||||
init() {
|
||||
configureAudioSession()
|
||||
setupRemoteCommandCenter()
|
||||
}
|
||||
|
||||
// MARK: - Public API
|
||||
|
||||
/// Load audio for a specific chapter. Triggers TTS generation if not cached.
|
||||
func load(slug: String, chapter: Int, chapterTitle: String,
|
||||
bookTitle: String, coverURL: String, voice: String, speed: Double,
|
||||
chapters: [ChapterIndexBrief], nextChapter: Int?, prevChapter: Int?) {
|
||||
generationTask?.cancel()
|
||||
prefetchTask?.cancel()
|
||||
stop()
|
||||
|
||||
self.slug = slug
|
||||
self.chapter = chapter
|
||||
self.chapterTitle = chapterTitle
|
||||
self.bookTitle = bookTitle
|
||||
self.coverURL = coverURL
|
||||
self.voice = voice
|
||||
self.speed = speed
|
||||
self.chapters = chapters
|
||||
self.nextChapter = nextChapter
|
||||
self.prevChapter = prevChapter
|
||||
self.nextPrefetchStatus = .none
|
||||
self.nextAudioURL = ""
|
||||
self.nextPrefetchedChapter = nil
|
||||
|
||||
// Reset sleep timer start chapter if it's a chapter-based timer
|
||||
if case .chapters = sleepTimer {
|
||||
sleepTimerStartChapter = chapter
|
||||
}
|
||||
|
||||
status = .generating
|
||||
generationProgress = 0
|
||||
|
||||
// Invalidate cover cache if the book changed.
|
||||
if coverURL != cachedCoverURL {
|
||||
cachedCoverArtwork = nil
|
||||
cachedCoverURL = coverURL
|
||||
prefetchCoverArtwork(from: coverURL)
|
||||
}
|
||||
|
||||
generationTask = Task { await generateAudio() }
|
||||
}
|
||||
|
||||
func play() {
|
||||
player?.play()
|
||||
player?.rate = Float(speed)
|
||||
isPlaying = true
|
||||
updateNowPlaying()
|
||||
}
|
||||
|
||||
func pause() {
|
||||
player?.pause()
|
||||
isPlaying = false
|
||||
updateNowPlaying()
|
||||
}
|
||||
|
||||
func togglePlayPause() {
|
||||
isPlaying ? pause() : play()
|
||||
}
|
||||
|
||||
func seek(to seconds: Double) {
|
||||
let time = CMTime(seconds: seconds, preferredTimescale: 600)
|
||||
currentTime = seconds // optimistic UI update
|
||||
player?.seek(to: time, toleranceBefore: .zero, toleranceAfter: .zero) { [weak self] _ in
|
||||
guard let self else { return }
|
||||
Task { @MainActor in self.updateNowPlaying() }
|
||||
}
|
||||
}
|
||||
|
||||
func skip(by seconds: Double) {
|
||||
seek(to: max(0, min(currentTime + seconds, duration)))
|
||||
}
|
||||
|
||||
func setSpeed(_ newSpeed: Double) {
|
||||
speed = newSpeed
|
||||
if isPlaying { player?.rate = Float(newSpeed) }
|
||||
updateNowPlaying()
|
||||
}
|
||||
|
||||
func setSleepTimer(_ option: SleepTimerOption?) {
|
||||
// Cancel existing timer + countdown
|
||||
sleepTimerTask?.cancel()
|
||||
sleepTimerTask = nil
|
||||
sleepTimerCountdownTask?.cancel()
|
||||
sleepTimerCountdownTask = nil
|
||||
sleepTimerDeadline = nil
|
||||
|
||||
sleepTimer = option
|
||||
|
||||
guard let option else {
|
||||
sleepTimerRemainingText = ""
|
||||
return
|
||||
}
|
||||
|
||||
// Start timer based on option
|
||||
switch option {
|
||||
case .chapters(let count):
|
||||
sleepTimerStartChapter = chapter
|
||||
// Update display immediately; chapter changes are tracked in handlePlaybackFinished.
|
||||
updateChapterTimerLabel(chaptersRemaining: count)
|
||||
|
||||
case .minutes(let minutes):
|
||||
let deadline = Date().addingTimeInterval(Double(minutes) * 60)
|
||||
sleepTimerDeadline = deadline
|
||||
// Stop playback when the deadline is reached.
|
||||
sleepTimerTask = Task { [weak self] in
|
||||
try? await Task.sleep(nanoseconds: UInt64(minutes) * 60 * 1_000_000_000)
|
||||
guard let self, !Task.isCancelled else { return }
|
||||
await MainActor.run {
|
||||
self.stop()
|
||||
self.sleepTimer = nil
|
||||
self.sleepTimerRemainingText = ""
|
||||
}
|
||||
}
|
||||
// 1-second tick to keep the countdown label fresh.
|
||||
sleepTimerCountdownTask = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
try? await Task.sleep(nanoseconds: 1_000_000_000)
|
||||
guard let self, !Task.isCancelled else { return }
|
||||
await MainActor.run {
|
||||
guard let deadline = self.sleepTimerDeadline else { return }
|
||||
let remaining = max(0, deadline.timeIntervalSinceNow)
|
||||
self.sleepTimerRemainingText = Self.formatCountdown(remaining)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Set initial label without waiting for the first tick.
|
||||
sleepTimerRemainingText = Self.formatCountdown(Double(minutes) * 60)
|
||||
}
|
||||
}
|
||||
|
||||
private func updateChapterTimerLabel(chaptersRemaining: Int) {
|
||||
sleepTimerRemainingText = chaptersRemaining == 1 ? "1 ch left" : "\(chaptersRemaining) ch left"
|
||||
}
|
||||
|
||||
private static func formatCountdown(_ seconds: Double) -> String {
|
||||
let s = Int(max(0, seconds))
|
||||
let m = s / 60
|
||||
let sec = s % 60
|
||||
return "\(m):\(String(format: "%02d", sec))"
|
||||
}
|
||||
|
||||
func stop() {
|
||||
player?.pause()
|
||||
teardownPlayer()
|
||||
isPlaying = false
|
||||
currentTime = 0
|
||||
duration = 0
|
||||
audioURL = ""
|
||||
status = .idle
|
||||
|
||||
// Cancel sleep timer + countdown
|
||||
sleepTimerTask?.cancel()
|
||||
sleepTimerTask = nil
|
||||
sleepTimerCountdownTask?.cancel()
|
||||
sleepTimerCountdownTask = nil
|
||||
sleepTimerDeadline = nil
|
||||
sleepTimer = nil
|
||||
sleepTimerRemainingText = ""
|
||||
}
|
||||
|
||||
// MARK: - Audio generation
|
||||
|
||||
private func generateAudio() async {
|
||||
guard !slug.isEmpty, chapter > 0 else { return }
|
||||
do {
|
||||
// Fast path: audio already in MinIO — get a presigned URL and play immediately.
|
||||
if let presignedURL = try? await APIClient.shared.presignAudio(slug: slug, chapter: chapter, voice: voice) {
|
||||
audioURL = presignedURL
|
||||
status = .ready
|
||||
generationProgress = 100
|
||||
await playURL(presignedURL)
|
||||
await prefetchNext()
|
||||
return
|
||||
}
|
||||
|
||||
// Slow path: trigger TTS generation (async — returns 202 immediately).
|
||||
status = .generating
|
||||
generationProgress = 10
|
||||
let trigger = try await APIClient.shared.triggerAudio(slug: slug, chapter: chapter, voice: voice, speed: speed)
|
||||
|
||||
let playableURL: String
|
||||
if trigger.isAsync {
|
||||
// 202 Accepted: poll until done.
|
||||
generationProgress = 30
|
||||
playableURL = try await APIClient.shared.pollAudioStatus(slug: slug, chapter: chapter, voice: voice)
|
||||
} else {
|
||||
// 200: already cached URL returned inline.
|
||||
guard let url = trigger.url, !url.isEmpty else {
|
||||
throw URLError(.badServerResponse)
|
||||
}
|
||||
playableURL = url
|
||||
}
|
||||
|
||||
audioURL = playableURL
|
||||
status = .ready
|
||||
generationProgress = 100
|
||||
await playURL(playableURL)
|
||||
await prefetchNext()
|
||||
} catch is CancellationError {
|
||||
// Cancelled — no-op
|
||||
} catch {
|
||||
status = .error(error.localizedDescription)
|
||||
errorMessage = error.localizedDescription
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Prefetch next chapter
|
||||
// Always prefetch regardless of autoNext — faster playback when the user
|
||||
// manually navigates forward. autoNext only controls whether we auto-navigate.
|
||||
|
||||
private func prefetchNext() async {
|
||||
guard let next = nextChapter, !Task.isCancelled else { return }
|
||||
nextPrefetchStatus = .prefetching
|
||||
nextPrefetchedChapter = next
|
||||
do {
|
||||
// Fast path: already in MinIO.
|
||||
if let presignedURL = try? await APIClient.shared.presignAudio(slug: slug, chapter: next, voice: voice) {
|
||||
nextAudioURL = presignedURL
|
||||
nextPrefetchStatus = .prefetched
|
||||
return
|
||||
}
|
||||
// Slow path: trigger generation; poll until done (background — won't block playback).
|
||||
let trigger = try await APIClient.shared.triggerAudio(slug: slug, chapter: next, voice: voice, speed: speed)
|
||||
let url: String
|
||||
if trigger.isAsync {
|
||||
url = try await APIClient.shared.pollAudioStatus(slug: slug, chapter: next, voice: voice)
|
||||
} else {
|
||||
guard let u = trigger.url, !u.isEmpty else { throw URLError(.badServerResponse) }
|
||||
url = u
|
||||
}
|
||||
nextAudioURL = url
|
||||
nextPrefetchStatus = .prefetched
|
||||
} catch {
|
||||
nextPrefetchStatus = .failed
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - AVPlayer management
|
||||
|
||||
private func playURL(_ urlString: String) async {
|
||||
// Resolve relative paths (e.g. "/api/audio/...") to absolute URLs.
|
||||
let resolved: URL?
|
||||
if urlString.hasPrefix("http://") || urlString.hasPrefix("https://") {
|
||||
resolved = URL(string: urlString)
|
||||
} else {
|
||||
resolved = URL(string: urlString, relativeTo: await APIClient.shared.baseURL)?.absoluteURL
|
||||
}
|
||||
guard let url = resolved else { return }
|
||||
teardownPlayer()
|
||||
let item = AVPlayerItem(url: url)
|
||||
playerItem = item
|
||||
player = AVPlayer(playerItem: item)
|
||||
|
||||
// KVO: update duration as soon as asset metadata is loaded.
|
||||
durationObserver = item.publisher(for: \.duration)
|
||||
.receive(on: RunLoop.main)
|
||||
.sink { [weak self] dur in
|
||||
guard let self else { return }
|
||||
let secs = dur.seconds
|
||||
if secs.isFinite && secs > 0 {
|
||||
self.duration = secs
|
||||
self.updateNowPlaying()
|
||||
}
|
||||
}
|
||||
|
||||
// KVO: set playback rate once the item is ready.
|
||||
// Do NOT call player?.play() unconditionally — let readyToPlay trigger it
|
||||
// so we don't race between AVPlayer's internal buffering and our call.
|
||||
statusObserver = item.publisher(for: \.status)
|
||||
.receive(on: RunLoop.main)
|
||||
.sink { [weak self] itemStatus in
|
||||
guard let self else { return }
|
||||
if itemStatus == .readyToPlay {
|
||||
self.player?.rate = Float(self.speed)
|
||||
self.isPlaying = true
|
||||
self.updateNowPlaying()
|
||||
} else if itemStatus == .failed {
|
||||
self.status = .error(item.error?.localizedDescription ?? "Playback failed")
|
||||
self.errorMessage = item.error?.localizedDescription ?? "Playback failed"
|
||||
}
|
||||
}
|
||||
|
||||
// Periodic time observer for seek bar position.
|
||||
timeObserver = player?.addPeriodicTimeObserver(
|
||||
forInterval: CMTime(seconds: 0.5, preferredTimescale: 600),
|
||||
queue: .main
|
||||
) { [weak self] time in
|
||||
guard let self else { return }
|
||||
Task { @MainActor in
|
||||
let secs = time.seconds
|
||||
if secs.isFinite && secs >= 0 {
|
||||
self.currentTime = secs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Observe when playback ends.
|
||||
finishObserver = NotificationCenter.default
|
||||
.publisher(for: AVPlayerItem.didPlayToEndTimeNotification, object: item)
|
||||
.sink { [weak self] _ in
|
||||
Task { @MainActor in
|
||||
self?.handlePlaybackFinished()
|
||||
}
|
||||
}
|
||||
|
||||
// Kick off buffering — actual playback starts via statusObserver above.
|
||||
player?.play()
|
||||
}
|
||||
|
||||
private func teardownPlayer() {
|
||||
if let observer = timeObserver { player?.removeTimeObserver(observer) }
|
||||
timeObserver = nil
|
||||
statusObserver = nil
|
||||
durationObserver = nil
|
||||
finishObserver = nil
|
||||
player = nil
|
||||
playerItem = nil
|
||||
}
|
||||
|
||||
private func handlePlaybackFinished() {
|
||||
isPlaying = false
|
||||
|
||||
guard let next = nextChapter else { return }
|
||||
|
||||
// Check chapter-based sleep timer
|
||||
if case .chapters(let count) = sleepTimer {
|
||||
let chaptersPlayed = chapter - sleepTimerStartChapter + 1
|
||||
if chaptersPlayed >= count {
|
||||
stop()
|
||||
return
|
||||
}
|
||||
// Update the remaining chapters label.
|
||||
let remaining = count - chaptersPlayed
|
||||
updateChapterTimerLabel(chaptersRemaining: remaining)
|
||||
}
|
||||
|
||||
// Always notify the view that the chapter finished (it may update UI).
|
||||
NotificationCenter.default.post(
|
||||
name: .audioDidFinishChapter,
|
||||
object: nil,
|
||||
userInfo: ["next": next, "autoNext": autoNext]
|
||||
)
|
||||
|
||||
// If autoNext is on, load the next chapter internally right away.
|
||||
// We already have the metadata in `chapters`, so we can reconstruct
|
||||
// everything without waiting for the view to navigate.
|
||||
guard autoNext else { return }
|
||||
|
||||
let nextTitle = chapters.first(where: { $0.number == next })?.title ?? ""
|
||||
let nextNextChapter = chapters.first(where: { $0.number > next })?.number
|
||||
let nextPrevChapter: Int? = chapter // Current chapter becomes previous for the next one
|
||||
|
||||
// If we already prefetched a URL for the next chapter, skip straight to
|
||||
// playback and kick off generation in the background for the one after.
|
||||
if nextPrefetchStatus == .prefetched, !nextAudioURL.isEmpty {
|
||||
let url = nextAudioURL
|
||||
|
||||
// Advance state before tearing down the current player.
|
||||
chapter = next
|
||||
chapterTitle = nextTitle
|
||||
nextChapter = nextNextChapter
|
||||
prevChapter = nextPrevChapter
|
||||
nextPrefetchStatus = .none
|
||||
nextAudioURL = ""
|
||||
nextPrefetchedChapter = nil
|
||||
audioURL = url
|
||||
status = .ready
|
||||
generationProgress = 100
|
||||
|
||||
// Update sleep timer start chapter if using chapter-based timer
|
||||
if case .chapters = sleepTimer {
|
||||
sleepTimerStartChapter = next
|
||||
}
|
||||
|
||||
generationTask = Task {
|
||||
await playURL(url)
|
||||
await prefetchNext()
|
||||
}
|
||||
} else {
|
||||
// No prefetch available — do a full load.
|
||||
load(
|
||||
slug: slug,
|
||||
chapter: next,
|
||||
chapterTitle: nextTitle,
|
||||
bookTitle: bookTitle,
|
||||
coverURL: coverURL,
|
||||
voice: voice,
|
||||
speed: speed,
|
||||
chapters: chapters,
|
||||
nextChapter: nextNextChapter,
|
||||
prevChapter: nextPrevChapter
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Cover art prefetch
|
||||
|
||||
private func prefetchCoverArtwork(from urlString: String) {
|
||||
guard !urlString.isEmpty, let url = URL(string: urlString) else { return }
|
||||
KingfisherManager.shared.retrieveImage(with: url) { [weak self] result in
|
||||
guard let self else { return }
|
||||
if case .success(let value) = result {
|
||||
let image = value.image
|
||||
let artwork = MPMediaItemArtwork(boundsSize: image.size) { _ in image }
|
||||
Task { @MainActor in
|
||||
self.cachedCoverArtwork = artwork
|
||||
self.updateNowPlaying()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Audio Session
|
||||
|
||||
private func configureAudioSession() {
|
||||
do {
|
||||
try AVAudioSession.sharedInstance().setCategory(.playback, mode: .spokenAudio)
|
||||
try AVAudioSession.sharedInstance().setActive(true)
|
||||
} catch {
|
||||
// Non-fatal
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Lock Screen / Control Center
|
||||
|
||||
private func setupRemoteCommandCenter() {
|
||||
let center = MPRemoteCommandCenter.shared()
|
||||
center.playCommand.addTarget { [weak self] _ in
|
||||
self?.play()
|
||||
return .success
|
||||
}
|
||||
center.pauseCommand.addTarget { [weak self] _ in
|
||||
self?.pause()
|
||||
return .success
|
||||
}
|
||||
center.togglePlayPauseCommand.addTarget { [weak self] _ in
|
||||
self?.togglePlayPause()
|
||||
return .success
|
||||
}
|
||||
center.skipForwardCommand.preferredIntervals = [15]
|
||||
center.skipForwardCommand.addTarget { [weak self] _ in
|
||||
self?.skip(by: 15)
|
||||
return .success
|
||||
}
|
||||
center.skipBackwardCommand.preferredIntervals = [15]
|
||||
center.skipBackwardCommand.addTarget { [weak self] _ in
|
||||
self?.skip(by: -15)
|
||||
return .success
|
||||
}
|
||||
center.changePlaybackPositionCommand.addTarget { [weak self] event in
|
||||
if let e = event as? MPChangePlaybackPositionCommandEvent {
|
||||
self?.seek(to: e.positionTime)
|
||||
}
|
||||
return .success
|
||||
}
|
||||
}
|
||||
|
||||
private func updateNowPlaying() {
|
||||
var info: [String: Any] = [
|
||||
MPMediaItemPropertyTitle: chapterTitle.isEmpty ? "Chapter \(chapter)" : chapterTitle,
|
||||
MPMediaItemPropertyArtist: bookTitle,
|
||||
MPNowPlayingInfoPropertyElapsedPlaybackTime: currentTime,
|
||||
MPMediaItemPropertyPlaybackDuration: duration,
|
||||
MPNowPlayingInfoPropertyPlaybackRate: isPlaying ? speed : 0.0
|
||||
]
|
||||
// Use cached artwork — downloaded once in prefetchCoverArtwork().
|
||||
if let artwork = cachedCoverArtwork {
|
||||
info[MPMediaItemPropertyArtwork] = artwork
|
||||
}
|
||||
MPNowPlayingInfoCenter.default().nowPlayingInfo = info
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Supporting types
|
||||
|
||||
enum AudioPlayerStatus: Equatable {
|
||||
case idle
|
||||
case generating // covers both "loading" and "generating TTS" phases
|
||||
case ready
|
||||
case error(String)
|
||||
|
||||
static func == (lhs: AudioPlayerStatus, rhs: AudioPlayerStatus) -> Bool {
|
||||
switch (lhs, rhs) {
|
||||
case (.idle, .idle), (.generating, .generating), (.ready, .ready):
|
||||
return true
|
||||
case (.error(let a), .error(let b)):
|
||||
return a == b
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum SleepTimerOption: Equatable {
|
||||
case chapters(Int) // Stop after N chapters
|
||||
case minutes(Int) // Stop after N minutes
|
||||
}
|
||||
|
||||
extension Notification.Name {
|
||||
static let audioDidFinishChapter = Notification.Name("audioDidFinishChapter")
|
||||
static let skipToNextChapter = Notification.Name("skipToNextChapter")
|
||||
static let skipToPrevChapter = Notification.Name("skipToPrevChapter")
|
||||
}
|
||||
159
ios/LibNovel/LibNovel/Services/AuthStore.swift
Normal file
159
ios/LibNovel/LibNovel/Services/AuthStore.swift
Normal file
@@ -0,0 +1,159 @@
|
||||
import Foundation
|
||||
import Combine
|
||||
|
||||
// MARK: - AuthStore
|
||||
// Owns the authenticated user, the HMAC auth token, and user settings.
|
||||
// Persists the token to Keychain so the user stays logged in across launches.
|
||||
|
||||
@MainActor
|
||||
final class AuthStore: ObservableObject {
|
||||
@Published var user: AppUser?
|
||||
@Published var settings: UserSettings = .default
|
||||
@Published var isLoading: Bool = false
|
||||
@Published var error: String?
|
||||
|
||||
var isAuthenticated: Bool { user != nil }
|
||||
|
||||
private let keychainKey = "libnovel_auth_token"
|
||||
|
||||
init() {
|
||||
// Restore token from Keychain and validate it on launch
|
||||
if let token = loadToken() {
|
||||
Task { await validateToken(token) }
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Login / Register
|
||||
|
||||
func login(username: String, password: String) async {
|
||||
isLoading = true
|
||||
error = nil
|
||||
do {
|
||||
let response = try await APIClient.shared.login(username: username, password: password)
|
||||
await APIClient.shared.setAuthCookie(response.token)
|
||||
saveToken(response.token)
|
||||
user = response.user
|
||||
await loadSettings()
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
isLoading = false
|
||||
}
|
||||
|
||||
func register(username: String, password: String) async {
|
||||
isLoading = true
|
||||
error = nil
|
||||
do {
|
||||
let response = try await APIClient.shared.register(username: username, password: password)
|
||||
await APIClient.shared.setAuthCookie(response.token)
|
||||
saveToken(response.token)
|
||||
user = response.user
|
||||
await loadSettings()
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
isLoading = false
|
||||
}
|
||||
|
||||
func logout() async {
|
||||
do {
|
||||
try await APIClient.shared.logout()
|
||||
} catch {
|
||||
// Best-effort; clear local state regardless
|
||||
}
|
||||
clearToken()
|
||||
user = nil
|
||||
settings = .default
|
||||
}
|
||||
|
||||
// MARK: - Settings
|
||||
|
||||
func loadSettings() async {
|
||||
do {
|
||||
settings = try await APIClient.shared.settings()
|
||||
} catch {
|
||||
// Use defaults if settings endpoint fails
|
||||
}
|
||||
}
|
||||
|
||||
func saveSettings(_ updated: UserSettings) async {
|
||||
do {
|
||||
try await APIClient.shared.updateSettings(updated)
|
||||
settings = updated
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Token validation
|
||||
|
||||
/// Re-validates the current session and refreshes `user` + `settings`.
|
||||
/// Call this after any operation that may change the user record (e.g. avatar upload).
|
||||
func validateToken() async {
|
||||
guard let token = loadToken() else { return }
|
||||
await validateToken(token)
|
||||
}
|
||||
|
||||
private func validateToken(_ token: String) async {
|
||||
await APIClient.shared.setAuthCookie(token)
|
||||
// Use /api/auth/me to restore the user record and confirm the token is still valid
|
||||
do {
|
||||
async let me: AppUser = APIClient.shared.fetch("/api/auth/me")
|
||||
async let s: UserSettings = APIClient.shared.settings()
|
||||
var (restoredUser, restoredSettings) = try await (me, s)
|
||||
// /api/auth/me returns the raw MinIO object key for avatar_url, not a presigned URL.
|
||||
// Exchange the key for a fresh presigned GET URL so KFImage can display it.
|
||||
if let key = restoredUser.avatarURL, !key.hasPrefix("http") {
|
||||
if let presignedURL = try? await APIClient.shared.fetchAvatarPresignedURL() {
|
||||
restoredUser = AppUser(
|
||||
id: restoredUser.id,
|
||||
username: restoredUser.username,
|
||||
role: restoredUser.role,
|
||||
created: restoredUser.created,
|
||||
avatarURL: presignedURL
|
||||
)
|
||||
}
|
||||
}
|
||||
user = restoredUser
|
||||
settings = restoredSettings
|
||||
} catch let e as APIError {
|
||||
if case .httpError(let code, _) = e, code == 401 {
|
||||
clearToken()
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// MARK: - Keychain helpers
|
||||
|
||||
private func saveToken(_ token: String) {
|
||||
let data = Data(token.utf8)
|
||||
let query: [String: Any] = [
|
||||
kSecClass as String: kSecClassGenericPassword,
|
||||
kSecAttrAccount as String: keychainKey,
|
||||
kSecValueData as String: data
|
||||
]
|
||||
SecItemDelete(query as CFDictionary)
|
||||
SecItemAdd(query as CFDictionary, nil)
|
||||
}
|
||||
|
||||
private func loadToken() -> String? {
|
||||
let query: [String: Any] = [
|
||||
kSecClass as String: kSecClassGenericPassword,
|
||||
kSecAttrAccount as String: keychainKey,
|
||||
kSecReturnData as String: true,
|
||||
kSecMatchLimit as String: kSecMatchLimitOne
|
||||
]
|
||||
var item: CFTypeRef?
|
||||
guard SecItemCopyMatching(query as CFDictionary, &item) == errSecSuccess,
|
||||
let data = item as? Data else { return nil }
|
||||
return String(data: data, encoding: .utf8)
|
||||
}
|
||||
|
||||
private func clearToken() {
|
||||
let query: [String: Any] = [
|
||||
kSecClass as String: kSecClassGenericPassword,
|
||||
kSecAttrAccount as String: keychainKey
|
||||
]
|
||||
SecItemDelete(query as CFDictionary)
|
||||
}
|
||||
}
|
||||
47
ios/LibNovel/LibNovel/ViewModels/BookDetailViewModel.swift
Normal file
47
ios/LibNovel/LibNovel/ViewModels/BookDetailViewModel.swift
Normal file
@@ -0,0 +1,47 @@
|
||||
import Foundation
|
||||
|
||||
@MainActor
|
||||
final class BookDetailViewModel: ObservableObject {
|
||||
let slug: String
|
||||
|
||||
@Published var book: Book?
|
||||
@Published var chapters: [ChapterIndex] = []
|
||||
@Published var saved: Bool = false
|
||||
@Published var lastChapter: Int?
|
||||
@Published var isLoading = false
|
||||
@Published var error: String?
|
||||
|
||||
init(slug: String) {
|
||||
self.slug = slug
|
||||
}
|
||||
|
||||
func load() async {
|
||||
isLoading = true
|
||||
error = nil
|
||||
do {
|
||||
let detail = try await APIClient.shared.bookDetail(slug: slug)
|
||||
book = detail.book
|
||||
chapters = detail.chapters
|
||||
saved = detail.saved
|
||||
lastChapter = detail.lastChapter
|
||||
} catch {
|
||||
if !(error is CancellationError) {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
isLoading = false
|
||||
}
|
||||
|
||||
func toggleSaved() async {
|
||||
do {
|
||||
if saved {
|
||||
try await APIClient.shared.unsaveBook(slug: slug)
|
||||
} else {
|
||||
try await APIClient.shared.saveBook(slug: slug)
|
||||
}
|
||||
saved.toggle()
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
}
|
||||
73
ios/LibNovel/LibNovel/ViewModels/BrowseViewModel.swift
Normal file
73
ios/LibNovel/LibNovel/ViewModels/BrowseViewModel.swift
Normal file
@@ -0,0 +1,73 @@
|
||||
import Foundation
|
||||
|
||||
@MainActor
|
||||
final class BrowseViewModel: ObservableObject {
|
||||
@Published var novels: [BrowseNovel] = []
|
||||
@Published var sort: String = "popular"
|
||||
@Published var genre: String = "all"
|
||||
@Published var status: String = "all"
|
||||
@Published var searchQuery: String = ""
|
||||
@Published var isLoading = false
|
||||
@Published var hasNext = false
|
||||
@Published var error: String?
|
||||
|
||||
private var currentPage = 1
|
||||
private var isSearchMode = false
|
||||
|
||||
func loadFirstPage() async {
|
||||
currentPage = 1
|
||||
novels = []
|
||||
isSearchMode = false
|
||||
await loadPage(1)
|
||||
}
|
||||
|
||||
func loadNextPage() async {
|
||||
guard hasNext, !isLoading else { return }
|
||||
await loadPage(currentPage + 1)
|
||||
}
|
||||
|
||||
func search() async {
|
||||
guard !searchQuery.isEmpty else { await loadFirstPage(); return }
|
||||
isLoading = true
|
||||
isSearchMode = true
|
||||
novels = []
|
||||
error = nil
|
||||
do {
|
||||
let result = try await APIClient.shared.search(query: searchQuery)
|
||||
novels = result.results
|
||||
hasNext = false
|
||||
} catch {
|
||||
if !(error is CancellationError) {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
isLoading = false
|
||||
}
|
||||
|
||||
func clearSearch() {
|
||||
searchQuery = ""
|
||||
Task { await loadFirstPage() }
|
||||
}
|
||||
|
||||
private func loadPage(_ page: Int) async {
|
||||
isLoading = true
|
||||
error = nil
|
||||
do {
|
||||
let result = try await APIClient.shared.browse(
|
||||
page: page, genre: genre, sort: sort, status: status
|
||||
)
|
||||
if page == 1 {
|
||||
novels = result.novels
|
||||
} else {
|
||||
novels.append(contentsOf: result.novels)
|
||||
}
|
||||
hasNext = result.hasNext
|
||||
currentPage = page
|
||||
} catch {
|
||||
if !(error is CancellationError) {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
isLoading = false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import Foundation
|
||||
|
||||
@MainActor
|
||||
final class ChapterReaderViewModel: ObservableObject {
|
||||
let slug: String
|
||||
private(set) var chapter: Int
|
||||
|
||||
@Published var content: ChapterResponse?
|
||||
@Published var isLoading = false
|
||||
@Published var error: String?
|
||||
|
||||
init(slug: String, chapter: Int) {
|
||||
self.slug = slug
|
||||
self.chapter = chapter
|
||||
}
|
||||
|
||||
/// Switch to a different chapter in-place: resets state and updates `chapter`
|
||||
/// so that `.task(id: currentChapter)` in the View re-fires `load()`.
|
||||
func switchChapter(to newChapter: Int) {
|
||||
guard newChapter != chapter else { return }
|
||||
chapter = newChapter
|
||||
content = nil
|
||||
error = nil
|
||||
}
|
||||
|
||||
func load() async {
|
||||
isLoading = true
|
||||
error = nil
|
||||
do {
|
||||
content = try await APIClient.shared.chapterContent(slug: slug, chapter: chapter)
|
||||
// Record reading progress
|
||||
try? await APIClient.shared.setProgress(slug: slug, chapter: chapter)
|
||||
} catch {
|
||||
if !(error is CancellationError) {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
isLoading = false
|
||||
}
|
||||
|
||||
func toggleAudio(audioPlayer: AudioPlayerService, settings: UserSettings) {
|
||||
guard let content else { return }
|
||||
|
||||
// Only treat as "current" if the player is active (not idle/stopped).
|
||||
// If the user stopped playback, isActive is false — we must re-load.
|
||||
let isCurrent = audioPlayer.isActive &&
|
||||
audioPlayer.slug == slug &&
|
||||
audioPlayer.chapter == chapter
|
||||
|
||||
if isCurrent {
|
||||
audioPlayer.togglePlayPause()
|
||||
} else {
|
||||
let nextChapter: Int? = content.next
|
||||
let prevChapter: Int? = content.prev
|
||||
audioPlayer.load(
|
||||
slug: slug,
|
||||
chapter: chapter,
|
||||
chapterTitle: content.chapter.title,
|
||||
bookTitle: content.book.title,
|
||||
coverURL: content.book.cover,
|
||||
voice: settings.voice,
|
||||
speed: settings.speed,
|
||||
chapters: content.chapters,
|
||||
nextChapter: nextChapter,
|
||||
prevChapter: prevChapter
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
28
ios/LibNovel/LibNovel/ViewModels/HomeViewModel.swift
Normal file
28
ios/LibNovel/LibNovel/ViewModels/HomeViewModel.swift
Normal file
@@ -0,0 +1,28 @@
|
||||
import Foundation
|
||||
|
||||
@MainActor
|
||||
final class HomeViewModel: ObservableObject {
|
||||
@Published var continueReading: [ContinueReadingItem] = []
|
||||
@Published var recentlyUpdated: [Book] = []
|
||||
@Published var stats: HomeStats?
|
||||
@Published var isLoading = false
|
||||
@Published var error: String?
|
||||
|
||||
func load() async {
|
||||
isLoading = true
|
||||
error = nil
|
||||
do {
|
||||
let data = try await APIClient.shared.homeData()
|
||||
continueReading = data.continueReading.map {
|
||||
ContinueReadingItem(book: $0.book, chapter: $0.chapter)
|
||||
}
|
||||
recentlyUpdated = data.recentlyUpdated
|
||||
stats = data.stats
|
||||
} catch {
|
||||
if !(error is CancellationError) {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
isLoading = false
|
||||
}
|
||||
}
|
||||
21
ios/LibNovel/LibNovel/ViewModels/LibraryViewModel.swift
Normal file
21
ios/LibNovel/LibNovel/ViewModels/LibraryViewModel.swift
Normal file
@@ -0,0 +1,21 @@
|
||||
import Foundation
|
||||
|
||||
@MainActor
|
||||
final class LibraryViewModel: ObservableObject {
|
||||
@Published var items: [LibraryItem] = []
|
||||
@Published var isLoading = false
|
||||
@Published var error: String?
|
||||
|
||||
func load() async {
|
||||
isLoading = true
|
||||
error = nil
|
||||
do {
|
||||
items = try await APIClient.shared.library()
|
||||
} catch {
|
||||
if !(error is CancellationError) {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
isLoading = false
|
||||
}
|
||||
}
|
||||
40
ios/LibNovel/LibNovel/ViewModels/ProfileViewModel.swift
Normal file
40
ios/LibNovel/LibNovel/ViewModels/ProfileViewModel.swift
Normal file
@@ -0,0 +1,40 @@
|
||||
import Foundation
|
||||
|
||||
@MainActor
|
||||
final class ProfileViewModel: ObservableObject {
|
||||
@Published var sessions: [UserSession] = []
|
||||
@Published var voices: [String] = []
|
||||
@Published var sessionsLoading = false
|
||||
@Published var error: String?
|
||||
|
||||
func loadSessions() async {
|
||||
sessionsLoading = true
|
||||
do {
|
||||
sessions = try await APIClient.shared.sessions()
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
sessionsLoading = false
|
||||
}
|
||||
|
||||
func loadVoices() async {
|
||||
guard voices.isEmpty else { return }
|
||||
do {
|
||||
voices = try await APIClient.shared.voices()
|
||||
} catch {
|
||||
// Use hardcoded fallback — same as Go server helpers.go
|
||||
voices = ["af_bella", "af_sky", "af_sarah", "af_nicole",
|
||||
"am_adam", "am_michael", "bf_emma", "bf_isabella",
|
||||
"bm_george", "bm_lewis"]
|
||||
}
|
||||
}
|
||||
|
||||
func revokeSession(id: String) async {
|
||||
do {
|
||||
try await APIClient.shared.revokeSession(id: id)
|
||||
sessions.removeAll { $0.id == id }
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
}
|
||||
123
ios/LibNovel/LibNovel/Views/Auth/AuthView.swift
Normal file
123
ios/LibNovel/LibNovel/Views/Auth/AuthView.swift
Normal file
@@ -0,0 +1,123 @@
|
||||
import SwiftUI
|
||||
|
||||
struct AuthView: View {
|
||||
@EnvironmentObject var authStore: AuthStore
|
||||
@State private var mode: Mode = .login
|
||||
@State private var username: String = ""
|
||||
@State private var password: String = ""
|
||||
@State private var confirmPassword: String = ""
|
||||
@FocusState private var focusedField: Field?
|
||||
|
||||
enum Mode { case login, register }
|
||||
enum Field { case username, password, confirmPassword }
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
VStack(spacing: 0) {
|
||||
// Logo / header
|
||||
VStack(spacing: 8) {
|
||||
Image(systemName: "books.vertical.fill")
|
||||
.font(.system(size: 56))
|
||||
.foregroundStyle(.amber)
|
||||
Text("LibNovel")
|
||||
.font(.largeTitle.bold())
|
||||
}
|
||||
.padding(.top, 60)
|
||||
.padding(.bottom, 40)
|
||||
|
||||
// Tab switcher
|
||||
Picker("Mode", selection: $mode) {
|
||||
Text("Sign In").tag(Mode.login)
|
||||
Text("Create Account").tag(Mode.register)
|
||||
}
|
||||
.pickerStyle(.segmented)
|
||||
.padding(.horizontal, 24)
|
||||
.padding(.bottom, 32)
|
||||
|
||||
// Form
|
||||
VStack(spacing: 16) {
|
||||
TextField("Username", text: $username)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.textInputAutocapitalization(.never)
|
||||
.autocorrectionDisabled()
|
||||
.focused($focusedField, equals: .username)
|
||||
.submitLabel(.next)
|
||||
.onSubmit { focusedField = .password }
|
||||
|
||||
SecureField("Password", text: $password)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.focused($focusedField, equals: .password)
|
||||
.submitLabel(mode == .register ? .next : .go)
|
||||
.onSubmit {
|
||||
if mode == .register { focusedField = .confirmPassword }
|
||||
else { submit() }
|
||||
}
|
||||
|
||||
if mode == .register {
|
||||
SecureField("Confirm Password", text: $confirmPassword)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.focused($focusedField, equals: .confirmPassword)
|
||||
.submitLabel(.go)
|
||||
.onSubmit { submit() }
|
||||
.transition(.opacity.combined(with: .move(edge: .top)))
|
||||
}
|
||||
}
|
||||
.padding(.horizontal, 24)
|
||||
.animation(.easeInOut(duration: 0.2), value: mode)
|
||||
|
||||
if let error = authStore.error {
|
||||
Text(error)
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.red)
|
||||
.multilineTextAlignment(.center)
|
||||
.padding(.horizontal, 24)
|
||||
.padding(.top, 8)
|
||||
}
|
||||
|
||||
Button(action: submit) {
|
||||
Group {
|
||||
if authStore.isLoading {
|
||||
ProgressView()
|
||||
.progressViewStyle(.circular)
|
||||
.tint(.white)
|
||||
} else {
|
||||
Text(mode == .login ? "Sign In" : "Create Account")
|
||||
.fontWeight(.semibold)
|
||||
}
|
||||
}
|
||||
.frame(maxWidth: .infinity)
|
||||
.frame(height: 50)
|
||||
}
|
||||
.buttonStyle(.borderedProminent)
|
||||
.tint(.amber)
|
||||
.padding(.horizontal, 24)
|
||||
.padding(.top, 24)
|
||||
.disabled(authStore.isLoading || !formIsValid)
|
||||
|
||||
Spacer()
|
||||
}
|
||||
.toolbar(.hidden, for: .navigationBar)
|
||||
}
|
||||
.onChange(of: mode) { _, _ in
|
||||
authStore.error = nil
|
||||
confirmPassword = ""
|
||||
}
|
||||
}
|
||||
|
||||
private var formIsValid: Bool {
|
||||
let base = !username.isEmpty && password.count >= 4
|
||||
if mode == .register { return base && password == confirmPassword }
|
||||
return base
|
||||
}
|
||||
|
||||
private func submit() {
|
||||
focusedField = nil
|
||||
Task {
|
||||
if mode == .login {
|
||||
await authStore.login(username: username, password: password)
|
||||
} else {
|
||||
await authStore.register(username: username, password: password)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
390
ios/LibNovel/LibNovel/Views/BookDetail/BookDetailView.swift
Normal file
390
ios/LibNovel/LibNovel/Views/BookDetail/BookDetailView.swift
Normal file
@@ -0,0 +1,390 @@
|
||||
import SwiftUI
|
||||
import Kingfisher
|
||||
|
||||
struct BookDetailView: View {
|
||||
let slug: String
|
||||
@StateObject private var vm: BookDetailViewModel
|
||||
@EnvironmentObject var authStore: AuthStore
|
||||
@EnvironmentObject var audioPlayer: AudioPlayerService
|
||||
@State private var summaryExpanded = false
|
||||
@State private var chapterPage = 0
|
||||
private let pageSize = 50
|
||||
|
||||
init(slug: String) {
|
||||
self.slug = slug
|
||||
_vm = StateObject(wrappedValue: BookDetailViewModel(slug: slug))
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
ZStack(alignment: .top) {
|
||||
// Scroll content
|
||||
ScrollView {
|
||||
VStack(alignment: .leading, spacing: 0) {
|
||||
if vm.isLoading {
|
||||
ProgressView().frame(maxWidth: .infinity).padding(.top, 120)
|
||||
} else if let book = vm.book {
|
||||
heroSection(book: book)
|
||||
metaSection(book: book)
|
||||
Divider().padding(.horizontal)
|
||||
chapterSection(book: book)
|
||||
Divider().padding(.horizontal)
|
||||
CommentsView(slug: slug)
|
||||
}
|
||||
}
|
||||
}
|
||||
.ignoresSafeArea(edges: .top)
|
||||
}
|
||||
.navigationBarTitleDisplayMode(.inline)
|
||||
.toolbar { bookmarkButton }
|
||||
.task { await vm.load() }
|
||||
.errorAlert($vm.error)
|
||||
}
|
||||
|
||||
// MARK: - Hero
|
||||
|
||||
@ViewBuilder
|
||||
private func heroSection(book: Book) -> some View {
|
||||
ZStack(alignment: .bottom) {
|
||||
// Full-bleed blurred background
|
||||
KFImage(URL(string: book.cover))
|
||||
.resizable()
|
||||
.scaledToFill()
|
||||
.frame(maxWidth: .infinity)
|
||||
.frame(height: 320)
|
||||
.blur(radius: 24)
|
||||
.clipped()
|
||||
.overlay(
|
||||
LinearGradient(
|
||||
colors: [.black.opacity(0.15), .black.opacity(0.68)],
|
||||
startPoint: .top,
|
||||
endPoint: .bottom
|
||||
)
|
||||
)
|
||||
|
||||
// Cover + info column centered
|
||||
VStack(spacing: 16) {
|
||||
// Isolated cover with 3D-style shadow
|
||||
KFImage(URL(string: book.cover))
|
||||
.resizable()
|
||||
.placeholder {
|
||||
RoundedRectangle(cornerRadius: 12)
|
||||
.fill(Color(.systemGray5))
|
||||
}
|
||||
.scaledToFill()
|
||||
.frame(width: 130, height: 188)
|
||||
.clipShape(RoundedRectangle(cornerRadius: 12))
|
||||
.shadow(color: .black.opacity(0.55), radius: 18, x: 0, y: 10)
|
||||
.shadow(color: .black.opacity(0.3), radius: 6, x: 0, y: 3)
|
||||
|
||||
// Title + author
|
||||
VStack(spacing: 6) {
|
||||
Text(book.title)
|
||||
.font(.title3.bold())
|
||||
.foregroundStyle(.white)
|
||||
.multilineTextAlignment(.center)
|
||||
.lineLimit(3)
|
||||
.padding(.horizontal, 32)
|
||||
|
||||
Text(book.author)
|
||||
.font(.subheadline)
|
||||
.foregroundStyle(.white.opacity(0.75))
|
||||
}
|
||||
|
||||
// Genre tags
|
||||
if !book.genres.isEmpty {
|
||||
HStack(spacing: 8) {
|
||||
ForEach(book.genres.prefix(3), id: \.self) { genre in
|
||||
TagChip(label: genre).colorScheme(.dark)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Status badge
|
||||
if !book.status.isEmpty {
|
||||
StatusBadge(status: book.status)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
.padding(.bottom, 28)
|
||||
}
|
||||
.frame(minHeight: 320)
|
||||
}
|
||||
|
||||
// MARK: - Meta section (summary + CTAs)
|
||||
|
||||
@ViewBuilder
|
||||
private func metaSection(book: Book) -> some View {
|
||||
VStack(alignment: .leading, spacing: 0) {
|
||||
// Quick stats row
|
||||
HStack(spacing: 0) {
|
||||
MetaStat(value: "\(book.totalChapters)", label: "Chapters",
|
||||
icon: "doc.text")
|
||||
Divider().frame(height: 36)
|
||||
MetaStat(value: book.status.capitalized.isEmpty ? "—" : book.status.capitalized,
|
||||
label: "Status", icon: "flag")
|
||||
if book.ranking > 0 {
|
||||
Divider().frame(height: 36)
|
||||
MetaStat(value: "#\(book.ranking)", label: "Rank",
|
||||
icon: "chart.bar.fill")
|
||||
}
|
||||
}
|
||||
.padding(.vertical, 16)
|
||||
.frame(maxWidth: .infinity)
|
||||
|
||||
Divider().padding(.horizontal)
|
||||
|
||||
// Summary
|
||||
VStack(alignment: .leading, spacing: 8) {
|
||||
Text("About")
|
||||
.font(.headline)
|
||||
|
||||
Text(book.summary)
|
||||
.font(.subheadline)
|
||||
.foregroundStyle(.secondary)
|
||||
.lineLimit(summaryExpanded ? nil : 4)
|
||||
.animation(.easeInOut(duration: 0.2), value: summaryExpanded)
|
||||
|
||||
if book.summary.count > 200 {
|
||||
Button(summaryExpanded ? "Less" : "More") {
|
||||
withAnimation { summaryExpanded.toggle() }
|
||||
}
|
||||
.font(.caption.bold())
|
||||
.foregroundStyle(.amber)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
.padding(.vertical, 16)
|
||||
|
||||
Divider().padding(.horizontal)
|
||||
|
||||
// CTA buttons
|
||||
HStack(spacing: 10) {
|
||||
if let last = vm.lastChapter, last > 0 {
|
||||
NavigationLink(value: NavDestination.chapter(slug, last)) {
|
||||
Label("Continue Ch.\(last)", systemImage: "play.fill")
|
||||
.frame(maxWidth: .infinity)
|
||||
.fontWeight(.semibold)
|
||||
}
|
||||
.buttonStyle(.borderedProminent)
|
||||
.tint(.amber)
|
||||
|
||||
NavigationLink(value: NavDestination.chapter(slug, 1)) {
|
||||
Label("Ch.1", systemImage: "arrow.counterclockwise")
|
||||
.frame(maxWidth: .infinity)
|
||||
}
|
||||
.buttonStyle(.bordered)
|
||||
.tint(.secondary)
|
||||
} else {
|
||||
NavigationLink(value: NavDestination.chapter(slug, 1)) {
|
||||
Label("Start Reading", systemImage: "book.fill")
|
||||
.frame(maxWidth: .infinity)
|
||||
.fontWeight(.semibold)
|
||||
}
|
||||
.buttonStyle(.borderedProminent)
|
||||
.tint(.amber)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
.padding(.vertical, 16)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Chapter list
|
||||
|
||||
@ViewBuilder
|
||||
private func chapterSection(book: Book) -> some View {
|
||||
let chapters = vm.chapters
|
||||
let total = chapters.count
|
||||
let start = chapterPage * pageSize
|
||||
let end = min(start + pageSize, total)
|
||||
let pageChapters = Array(chapters[start..<end])
|
||||
|
||||
VStack(alignment: .leading, spacing: 0) {
|
||||
// Section header
|
||||
HStack {
|
||||
Text("Chapters")
|
||||
.font(.headline)
|
||||
Spacer()
|
||||
if total > 0 {
|
||||
Text("\(start + 1)–\(end) of \(total)")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
.padding(.vertical, 14)
|
||||
|
||||
if vm.isLoading {
|
||||
ProgressView().frame(maxWidth: .infinity).padding()
|
||||
} else {
|
||||
ForEach(pageChapters) { ch in
|
||||
NavigationLink(value: NavDestination.chapter(slug, ch.number)) {
|
||||
ChapterRow(chapter: ch, isCurrent: ch.number == vm.lastChapter,
|
||||
totalChapters: total)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
Divider().padding(.leading)
|
||||
}
|
||||
}
|
||||
|
||||
// Pagination bar
|
||||
if total > pageSize {
|
||||
HStack {
|
||||
Button {
|
||||
withAnimation { chapterPage -= 1 }
|
||||
} label: {
|
||||
Image(systemName: "chevron.left")
|
||||
Text("Previous")
|
||||
}
|
||||
.disabled(chapterPage == 0)
|
||||
|
||||
Spacer()
|
||||
|
||||
Text("Page \(chapterPage + 1) of \((total + pageSize - 1) / pageSize)")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
|
||||
Spacer()
|
||||
|
||||
Button {
|
||||
withAnimation { chapterPage += 1 }
|
||||
} label: {
|
||||
Text("Next")
|
||||
Image(systemName: "chevron.right")
|
||||
}
|
||||
.disabled(end >= total)
|
||||
}
|
||||
.font(.subheadline)
|
||||
.foregroundStyle(.amber)
|
||||
.padding()
|
||||
}
|
||||
|
||||
Color.clear.frame(height: 32)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Bookmark toolbar
|
||||
|
||||
@ToolbarContentBuilder
|
||||
private var bookmarkButton: some ToolbarContent {
|
||||
ToolbarItem(placement: .topBarTrailing) {
|
||||
Button {
|
||||
Task { await vm.toggleSaved() }
|
||||
} label: {
|
||||
Image(systemName: vm.saved ? "bookmark.fill" : "bookmark")
|
||||
.foregroundStyle(vm.saved ? .amber : .primary)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Chapter row
|
||||
|
||||
private struct ChapterRow: View {
|
||||
let chapter: ChapterIndex
|
||||
let isCurrent: Bool
|
||||
let totalChapters: Int
|
||||
|
||||
private var progressFraction: Double {
|
||||
guard totalChapters > 1 else { return 0 }
|
||||
return Double(chapter.number) / Double(totalChapters)
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
HStack(spacing: 10) {
|
||||
// Number badge
|
||||
ZStack {
|
||||
Circle()
|
||||
.fill(isCurrent ? Color.amber : Color(.systemGray6))
|
||||
Text("\(chapter.number)")
|
||||
.font(.caption2.bold().monospacedDigit())
|
||||
.foregroundStyle(isCurrent ? .black : .secondary)
|
||||
}
|
||||
.frame(width: 32, height: 32)
|
||||
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
let displayTitle: String = {
|
||||
let stripped = chapter.title.strippingTrailingDate()
|
||||
if stripped.isEmpty || stripped == "Chapter \(chapter.number)" {
|
||||
return "Chapter \(chapter.number)"
|
||||
}
|
||||
return stripped
|
||||
}()
|
||||
|
||||
Text(displayTitle)
|
||||
.font(.subheadline)
|
||||
.fontWeight(isCurrent ? .semibold : .regular)
|
||||
.foregroundStyle(isCurrent ? .amber : .primary)
|
||||
.lineLimit(1)
|
||||
}
|
||||
|
||||
Spacer(minLength: 8)
|
||||
|
||||
VStack(alignment: .trailing, spacing: 2) {
|
||||
if !chapter.dateLabel.isEmpty {
|
||||
Text(chapter.dateLabel)
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.tertiary)
|
||||
}
|
||||
}
|
||||
|
||||
Image(systemName: "chevron.right")
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.tertiary)
|
||||
}
|
||||
.padding(.horizontal, 16)
|
||||
.padding(.vertical, 10)
|
||||
.contentShape(Rectangle())
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Supporting components
|
||||
|
||||
private struct MetaStat: View {
|
||||
let value: String
|
||||
let label: String
|
||||
let icon: String
|
||||
|
||||
var body: some View {
|
||||
VStack(spacing: 4) {
|
||||
Image(systemName: icon)
|
||||
.font(.caption)
|
||||
.foregroundStyle(.amber)
|
||||
Text(value)
|
||||
.font(.subheadline.bold())
|
||||
.lineLimit(1)
|
||||
.minimumScaleFactor(0.7)
|
||||
Text(label)
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
.frame(maxWidth: .infinity)
|
||||
}
|
||||
}
|
||||
|
||||
private struct StatusBadge: View {
|
||||
let status: String
|
||||
|
||||
private var color: Color {
|
||||
switch status.lowercased() {
|
||||
case "ongoing", "active": return .green
|
||||
case "completed": return .blue
|
||||
case "hiatus": return .orange
|
||||
default: return .secondary
|
||||
}
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
HStack(spacing: 4) {
|
||||
Circle()
|
||||
.fill(color)
|
||||
.frame(width: 6, height: 6)
|
||||
Text(status.capitalized)
|
||||
.font(.caption.weight(.medium))
|
||||
.foregroundStyle(color)
|
||||
}
|
||||
.padding(.horizontal, 10)
|
||||
.padding(.vertical, 4)
|
||||
.background(color.opacity(0.12), in: Capsule())
|
||||
}
|
||||
}
|
||||
320
ios/LibNovel/LibNovel/Views/BookDetail/CommentsView.swift
Normal file
320
ios/LibNovel/LibNovel/Views/BookDetail/CommentsView.swift
Normal file
@@ -0,0 +1,320 @@
|
||||
import SwiftUI
|
||||
|
||||
// MARK: - ViewModel
|
||||
|
||||
@MainActor
|
||||
class CommentsViewModel: ObservableObject {
|
||||
let slug: String
|
||||
|
||||
@Published var comments: [BookComment] = []
|
||||
@Published var myVotes: [String: String] = [:] // commentId → "up" | "down"
|
||||
@Published var isLoading = true
|
||||
@Published var error: String?
|
||||
|
||||
@Published var newBody = ""
|
||||
@Published var isPosting = false
|
||||
@Published var postError: String?
|
||||
|
||||
private var votingIds: Set<String> = []
|
||||
|
||||
init(slug: String) {
|
||||
self.slug = slug
|
||||
}
|
||||
|
||||
func load() async {
|
||||
isLoading = true
|
||||
error = nil
|
||||
do {
|
||||
let response = try await APIClient.shared.fetchComments(slug: slug)
|
||||
comments = response.comments
|
||||
myVotes = response.myVotes
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
isLoading = false
|
||||
}
|
||||
|
||||
func postComment() async {
|
||||
let text = newBody.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !text.isEmpty, !isPosting else { return }
|
||||
if text.count > 2000 {
|
||||
postError = "Comment too long (max 2000 characters)."
|
||||
return
|
||||
}
|
||||
isPosting = true
|
||||
postError = nil
|
||||
do {
|
||||
let created = try await APIClient.shared.postComment(slug: slug, body: text)
|
||||
comments.insert(created, at: 0)
|
||||
newBody = ""
|
||||
} catch let apiError as APIError {
|
||||
switch apiError {
|
||||
case .httpError(401, _): postError = "You must be logged in to comment."
|
||||
default: postError = apiError.localizedDescription
|
||||
}
|
||||
} catch {
|
||||
postError = error.localizedDescription
|
||||
}
|
||||
isPosting = false
|
||||
}
|
||||
|
||||
func vote(commentId: String, vote: String) async {
|
||||
guard !votingIds.contains(commentId) else { return }
|
||||
votingIds.insert(commentId)
|
||||
defer { votingIds.remove(commentId) }
|
||||
do {
|
||||
let updated = try await APIClient.shared.voteComment(commentId: commentId, vote: vote)
|
||||
// Update the comment in the list
|
||||
if let idx = comments.firstIndex(where: { $0.id == commentId }) {
|
||||
comments[idx] = updated
|
||||
}
|
||||
// Toggle myVotes
|
||||
let prev = myVotes[commentId]
|
||||
if prev == vote {
|
||||
myVotes.removeValue(forKey: commentId)
|
||||
} else {
|
||||
myVotes[commentId] = vote
|
||||
}
|
||||
} catch {
|
||||
// Silently ignore vote errors — don't disrupt the UI
|
||||
}
|
||||
}
|
||||
|
||||
func isVoting(_ commentId: String) -> Bool {
|
||||
votingIds.contains(commentId)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - CommentsView
|
||||
|
||||
struct CommentsView: View {
|
||||
@StateObject private var vm: CommentsViewModel
|
||||
@EnvironmentObject private var authStore: AuthStore
|
||||
|
||||
init(slug: String) {
|
||||
_vm = StateObject(wrappedValue: CommentsViewModel(slug: slug))
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 0) {
|
||||
// Section header
|
||||
HStack {
|
||||
Text("Comments")
|
||||
.font(.headline)
|
||||
if !vm.isLoading && !vm.comments.isEmpty {
|
||||
Text("(\(vm.comments.count))")
|
||||
.font(.subheadline)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
Spacer()
|
||||
}
|
||||
.padding(.horizontal)
|
||||
.padding(.vertical, 14)
|
||||
|
||||
Divider().padding(.horizontal)
|
||||
|
||||
// Post form
|
||||
postForm
|
||||
.padding(.horizontal)
|
||||
.padding(.vertical, 12)
|
||||
|
||||
Divider().padding(.horizontal)
|
||||
|
||||
// Comment list
|
||||
if vm.isLoading {
|
||||
loadingPlaceholder
|
||||
} else if let err = vm.error {
|
||||
Text(err)
|
||||
.font(.subheadline)
|
||||
.foregroundStyle(.red)
|
||||
.padding()
|
||||
} else if vm.comments.isEmpty {
|
||||
Text("No comments yet. Be the first!")
|
||||
.font(.subheadline)
|
||||
.foregroundStyle(.secondary)
|
||||
.padding()
|
||||
} else {
|
||||
ForEach(vm.comments) { comment in
|
||||
CommentRow(
|
||||
comment: comment,
|
||||
myVote: vm.myVotes[comment.id],
|
||||
isVoting: vm.isVoting(comment.id)
|
||||
) { vote in
|
||||
Task { await vm.vote(commentId: comment.id, vote: vote) }
|
||||
}
|
||||
Divider().padding(.leading, 16)
|
||||
}
|
||||
}
|
||||
|
||||
Color.clear.frame(height: 16)
|
||||
}
|
||||
.task { await vm.load() }
|
||||
}
|
||||
|
||||
// MARK: - Post form
|
||||
|
||||
@ViewBuilder
|
||||
private var postForm: some View {
|
||||
VStack(alignment: .leading, spacing: 8) {
|
||||
ZStack(alignment: .topLeading) {
|
||||
if vm.newBody.isEmpty {
|
||||
Text("Write a comment…")
|
||||
.font(.subheadline)
|
||||
.foregroundStyle(.tertiary)
|
||||
.padding(.top, 8)
|
||||
.padding(.leading, 4)
|
||||
}
|
||||
TextEditor(text: $vm.newBody)
|
||||
.font(.subheadline)
|
||||
.frame(minHeight: 72, maxHeight: 160)
|
||||
.scrollContentBackground(.hidden)
|
||||
}
|
||||
.padding(10)
|
||||
.background(Color(.systemGray6), in: RoundedRectangle(cornerRadius: 10))
|
||||
|
||||
HStack {
|
||||
let count = vm.newBody.count
|
||||
Text("\(count)/2000")
|
||||
.font(.caption2)
|
||||
.monospacedDigit()
|
||||
.foregroundStyle(count > 2000 ? .red : .tertiary)
|
||||
|
||||
Spacer()
|
||||
|
||||
if let err = vm.postError {
|
||||
Text(err)
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.red)
|
||||
.lineLimit(1)
|
||||
}
|
||||
|
||||
Button {
|
||||
Task { await vm.postComment() }
|
||||
} label: {
|
||||
if vm.isPosting {
|
||||
ProgressView().controlSize(.small)
|
||||
} else {
|
||||
Text("Post")
|
||||
.fontWeight(.semibold)
|
||||
}
|
||||
}
|
||||
.buttonStyle(.borderedProminent)
|
||||
.tint(.amber)
|
||||
.controlSize(.small)
|
||||
.disabled(vm.isPosting || vm.newBody.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || vm.newBody.count > 2000)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Loading skeleton
|
||||
|
||||
@ViewBuilder
|
||||
private var loadingPlaceholder: some View {
|
||||
VStack(spacing: 12) {
|
||||
ForEach(0..<3, id: \.self) { _ in
|
||||
VStack(alignment: .leading, spacing: 8) {
|
||||
RoundedRectangle(cornerRadius: 4)
|
||||
.fill(Color(.systemGray5))
|
||||
.frame(width: 100, height: 12)
|
||||
RoundedRectangle(cornerRadius: 4)
|
||||
.fill(Color(.systemGray6))
|
||||
.frame(maxWidth: .infinity)
|
||||
.frame(height: 12)
|
||||
RoundedRectangle(cornerRadius: 4)
|
||||
.fill(Color(.systemGray6))
|
||||
.frame(width: 200, height: 12)
|
||||
}
|
||||
.padding(.horizontal)
|
||||
.redacted(reason: .placeholder)
|
||||
}
|
||||
}
|
||||
.padding(.vertical, 12)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - CommentRow
|
||||
|
||||
private struct CommentRow: View {
|
||||
let comment: BookComment
|
||||
let myVote: String?
|
||||
let isVoting: Bool
|
||||
let onVote: (String) -> Void
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
// Username + date
|
||||
HStack(spacing: 6) {
|
||||
Text(comment.username.isEmpty ? "Anonymous" : comment.username)
|
||||
.font(.subheadline.weight(.medium))
|
||||
Text("·")
|
||||
.foregroundStyle(.tertiary)
|
||||
Text(formattedDate(comment.created))
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
Spacer()
|
||||
}
|
||||
|
||||
// Body
|
||||
Text(comment.body)
|
||||
.font(.subheadline)
|
||||
.foregroundStyle(.primary)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
|
||||
// Vote row
|
||||
HStack(spacing: 16) {
|
||||
// Upvote
|
||||
Button {
|
||||
onVote("up")
|
||||
} label: {
|
||||
HStack(spacing: 4) {
|
||||
Image(systemName: myVote == "up" ? "hand.thumbsup.fill" : "hand.thumbsup")
|
||||
.font(.caption)
|
||||
Text("\(comment.upvotes)")
|
||||
.font(.caption.monospacedDigit())
|
||||
}
|
||||
.foregroundStyle(myVote == "up" ? Color.amber : .secondary)
|
||||
}
|
||||
.disabled(isVoting)
|
||||
|
||||
// Downvote
|
||||
Button {
|
||||
onVote("down")
|
||||
} label: {
|
||||
HStack(spacing: 4) {
|
||||
Image(systemName: myVote == "down" ? "hand.thumbsdown.fill" : "hand.thumbsdown")
|
||||
.font(.caption)
|
||||
Text("\(comment.downvotes)")
|
||||
.font(.caption.monospacedDigit())
|
||||
}
|
||||
.foregroundStyle(myVote == "down" ? .red : .secondary)
|
||||
}
|
||||
.disabled(isVoting)
|
||||
|
||||
Spacer()
|
||||
}
|
||||
}
|
||||
.padding(.horizontal, 16)
|
||||
.padding(.vertical, 12)
|
||||
.opacity(isVoting ? 0.6 : 1)
|
||||
}
|
||||
|
||||
private func formattedDate(_ iso: String) -> String {
|
||||
// PocketBase returns "2006-01-02 15:04:05.999Z" format
|
||||
let formatter = ISO8601DateFormatter()
|
||||
formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
|
||||
if let date = formatter.date(from: iso) {
|
||||
let rel = RelativeDateTimeFormatter()
|
||||
rel.unitsStyle = .abbreviated
|
||||
return rel.localizedString(for: date, relativeTo: Date())
|
||||
}
|
||||
// Fallback: try space-separated format
|
||||
let df = DateFormatter()
|
||||
df.dateFormat = "yyyy-MM-dd HH:mm:ss.SSSZ"
|
||||
if let date = df.date(from: iso) {
|
||||
let rel = RelativeDateTimeFormatter()
|
||||
rel.unitsStyle = .abbreviated
|
||||
return rel.localizedString(for: date, relativeTo: Date())
|
||||
}
|
||||
return String(iso.prefix(10))
|
||||
}
|
||||
}
|
||||
192
ios/LibNovel/LibNovel/Views/Browse/BrowseView.swift
Normal file
192
ios/LibNovel/LibNovel/Views/Browse/BrowseView.swift
Normal file
@@ -0,0 +1,192 @@
|
||||
import SwiftUI
|
||||
|
||||
struct BrowseView: View {
|
||||
@StateObject private var vm = BrowseViewModel()
|
||||
@State private var showFilters = false
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
VStack(spacing: 0) {
|
||||
// Search bar
|
||||
HStack {
|
||||
Image(systemName: "magnifyingglass").foregroundStyle(.secondary)
|
||||
TextField("Search novels...", text: $vm.searchQuery)
|
||||
.textInputAutocapitalization(.never)
|
||||
.autocorrectionDisabled()
|
||||
.submitLabel(.search)
|
||||
.onSubmit { Task { await vm.search() } }
|
||||
if !vm.searchQuery.isEmpty {
|
||||
Button { vm.clearSearch() } label: {
|
||||
Image(systemName: "xmark.circle.fill").foregroundStyle(.secondary)
|
||||
}
|
||||
}
|
||||
}
|
||||
.padding(10)
|
||||
.background(Color(.systemGray6), in: RoundedRectangle(cornerRadius: 10))
|
||||
.padding(.horizontal)
|
||||
.padding(.vertical, 8)
|
||||
|
||||
// Filter chips row
|
||||
ScrollView(.horizontal, showsIndicators: false) {
|
||||
HStack(spacing: 8) {
|
||||
ChipButton(label: "Sort: \(vm.sort.capitalized)", isSelected: vm.sort != "popular", style: .outlined) {
|
||||
showFilters = true
|
||||
}
|
||||
ChipButton(label: "Genre: \(vm.genre == "all" ? "All" : vm.genre.capitalized)", isSelected: vm.genre != "all", style: .outlined) {
|
||||
showFilters = true
|
||||
}
|
||||
ChipButton(label: "Status: \(vm.status == "all" ? "All" : vm.status.capitalized)", isSelected: vm.status != "all", style: .outlined) {
|
||||
showFilters = true
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
}
|
||||
.padding(.bottom, 4)
|
||||
|
||||
Divider()
|
||||
|
||||
// Results
|
||||
if vm.isLoading && vm.novels.isEmpty {
|
||||
ProgressView().frame(maxWidth: .infinity, maxHeight: .infinity)
|
||||
} else if vm.novels.isEmpty && !vm.isLoading {
|
||||
VStack(spacing: 16) {
|
||||
if let errMsg = vm.error {
|
||||
Image(systemName: "wifi.slash")
|
||||
.font(.largeTitle)
|
||||
.foregroundStyle(.secondary)
|
||||
Text(errMsg)
|
||||
.multilineTextAlignment(.center)
|
||||
.foregroundStyle(.secondary)
|
||||
.padding(.horizontal)
|
||||
Button("Retry") { Task { await vm.loadFirstPage() } }
|
||||
.buttonStyle(.borderedProminent)
|
||||
.tint(.amber)
|
||||
} else {
|
||||
EmptyStateView(icon: "magnifyingglass", title: "No results", message: "Try a different search or filter.")
|
||||
}
|
||||
}
|
||||
.frame(maxWidth: .infinity, maxHeight: .infinity)
|
||||
} else {
|
||||
ScrollView {
|
||||
LazyVGrid(columns: [GridItem(.adaptive(minimum: 150), spacing: 12)], spacing: 16) {
|
||||
ForEach(vm.novels) { novel in
|
||||
NavigationLink(value: NavDestination.book(novel.slug)) {
|
||||
BrowseCard(novel: novel)
|
||||
.bookCoverZoomSource(slug: novel.slug)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
|
||||
// Infinite scroll trigger
|
||||
if vm.hasNext {
|
||||
ProgressView()
|
||||
.frame(maxWidth: .infinity)
|
||||
.padding()
|
||||
.onAppear { Task { await vm.loadNextPage() } }
|
||||
}
|
||||
}
|
||||
.padding()
|
||||
}
|
||||
.refreshable { await vm.loadFirstPage() }
|
||||
}
|
||||
}
|
||||
.navigationTitle("Discover")
|
||||
.appNavigationDestination()
|
||||
.sheet(isPresented: $showFilters) {
|
||||
BrowseFiltersView(vm: vm)
|
||||
}
|
||||
.task { await vm.loadFirstPage() }
|
||||
.onChange(of: vm.sort) { _, _ in Task { await vm.loadFirstPage() } }
|
||||
.onChange(of: vm.genre) { _, _ in Task { await vm.loadFirstPage() } }
|
||||
.onChange(of: vm.status) { _, _ in Task { await vm.loadFirstPage() } }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Browse card
|
||||
|
||||
private struct BrowseCard: View {
|
||||
let novel: BrowseNovel
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
ZStack(alignment: .topLeading) {
|
||||
AsyncCoverImage(url: novel.cover)
|
||||
.frame(height: 200)
|
||||
.clipShape(RoundedRectangle(cornerRadius: 10))
|
||||
if !novel.rank.isEmpty {
|
||||
Text(novel.rank)
|
||||
.font(.caption2.bold())
|
||||
.padding(.horizontal, 6).padding(.vertical, 3)
|
||||
.background(.ultraThinMaterial, in: Capsule())
|
||||
.padding(6)
|
||||
}
|
||||
}
|
||||
Text(novel.title)
|
||||
.font(.caption.bold()).lineLimit(2)
|
||||
if !novel.chapters.isEmpty {
|
||||
Text(novel.chapters).font(.caption2).foregroundStyle(.secondary)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Filters sheet
|
||||
|
||||
struct BrowseFiltersView: View {
|
||||
@ObservedObject var vm: BrowseViewModel
|
||||
@Environment(\.dismiss) private var dismiss
|
||||
|
||||
let sortOptions = ["popular", "new", "updated", "rating", "rank"]
|
||||
let genreOptions = ["all", "action", "fantasy", "romance", "sci-fi", "mystery",
|
||||
"horror", "comedy", "drama", "adventure", "martial arts",
|
||||
"cultivation", "magic", "supernatural", "historical", "slice of life"]
|
||||
let statusOptions = ["all", "ongoing", "completed"]
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
Form {
|
||||
Section("Sort") {
|
||||
ForEach(sortOptions, id: \.self) { opt in
|
||||
HStack {
|
||||
Text(opt.capitalized)
|
||||
Spacer()
|
||||
if vm.sort == opt { Image(systemName: "checkmark").foregroundStyle(.amber) }
|
||||
}
|
||||
.contentShape(Rectangle())
|
||||
.onTapGesture { vm.sort = opt; dismiss() }
|
||||
}
|
||||
}
|
||||
Section("Genre") {
|
||||
ForEach(genreOptions, id: \.self) { opt in
|
||||
HStack {
|
||||
Text(opt == "all" ? "All Genres" : opt.capitalized)
|
||||
Spacer()
|
||||
if vm.genre == opt { Image(systemName: "checkmark").foregroundStyle(.amber) }
|
||||
}
|
||||
.contentShape(Rectangle())
|
||||
.onTapGesture { vm.genre = opt; dismiss() }
|
||||
}
|
||||
}
|
||||
Section("Status") {
|
||||
ForEach(statusOptions, id: \.self) { opt in
|
||||
HStack {
|
||||
Text(opt.capitalized)
|
||||
Spacer()
|
||||
if vm.status == opt { Image(systemName: "checkmark").foregroundStyle(.amber) }
|
||||
}
|
||||
.contentShape(Rectangle())
|
||||
.onTapGesture { vm.status = opt; dismiss() }
|
||||
}
|
||||
}
|
||||
}
|
||||
.navigationTitle("Filters")
|
||||
.navigationBarTitleDisplayMode(.inline)
|
||||
.toolbar {
|
||||
ToolbarItem(placement: .topBarTrailing) {
|
||||
Button("Done") { dismiss() }
|
||||
}
|
||||
}
|
||||
}
|
||||
.presentationDetents([.medium, .large])
|
||||
}
|
||||
}
|
||||
1202
ios/LibNovel/LibNovel/Views/ChapterReader/ChapterReaderView.swift
Normal file
1202
ios/LibNovel/LibNovel/Views/ChapterReader/ChapterReaderView.swift
Normal file
File diff suppressed because it is too large
Load Diff
143
ios/LibNovel/LibNovel/Views/Common/CommonViews.swift
Normal file
143
ios/LibNovel/LibNovel/Views/Common/CommonViews.swift
Normal file
@@ -0,0 +1,143 @@
|
||||
import SwiftUI
|
||||
import Kingfisher
|
||||
|
||||
// MARK: - Empty state placeholder used across all screens
|
||||
|
||||
struct EmptyStateView: View {
|
||||
let icon: String
|
||||
let title: String
|
||||
let message: String
|
||||
|
||||
var body: some View {
|
||||
VStack(spacing: 14) {
|
||||
Image(systemName: icon)
|
||||
.font(.system(size: 48))
|
||||
.foregroundStyle(.tertiary)
|
||||
Text(title)
|
||||
.font(.headline)
|
||||
Text(message)
|
||||
.font(.subheadline)
|
||||
.foregroundStyle(.secondary)
|
||||
.multilineTextAlignment(.center)
|
||||
.padding(.horizontal, 40)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Cover image card reused across screens
|
||||
|
||||
struct BookCard: View {
|
||||
let book: Book
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
AsyncCoverImage(url: book.cover)
|
||||
.frame(height: 200)
|
||||
.clipShape(RoundedRectangle(cornerRadius: 10))
|
||||
Text(book.title)
|
||||
.font(.caption.bold())
|
||||
.lineLimit(2)
|
||||
Text(book.author)
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.secondary)
|
||||
.lineLimit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Async cover image with disk/memory caching via Kingfisher
|
||||
|
||||
struct AsyncCoverImage: View {
|
||||
let url: String
|
||||
/// When true the placeholder is a plain colour fill — used for blurred hero backgrounds
|
||||
/// so the rounded-rect loading indicator doesn't bleed through.
|
||||
var isBackground: Bool = false
|
||||
|
||||
var body: some View {
|
||||
KFImage(URL(string: url))
|
||||
.resizable()
|
||||
.placeholder {
|
||||
if isBackground {
|
||||
Color(.systemGray6)
|
||||
} else {
|
||||
RoundedRectangle(cornerRadius: 10)
|
||||
.fill(Color(.systemGray5))
|
||||
.overlay(Image(systemName: "book.closed").foregroundStyle(.secondary))
|
||||
}
|
||||
}
|
||||
.scaledToFill()
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Tag chip
|
||||
|
||||
struct TagChip: View {
|
||||
let label: String
|
||||
var body: some View {
|
||||
Text(label)
|
||||
.font(.caption2.bold())
|
||||
.padding(.horizontal, 8)
|
||||
.padding(.vertical, 4)
|
||||
.background(Color(.systemGray5), in: Capsule())
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Unified chip button (filter/sort chips across all screens)
|
||||
//
|
||||
// .filled → amber background when selected (genre filter chips in Library)
|
||||
// .outlined → amber border + tint when selected, grey background (sort chips, browse filter chips)
|
||||
|
||||
enum ChipButtonStyle { case filled, outlined }
|
||||
|
||||
struct ChipButton: View {
|
||||
let label: String
|
||||
let isSelected: Bool
|
||||
var style: ChipButtonStyle = .filled
|
||||
let action: () -> Void
|
||||
|
||||
var body: some View {
|
||||
Button(action: action) {
|
||||
Text(label)
|
||||
.font(chipFont)
|
||||
.padding(.horizontal, chipHPad)
|
||||
.padding(.vertical, 6)
|
||||
.background(background)
|
||||
.foregroundStyle(foregroundColor)
|
||||
.overlay(border)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
|
||||
private var chipFont: Font {
|
||||
switch style {
|
||||
case .filled: return .caption.weight(isSelected ? .semibold : .regular)
|
||||
case .outlined: return .subheadline.weight(isSelected ? .semibold : .regular)
|
||||
}
|
||||
}
|
||||
|
||||
private var chipHPad: CGFloat { style == .outlined ? 14 : 12 }
|
||||
|
||||
@ViewBuilder
|
||||
private var background: some View {
|
||||
switch style {
|
||||
case .filled:
|
||||
Capsule().fill(isSelected ? Color.amber : Color(.systemGray5))
|
||||
case .outlined:
|
||||
Capsule()
|
||||
.fill(isSelected ? Color.amber.opacity(0.15) : Color(.systemGray6))
|
||||
.overlay(Capsule().stroke(isSelected ? Color.amber : .clear, lineWidth: 1.5))
|
||||
}
|
||||
}
|
||||
|
||||
private var foregroundColor: Color {
|
||||
switch style {
|
||||
case .filled: return isSelected ? .white : .primary
|
||||
case .outlined: return isSelected ? .amber : .primary
|
||||
}
|
||||
}
|
||||
|
||||
@ViewBuilder
|
||||
private var border: some View {
|
||||
// outlined style already has its border baked into `background`
|
||||
EmptyView()
|
||||
}
|
||||
}
|
||||
303
ios/LibNovel/LibNovel/Views/Home/HomeView.swift
Normal file
303
ios/LibNovel/LibNovel/Views/Home/HomeView.swift
Normal file
@@ -0,0 +1,303 @@
|
||||
import SwiftUI
|
||||
|
||||
struct HomeView: View {
|
||||
@StateObject private var vm = HomeViewModel()
|
||||
@EnvironmentObject var authStore: AuthStore
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
ScrollView {
|
||||
VStack(alignment: .leading, spacing: 0) {
|
||||
|
||||
// Large hero continue card (most recent in-progress book)
|
||||
if let hero = vm.continueReading.first {
|
||||
HeroContinueCard(item: hero)
|
||||
.padding(.horizontal)
|
||||
.padding(.top, 8)
|
||||
.padding(.bottom, 28)
|
||||
}
|
||||
|
||||
// Continue reading shelf (remaining items after the hero)
|
||||
let shelf = vm.continueReading.dropFirst()
|
||||
if !shelf.isEmpty {
|
||||
ShelfHeader(title: "Continue Reading")
|
||||
ScrollView(.horizontal, showsIndicators: false) {
|
||||
HStack(alignment: .top, spacing: 14) {
|
||||
ForEach(Array(shelf)) { item in
|
||||
NavigationLink(value: NavDestination.book(item.book.slug)) {
|
||||
ContinueReadingCard(item: item)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
.padding(.bottom, 4)
|
||||
}
|
||||
.padding(.bottom, 28)
|
||||
}
|
||||
|
||||
// Stats strip
|
||||
if let stats = vm.stats {
|
||||
StatsStrip(stats: stats)
|
||||
.padding(.horizontal)
|
||||
.padding(.bottom, 28)
|
||||
}
|
||||
|
||||
// Recently updated shelf
|
||||
if !vm.recentlyUpdated.isEmpty {
|
||||
ShelfHeader(title: "Recently Updated")
|
||||
ScrollView(.horizontal, showsIndicators: false) {
|
||||
HStack(alignment: .top, spacing: 14) {
|
||||
ForEach(vm.recentlyUpdated) { book in
|
||||
NavigationLink(value: NavDestination.book(book.slug)) {
|
||||
ShelfBookCard(book: book)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
.padding(.bottom, 4)
|
||||
}
|
||||
.padding(.bottom, 28)
|
||||
}
|
||||
|
||||
// Empty state
|
||||
if vm.continueReading.isEmpty && vm.recentlyUpdated.isEmpty && !vm.isLoading {
|
||||
EmptyStateView(
|
||||
icon: "books.vertical",
|
||||
title: "Your library is empty",
|
||||
message: "Head to Discover to find novels to read."
|
||||
)
|
||||
.frame(maxWidth: .infinity)
|
||||
.padding(.top, 60)
|
||||
}
|
||||
|
||||
if vm.isLoading {
|
||||
ProgressView()
|
||||
.frame(maxWidth: .infinity)
|
||||
.padding(.top, 60)
|
||||
}
|
||||
|
||||
Color.clear.frame(height: 20)
|
||||
}
|
||||
}
|
||||
.navigationTitle("Reading Now")
|
||||
.appNavigationDestination()
|
||||
.refreshable { await vm.load() }
|
||||
.task { await vm.load() }
|
||||
.errorAlert($vm.error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Hero card (full-width, Apple Books "Now Playing" style)
|
||||
|
||||
private struct HeroContinueCard: View {
|
||||
let item: ContinueReadingItem
|
||||
|
||||
var body: some View {
|
||||
NavigationLink(value: NavDestination.chapter(item.book.slug, item.chapter)) {
|
||||
ZStack(alignment: .bottomLeading) {
|
||||
// Blurred background
|
||||
AsyncCoverImage(url: item.book.cover, isBackground: true)
|
||||
.frame(maxWidth: .infinity)
|
||||
.frame(height: 220)
|
||||
.blur(radius: 22)
|
||||
.clipped()
|
||||
// Depth gradient: subtle amber tint at top, deep shadow at bottom
|
||||
.overlay(
|
||||
LinearGradient(
|
||||
stops: [
|
||||
.init(color: Color(red: 0.18, green: 0.12, blue: 0.02).opacity(0.55), location: 0),
|
||||
.init(color: .black.opacity(0.15), location: 0.35),
|
||||
.init(color: .black.opacity(0.78), location: 1)
|
||||
],
|
||||
startPoint: .top,
|
||||
endPoint: .bottom
|
||||
)
|
||||
)
|
||||
|
||||
// Content: cover on left, info stacked on right
|
||||
HStack(alignment: .bottom, spacing: 14) {
|
||||
AsyncCoverImage(url: item.book.cover)
|
||||
.frame(width: 96, height: 138)
|
||||
.clipShape(RoundedRectangle(cornerRadius: 10))
|
||||
.shadow(color: .black.opacity(0.55), radius: 12, y: 6)
|
||||
.bookCoverZoomSource(slug: item.book.slug)
|
||||
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
// Progress indicator
|
||||
if item.book.totalChapters > 0 {
|
||||
let pct = min(1.0, Double(item.chapter) / Double(item.book.totalChapters))
|
||||
GeometryReader { geo in
|
||||
ZStack(alignment: .leading) {
|
||||
Capsule().fill(Color.white.opacity(0.2))
|
||||
Capsule().fill(Color.amber.opacity(0.85))
|
||||
.frame(width: geo.size.width * pct)
|
||||
}
|
||||
}
|
||||
.frame(height: 3)
|
||||
.frame(maxWidth: 140)
|
||||
|
||||
Text("\(Int(pct * 100))% complete")
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.white.opacity(0.55))
|
||||
}
|
||||
|
||||
Text(item.book.title)
|
||||
.font(.headline)
|
||||
.foregroundStyle(.white)
|
||||
.lineLimit(2)
|
||||
|
||||
Text(item.book.author)
|
||||
.font(.subheadline)
|
||||
.foregroundStyle(.white.opacity(0.65))
|
||||
.lineLimit(1)
|
||||
|
||||
Spacer(minLength: 8)
|
||||
|
||||
HStack(spacing: 6) {
|
||||
Image(systemName: "play.fill")
|
||||
.font(.caption.bold())
|
||||
Text("Continue Ch.\(item.chapter)")
|
||||
.font(.subheadline.weight(.semibold))
|
||||
}
|
||||
.foregroundStyle(.black.opacity(0.85))
|
||||
.padding(.horizontal, 14)
|
||||
.padding(.vertical, 9)
|
||||
.background(Capsule().fill(Color.amber))
|
||||
}
|
||||
|
||||
Spacer(minLength: 0)
|
||||
}
|
||||
.padding(.horizontal, 16)
|
||||
.padding(.bottom, 18)
|
||||
}
|
||||
.clipShape(RoundedRectangle(cornerRadius: 16))
|
||||
.shadow(color: .black.opacity(0.25), radius: 14, y: 5)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Shelf header
|
||||
|
||||
private struct ShelfHeader: View {
|
||||
let title: String
|
||||
|
||||
var body: some View {
|
||||
Text(title)
|
||||
.font(.title3.bold())
|
||||
.padding(.horizontal)
|
||||
.padding(.bottom, 10)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Horizontal shelf: continue reading card
|
||||
|
||||
private struct ContinueReadingCard: View {
|
||||
let item: ContinueReadingItem
|
||||
|
||||
private var progressFraction: Double {
|
||||
guard item.book.totalChapters > 0 else { return 0 }
|
||||
return min(1.0, Double(item.chapter) / Double(item.book.totalChapters))
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
ZStack(alignment: .bottomTrailing) {
|
||||
AsyncCoverImage(url: item.book.cover)
|
||||
.frame(width: 110, height: 158)
|
||||
.clipShape(RoundedRectangle(cornerRadius: 8))
|
||||
|
||||
// Progress arc ring + chapter badge
|
||||
ZStack {
|
||||
Circle()
|
||||
.stroke(Color.white.opacity(0.18), lineWidth: 2.5)
|
||||
Circle()
|
||||
.trim(from: 0, to: progressFraction)
|
||||
.stroke(Color.amber, style: StrokeStyle(lineWidth: 2.5, lineCap: .round))
|
||||
.rotationEffect(.degrees(-90))
|
||||
Text("Ch.\(item.chapter)")
|
||||
.font(.system(size: 8, weight: .bold))
|
||||
.foregroundStyle(.white)
|
||||
.minimumScaleFactor(0.6)
|
||||
}
|
||||
.frame(width: 36, height: 36)
|
||||
.background(.ultraThinMaterial, in: Circle())
|
||||
.padding(5)
|
||||
}
|
||||
Text(item.book.title)
|
||||
.font(.caption.bold())
|
||||
.lineLimit(2)
|
||||
.frame(width: 110, alignment: .leading)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Horizontal shelf: recently updated book card
|
||||
|
||||
private struct ShelfBookCard: View {
|
||||
let book: Book
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
AsyncCoverImage(url: book.cover)
|
||||
.frame(width: 110, height: 158)
|
||||
.clipShape(RoundedRectangle(cornerRadius: 8))
|
||||
.shadow(color: .black.opacity(0.12), radius: 4, y: 2)
|
||||
.bookCoverZoomSource(slug: book.slug)
|
||||
|
||||
Text(book.title)
|
||||
.font(.caption.bold())
|
||||
.lineLimit(2)
|
||||
.frame(width: 110, alignment: .leading)
|
||||
|
||||
Text(book.author)
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.secondary)
|
||||
.lineLimit(1)
|
||||
.frame(width: 110, alignment: .leading)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Stats strip (compact inline)
|
||||
|
||||
private struct StatsStrip: View {
|
||||
let stats: HomeStats
|
||||
|
||||
var body: some View {
|
||||
HStack(spacing: 0) {
|
||||
StatPill(icon: "books.vertical.fill", value: "\(stats.totalBooks)", label: "Books")
|
||||
Divider().frame(height: 28)
|
||||
StatPill(icon: "text.alignleft", value: "\(stats.totalChapters)", label: "Chapters")
|
||||
Divider().frame(height: 28)
|
||||
StatPill(icon: "bookmark.fill", value: "\(stats.booksInProgress)", label: "In Progress")
|
||||
}
|
||||
.frame(maxWidth: .infinity)
|
||||
.padding(.vertical, 14)
|
||||
.background(.ultraThinMaterial, in: RoundedRectangle(cornerRadius: 14))
|
||||
}
|
||||
}
|
||||
|
||||
private struct StatPill: View {
|
||||
let icon: String
|
||||
let value: String
|
||||
let label: String
|
||||
|
||||
var body: some View {
|
||||
VStack(spacing: 4) {
|
||||
Image(systemName: icon)
|
||||
.font(.system(size: 14, weight: .medium))
|
||||
.foregroundStyle(Color.amber.opacity(0.8))
|
||||
Text(value)
|
||||
.font(.subheadline.bold().monospacedDigit())
|
||||
.foregroundStyle(.primary)
|
||||
Text(label)
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
.frame(maxWidth: .infinity)
|
||||
}
|
||||
}
|
||||
320
ios/LibNovel/LibNovel/Views/Library/LibraryView.swift
Normal file
320
ios/LibNovel/LibNovel/Views/Library/LibraryView.swift
Normal file
@@ -0,0 +1,320 @@
|
||||
import SwiftUI
|
||||
import Kingfisher
|
||||
|
||||
struct LibraryView: View {
|
||||
@StateObject private var vm = LibraryViewModel()
|
||||
@State private var sortOrder: SortOrder = .recentlyRead
|
||||
@State private var readingFilter: ReadingFilter = .all
|
||||
@State private var selectedGenre: String = "all"
|
||||
@State private var searchText = ""
|
||||
|
||||
enum SortOrder: String, CaseIterable {
|
||||
case recentlyRead = "Recent"
|
||||
case title = "Title"
|
||||
case author = "Author"
|
||||
case progress = "Progress"
|
||||
}
|
||||
|
||||
enum ReadingFilter: String, CaseIterable {
|
||||
case all = "All"
|
||||
case inProgress = "In Progress"
|
||||
case completed = "Completed"
|
||||
}
|
||||
|
||||
// All distinct genres across the library, sorted alphabetically.
|
||||
private var availableGenres: [String] {
|
||||
let all = vm.items.flatMap { $0.book.genres }
|
||||
let unique = Array(Set(all)).sorted()
|
||||
return unique
|
||||
}
|
||||
|
||||
private var filtered: [LibraryItem] {
|
||||
var result = vm.items
|
||||
|
||||
// 1. Reading filter
|
||||
switch readingFilter {
|
||||
case .all:
|
||||
break
|
||||
case .inProgress:
|
||||
result = result.filter { !isCompleted($0) }
|
||||
case .completed:
|
||||
result = result.filter { isCompleted($0) }
|
||||
}
|
||||
|
||||
// 2. Genre filter
|
||||
if selectedGenre != "all" {
|
||||
result = result.filter { $0.book.genres.contains(selectedGenre) }
|
||||
}
|
||||
|
||||
// 3. Sort
|
||||
switch sortOrder {
|
||||
case .recentlyRead:
|
||||
break // server returns by recency
|
||||
case .title:
|
||||
result = result.sorted { $0.book.title < $1.book.title }
|
||||
case .author:
|
||||
result = result.sorted { $0.book.author < $1.book.author }
|
||||
case .progress:
|
||||
result = result.sorted { ($0.lastChapter ?? 0) > ($1.lastChapter ?? 0) }
|
||||
}
|
||||
|
||||
// 4. Search
|
||||
if !searchText.isEmpty {
|
||||
result = result.filter {
|
||||
$0.book.title.localizedCaseInsensitiveContains(searchText) ||
|
||||
$0.book.author.localizedCaseInsensitiveContains(searchText)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private func isCompleted(_ item: LibraryItem) -> Bool {
|
||||
// Treat as completed if book status is "completed" OR
|
||||
// the user has read up to (or past) the total chapter count.
|
||||
if item.book.status.lowercased() == "completed",
|
||||
let ch = item.lastChapter,
|
||||
item.book.totalChapters > 0,
|
||||
ch >= item.book.totalChapters {
|
||||
return true
|
||||
}
|
||||
return item.book.status.lowercased() == "completed" && (item.lastChapter ?? 0) > 0
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
Group {
|
||||
if vm.isLoading && vm.items.isEmpty {
|
||||
ProgressView()
|
||||
.frame(maxWidth: .infinity, maxHeight: .infinity)
|
||||
} else if vm.items.isEmpty {
|
||||
EmptyStateView(
|
||||
icon: "bookmark",
|
||||
title: "No saved books",
|
||||
message: "Books you save or start reading will appear here."
|
||||
)
|
||||
} else {
|
||||
ScrollView {
|
||||
VStack(spacing: 0) {
|
||||
// Search bar
|
||||
HStack(spacing: 8) {
|
||||
Image(systemName: "magnifyingglass")
|
||||
.foregroundStyle(.secondary)
|
||||
TextField("Search library", text: $searchText)
|
||||
.font(.subheadline)
|
||||
if !searchText.isEmpty {
|
||||
Button { searchText = "" } label: {
|
||||
Image(systemName: "xmark.circle.fill")
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
}
|
||||
}
|
||||
.padding(.horizontal, 12)
|
||||
.padding(.vertical, 10)
|
||||
.background(Color(.systemGray6), in: RoundedRectangle(cornerRadius: 10))
|
||||
.padding(.horizontal)
|
||||
.padding(.top, 8)
|
||||
|
||||
// Reading filter (All / In Progress / Completed)
|
||||
Picker("", selection: $readingFilter) {
|
||||
ForEach(ReadingFilter.allCases, id: \.self) { f in
|
||||
Text(f.rawValue).tag(f)
|
||||
}
|
||||
}
|
||||
.pickerStyle(.segmented)
|
||||
.padding(.horizontal)
|
||||
.padding(.top, 12)
|
||||
|
||||
// Genre filter chips (only shown when genres are available)
|
||||
if !availableGenres.isEmpty {
|
||||
ScrollView(.horizontal, showsIndicators: false) {
|
||||
HStack(spacing: 8) {
|
||||
// "All" chip
|
||||
ChipButton(
|
||||
label: "All",
|
||||
isSelected: selectedGenre == "all",
|
||||
style: .filled
|
||||
) {
|
||||
withAnimation { selectedGenre = "all" }
|
||||
}
|
||||
ForEach(availableGenres, id: \.self) { genre in
|
||||
ChipButton(
|
||||
label: genre.capitalized,
|
||||
isSelected: selectedGenre == genre,
|
||||
style: .filled
|
||||
) {
|
||||
withAnimation {
|
||||
selectedGenre = selectedGenre == genre ? "all" : genre
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
}
|
||||
.padding(.top, 10)
|
||||
}
|
||||
|
||||
// Sort chips
|
||||
ScrollView(.horizontal, showsIndicators: false) {
|
||||
HStack(spacing: 8) {
|
||||
ForEach(SortOrder.allCases, id: \.self) { order in
|
||||
ChipButton(
|
||||
label: order.rawValue,
|
||||
isSelected: sortOrder == order,
|
||||
style: .outlined
|
||||
) {
|
||||
withAnimation { sortOrder = order }
|
||||
}
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
}
|
||||
.padding(.vertical, 10)
|
||||
|
||||
// Book count
|
||||
Text("\(filtered.count) book\(filtered.count == 1 ? "" : "s")")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.padding(.horizontal)
|
||||
.padding(.bottom, 4)
|
||||
|
||||
if filtered.isEmpty {
|
||||
VStack(spacing: 12) {
|
||||
Image(systemName: readingFilter == .completed ? "checkmark.circle" : "book")
|
||||
.font(.system(size: 40))
|
||||
.foregroundStyle(.secondary)
|
||||
Text(emptyMessage)
|
||||
.font(.subheadline)
|
||||
.foregroundStyle(.secondary)
|
||||
.multilineTextAlignment(.center)
|
||||
}
|
||||
.frame(maxWidth: .infinity)
|
||||
.padding(.top, 60)
|
||||
} else {
|
||||
// 3-column grid
|
||||
LazyVGrid(
|
||||
columns: [
|
||||
GridItem(.flexible(), spacing: 12),
|
||||
GridItem(.flexible(), spacing: 12),
|
||||
GridItem(.flexible(), spacing: 12)
|
||||
],
|
||||
spacing: 20
|
||||
) {
|
||||
ForEach(filtered) { item in
|
||||
NavigationLink(value: NavDestination.book(item.book.slug)) {
|
||||
LibraryBookCard(item: item)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
.padding(.bottom, 24)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.navigationTitle("Library")
|
||||
.appNavigationDestination()
|
||||
.refreshable { await vm.load() }
|
||||
.task { await vm.load() }
|
||||
.errorAlert($vm.error)
|
||||
}
|
||||
}
|
||||
|
||||
private var emptyMessage: String {
|
||||
switch readingFilter {
|
||||
case .all:
|
||||
return selectedGenre == "all" ? "No books match your search." : "No \(selectedGenre.capitalized) books in your library."
|
||||
case .inProgress:
|
||||
return "No books in progress."
|
||||
case .completed:
|
||||
return "No completed books yet."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Library book card (3-column)
|
||||
|
||||
private struct LibraryBookCard: View {
|
||||
let item: LibraryItem
|
||||
|
||||
private var progressFraction: Double {
|
||||
guard let ch = item.lastChapter, item.book.totalChapters > 0 else { return 0 }
|
||||
return Double(ch) / Double(item.book.totalChapters)
|
||||
}
|
||||
|
||||
private var isCompleted: Bool {
|
||||
progressFraction >= 1.0
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
ZStack(alignment: .topTrailing) {
|
||||
// Cover image
|
||||
KFImage(URL(string: item.book.cover))
|
||||
.resizable()
|
||||
.placeholder {
|
||||
RoundedRectangle(cornerRadius: 8)
|
||||
.fill(Color(.systemGray5))
|
||||
.overlay(
|
||||
Image(systemName: "book.closed")
|
||||
.foregroundStyle(.secondary)
|
||||
)
|
||||
}
|
||||
.scaledToFill()
|
||||
.frame(maxWidth: .infinity)
|
||||
.aspectRatio(2/3, contentMode: .fit)
|
||||
.clipShape(RoundedRectangle(cornerRadius: 8))
|
||||
.shadow(color: .black.opacity(0.14), radius: 4, y: 2)
|
||||
.bookCoverZoomSource(slug: item.book.slug)
|
||||
|
||||
// Progress arc or completed checkmark in top-right corner
|
||||
if isCompleted {
|
||||
Image(systemName: "checkmark.circle.fill")
|
||||
.font(.system(size: 18, weight: .semibold))
|
||||
.foregroundStyle(.white)
|
||||
.background(Circle().fill(Color.amber).padding(1))
|
||||
.padding(5)
|
||||
} else if progressFraction > 0 {
|
||||
ProgressArc(fraction: progressFraction)
|
||||
.frame(width: 28, height: 28)
|
||||
.padding(4)
|
||||
}
|
||||
}
|
||||
|
||||
// Title
|
||||
Text(item.book.title)
|
||||
.font(.caption.bold())
|
||||
.lineLimit(2)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
|
||||
// Chapter badge if present
|
||||
if let ch = item.lastChapter {
|
||||
Text(isCompleted ? "Finished" : "Ch.\(ch)")
|
||||
.font(.caption2)
|
||||
.foregroundStyle(isCompleted ? Color.amber : .secondary)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Circular progress arc overlay
|
||||
|
||||
private struct ProgressArc: View {
|
||||
let fraction: Double // 0...1
|
||||
|
||||
var body: some View {
|
||||
ZStack {
|
||||
Circle()
|
||||
.fill(.ultraThinMaterial)
|
||||
|
||||
Circle()
|
||||
.trim(from: 0, to: fraction)
|
||||
.stroke(Color.amber, style: StrokeStyle(lineWidth: 2.5, lineCap: .round))
|
||||
.rotationEffect(.degrees(-90))
|
||||
.animation(.easeInOut(duration: 0.5), value: fraction)
|
||||
}
|
||||
}
|
||||
}
|
||||
1094
ios/LibNovel/LibNovel/Views/Player/PlayerViews.swift
Normal file
1094
ios/LibNovel/LibNovel/Views/Player/PlayerViews.swift
Normal file
File diff suppressed because it is too large
Load Diff
161
ios/LibNovel/LibNovel/Views/Profile/AvatarCropView.swift
Normal file
161
ios/LibNovel/LibNovel/Views/Profile/AvatarCropView.swift
Normal file
@@ -0,0 +1,161 @@
|
||||
import SwiftUI
|
||||
|
||||
// MARK: - AvatarCropView
|
||||
// A sheet that lets the user pan and pinch a photo to fill a 1:1 square crop region.
|
||||
// Call: .sheet(item: $cropImage) { AvatarCropView(image: $0.image, onConfirm: { croppedData in … }) }
|
||||
|
||||
struct AvatarCropView: View {
|
||||
let image: UIImage
|
||||
let onConfirm: (Data) -> Void
|
||||
let onCancel: () -> Void
|
||||
|
||||
// Crop square side length (points) — matched to the web 400 px target
|
||||
private let cropSize: CGFloat = 280
|
||||
|
||||
// Pan/zoom state
|
||||
@State private var scale: CGFloat = 1.0
|
||||
@State private var lastScale: CGFloat = 1.0
|
||||
@State private var offset: CGSize = .zero
|
||||
@State private var lastOffset: CGSize = .zero
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
GeometryReader { geo in
|
||||
ZStack {
|
||||
Color.black.ignoresSafeArea()
|
||||
|
||||
// Draggable / pinchable image
|
||||
Image(uiImage: image)
|
||||
.resizable()
|
||||
.scaledToFill()
|
||||
.frame(width: geo.size.width, height: geo.size.height)
|
||||
.scaleEffect(scale)
|
||||
.offset(offset)
|
||||
.gesture(
|
||||
SimultaneousGesture(
|
||||
MagnificationGesture()
|
||||
.onChanged { value in
|
||||
scale = max(1.0, lastScale * value)
|
||||
}
|
||||
.onEnded { _ in
|
||||
lastScale = scale
|
||||
},
|
||||
DragGesture()
|
||||
.onChanged { value in
|
||||
offset = CGSize(
|
||||
width: lastOffset.width + value.translation.width,
|
||||
height: lastOffset.height + value.translation.height
|
||||
)
|
||||
}
|
||||
.onEnded { _ in
|
||||
lastOffset = offset
|
||||
}
|
||||
)
|
||||
)
|
||||
.clipped()
|
||||
|
||||
// Dim overlay with transparent crop square cut out
|
||||
CropOverlay(cropSize: cropSize, containerSize: geo.size)
|
||||
.allowsHitTesting(false)
|
||||
}
|
||||
}
|
||||
.navigationTitle("Crop Photo")
|
||||
.navigationBarTitleDisplayMode(.inline)
|
||||
.toolbar {
|
||||
ToolbarItem(placement: .topBarLeading) {
|
||||
Button("Cancel", action: onCancel)
|
||||
.foregroundStyle(.white)
|
||||
}
|
||||
ToolbarItem(placement: .topBarTrailing) {
|
||||
Button("Use Photo") {
|
||||
confirmCrop()
|
||||
}
|
||||
.fontWeight(.semibold)
|
||||
.foregroundStyle(.amber)
|
||||
}
|
||||
}
|
||||
.toolbarColorScheme(.dark, for: .navigationBar)
|
||||
}
|
||||
.onAppear { fitImageInitially() }
|
||||
}
|
||||
|
||||
// MARK: - Crop
|
||||
|
||||
private func fitImageInitially() {
|
||||
// Scale image so its shorter dimension fills the crop square
|
||||
let imgAspect = image.size.width / image.size.height
|
||||
if imgAspect > 1 {
|
||||
// wider than tall — fit height to cropSize
|
||||
scale = cropSize / image.size.height * (image.size.height / image.size.width)
|
||||
} else {
|
||||
scale = 1.0
|
||||
}
|
||||
scale = max(1.0, scale)
|
||||
lastScale = scale
|
||||
}
|
||||
|
||||
private func confirmCrop() {
|
||||
// Render image at current pan/zoom into a 400×400 bitmap
|
||||
let outputSize = CGSize(width: 400, height: 400)
|
||||
let renderer = UIGraphicsImageRenderer(size: outputSize)
|
||||
let cropped = renderer.image { ctx in
|
||||
// We need to map from the SwiftUI transform back to image pixels.
|
||||
// We render the raw UIImage into the output rect, applying the same
|
||||
// scale / offset proportionally (normalised by crop square / container).
|
||||
let screenCropSize: CGFloat = cropSize
|
||||
// Scale factor: pixels per SwiftUI point in the output
|
||||
let outputScale = outputSize.width / screenCropSize
|
||||
|
||||
ctx.cgContext.translateBy(x: outputSize.width / 2, y: outputSize.height / 2)
|
||||
ctx.cgContext.scaleBy(x: scale * outputScale, y: scale * outputScale)
|
||||
ctx.cgContext.translateBy(
|
||||
x: -image.size.width / 2 + (offset.width * outputScale / scale),
|
||||
y: -image.size.height / 2 + (offset.height * outputScale / scale)
|
||||
)
|
||||
image.draw(at: .zero)
|
||||
}
|
||||
|
||||
if let jpeg = cropped.jpegData(compressionQuality: 0.9) {
|
||||
onConfirm(jpeg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Crop overlay
|
||||
|
||||
private struct CropOverlay: View {
|
||||
let cropSize: CGFloat
|
||||
let containerSize: CGSize
|
||||
|
||||
var body: some View {
|
||||
Canvas { context, size in
|
||||
// Fill entire canvas with semi-transparent black
|
||||
context.fill(Path(CGRect(origin: .zero, size: size)), with: .color(.black.opacity(0.55)))
|
||||
// Cut out the crop square in the centre
|
||||
let origin = CGPoint(
|
||||
x: (size.width - cropSize) / 2,
|
||||
y: (size.height - cropSize) / 2
|
||||
)
|
||||
let cropRect = CGRect(origin: origin, size: CGSize(width: cropSize, height: cropSize))
|
||||
context.blendMode = .destinationOut
|
||||
context.fill(Path(ellipseIn: cropRect), with: .color(.white))
|
||||
}
|
||||
.compositingGroup()
|
||||
.overlay {
|
||||
// Amber circle border around the crop region
|
||||
let origin = CGPoint(
|
||||
x: (containerSize.width - cropSize) / 2,
|
||||
y: (containerSize.height - cropSize) / 2
|
||||
)
|
||||
Circle()
|
||||
.stroke(Color.amber.opacity(0.8), lineWidth: 2)
|
||||
.frame(width: cropSize, height: cropSize)
|
||||
.position(
|
||||
x: origin.x + cropSize / 2,
|
||||
y: origin.y + cropSize / 2
|
||||
)
|
||||
}
|
||||
.frame(width: containerSize.width, height: containerSize.height)
|
||||
.allowsHitTesting(false)
|
||||
}
|
||||
}
|
||||
333
ios/LibNovel/LibNovel/Views/Profile/ProfileView.swift
Normal file
333
ios/LibNovel/LibNovel/Views/Profile/ProfileView.swift
Normal file
@@ -0,0 +1,333 @@
|
||||
import SwiftUI
|
||||
import PhotosUI
|
||||
import Kingfisher
|
||||
|
||||
struct ProfileView: View {
|
||||
@EnvironmentObject var authStore: AuthStore
|
||||
@StateObject private var vm = ProfileViewModel()
|
||||
@State private var showChangePassword = false
|
||||
|
||||
// Avatar upload state
|
||||
@State private var photoPickerItem: PhotosPickerItem?
|
||||
@State private var pendingCropImage: UIImage? // image waiting to be cropped
|
||||
@State private var avatarURL: String? = nil
|
||||
@State private var avatarUploading = false
|
||||
@State private var avatarError: String?
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
List {
|
||||
// ── User header ────────────────────────────────────────────
|
||||
Section {
|
||||
HStack(spacing: 16) {
|
||||
// Tappable avatar circle
|
||||
PhotosPicker(selection: $photoPickerItem,
|
||||
matching: .images,
|
||||
photoLibrary: .shared()) {
|
||||
ZStack {
|
||||
Circle()
|
||||
.fill(Color(.systemGray5))
|
||||
.frame(width: 72, height: 72)
|
||||
|
||||
if avatarUploading {
|
||||
ProgressView()
|
||||
.frame(width: 72, height: 72)
|
||||
} else if let urlStr = avatarURL ?? authStore.user?.avatarURL,
|
||||
let url = URL(string: urlStr) {
|
||||
KFImage(url)
|
||||
.placeholder {
|
||||
Image(systemName: "person.circle.fill")
|
||||
.font(.system(size: 52))
|
||||
.foregroundStyle(.amber)
|
||||
}
|
||||
.resizable()
|
||||
.scaledToFill()
|
||||
.frame(width: 72, height: 72)
|
||||
.clipShape(Circle())
|
||||
} else {
|
||||
Image(systemName: "person.circle.fill")
|
||||
.font(.system(size: 52))
|
||||
.foregroundStyle(.amber)
|
||||
.frame(width: 72, height: 72)
|
||||
}
|
||||
|
||||
// Camera overlay badge
|
||||
if !avatarUploading {
|
||||
VStack {
|
||||
Spacer()
|
||||
HStack {
|
||||
Spacer()
|
||||
ZStack {
|
||||
Circle()
|
||||
.fill(Color.amber)
|
||||
.frame(width: 22, height: 22)
|
||||
Image(systemName: "camera.fill")
|
||||
.font(.system(size: 10, weight: .semibold))
|
||||
.foregroundStyle(.black)
|
||||
}
|
||||
.offset(x: 2, y: 2)
|
||||
}
|
||||
}
|
||||
.frame(width: 72, height: 72)
|
||||
}
|
||||
}
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.onChange(of: photoPickerItem) { _, item in
|
||||
guard let item else { return }
|
||||
Task { await loadImageForCrop(item) }
|
||||
}
|
||||
|
||||
VStack(alignment: .leading, spacing: 3) {
|
||||
Text(authStore.user?.username ?? "")
|
||||
.font(.headline)
|
||||
Text(authStore.user?.role.capitalized ?? "")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
if let err = avatarError {
|
||||
Text(err)
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.red)
|
||||
}
|
||||
}
|
||||
}
|
||||
.padding(.vertical, 6)
|
||||
}
|
||||
|
||||
// ── Reading settings ───────────────────────────────────────
|
||||
Section("Reading Settings") {
|
||||
voicePicker
|
||||
speedSlider
|
||||
Toggle("Auto-advance chapter", isOn: Binding(
|
||||
get: { authStore.settings.autoNext },
|
||||
set: { newVal in
|
||||
Task {
|
||||
var s = authStore.settings
|
||||
s.autoNext = newVal
|
||||
await authStore.saveSettings(s)
|
||||
}
|
||||
}
|
||||
))
|
||||
.tint(.amber)
|
||||
}
|
||||
|
||||
// ── Sessions ───────────────────────────────────────────────
|
||||
Section("Active Sessions") {
|
||||
if vm.sessionsLoading {
|
||||
ProgressView()
|
||||
} else {
|
||||
ForEach(vm.sessions) { session in
|
||||
SessionRow(session: session) {
|
||||
Task { await vm.revokeSession(id: session.id) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Account ────────────────────────────────────────────────
|
||||
Section("Account") {
|
||||
Button("Change Password") { showChangePassword = true }
|
||||
Button("Sign Out", role: .destructive) {
|
||||
Task { await authStore.logout() }
|
||||
}
|
||||
}
|
||||
}
|
||||
.navigationTitle("Profile")
|
||||
.task {
|
||||
await vm.loadSessions()
|
||||
}
|
||||
.sheet(isPresented: $showChangePassword) {
|
||||
ChangePasswordView()
|
||||
}
|
||||
.sheet(item: Binding(
|
||||
get: { pendingCropImage.map { CropImageItem(image: $0) } },
|
||||
set: { if $0 == nil { pendingCropImage = nil } }
|
||||
)) { item in
|
||||
AvatarCropView(image: item.image) { croppedData in
|
||||
pendingCropImage = nil
|
||||
Task { await uploadCroppedData(croppedData) }
|
||||
} onCancel: {
|
||||
pendingCropImage = nil
|
||||
}
|
||||
}
|
||||
.errorAlert($vm.error)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Avatar upload
|
||||
|
||||
/// Step 1: Load the raw image from the picker and show the crop sheet.
|
||||
private func loadImageForCrop(_ item: PhotosPickerItem) async {
|
||||
guard let data = try? await item.loadTransferable(type: Data.self),
|
||||
let image = UIImage(data: data) else {
|
||||
avatarError = "Could not read image"
|
||||
return
|
||||
}
|
||||
pendingCropImage = image
|
||||
}
|
||||
|
||||
/// Step 2: Called by AvatarCropView once the user confirms. Upload the cropped JPEG.
|
||||
private func uploadCroppedData(_ data: Data) async {
|
||||
avatarUploading = true
|
||||
avatarError = nil
|
||||
defer { avatarUploading = false }
|
||||
do {
|
||||
let url = try await APIClient.shared.uploadAvatar(data, mimeType: "image/jpeg")
|
||||
avatarURL = url
|
||||
// Refresh user record so the new avatar persists across sessions
|
||||
await authStore.validateToken()
|
||||
} catch {
|
||||
avatarError = "Upload failed: \(error.localizedDescription)"
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Voice picker
|
||||
|
||||
@ViewBuilder
|
||||
private var voicePicker: some View {
|
||||
Picker("TTS Voice", selection: Binding(
|
||||
get: { authStore.settings.voice },
|
||||
set: { newVoice in
|
||||
Task {
|
||||
var s = authStore.settings
|
||||
s.voice = newVoice
|
||||
await authStore.saveSettings(s)
|
||||
}
|
||||
}
|
||||
)) {
|
||||
if vm.voices.isEmpty {
|
||||
Text("Default").tag("af_bella")
|
||||
} else {
|
||||
ForEach(vm.voices, id: \.self) { v in
|
||||
Text(v).tag(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
.task { await vm.loadVoices() }
|
||||
}
|
||||
|
||||
// MARK: - Speed slider
|
||||
|
||||
@ViewBuilder
|
||||
private var speedSlider: some View {
|
||||
VStack(alignment: .leading, spacing: 4) {
|
||||
HStack {
|
||||
Text("Playback Speed")
|
||||
Spacer()
|
||||
Text("\(authStore.settings.speed, specifier: "%.1f")×")
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
Slider(
|
||||
value: Binding(
|
||||
get: { authStore.settings.speed },
|
||||
set: { newSpeed in
|
||||
Task {
|
||||
var s = authStore.settings
|
||||
s.speed = newSpeed
|
||||
await authStore.saveSettings(s)
|
||||
}
|
||||
}
|
||||
),
|
||||
in: 0.5...2.0, step: 0.25
|
||||
)
|
||||
.tint(.amber)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Session row
|
||||
|
||||
private struct SessionRow: View {
|
||||
let session: UserSession
|
||||
let onRevoke: () -> Void
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 4) {
|
||||
HStack {
|
||||
Image(systemName: "iphone")
|
||||
Text(session.userAgent.isEmpty ? "Unknown device" : session.userAgent)
|
||||
.font(.subheadline)
|
||||
.lineLimit(1)
|
||||
Spacer()
|
||||
if session.isCurrent {
|
||||
Text("This device")
|
||||
.font(.caption2.bold())
|
||||
.foregroundStyle(.amber)
|
||||
} else {
|
||||
Button("Revoke", role: .destructive, action: onRevoke)
|
||||
.font(.caption)
|
||||
}
|
||||
}
|
||||
Text("Last seen: \(session.lastSeen.prefix(10))")
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Change password sheet
|
||||
|
||||
struct ChangePasswordView: View {
|
||||
@Environment(\.dismiss) private var dismiss
|
||||
@EnvironmentObject var authStore: AuthStore
|
||||
@State private var current = ""
|
||||
@State private var newPwd = ""
|
||||
@State private var confirm = ""
|
||||
@State private var isLoading = false
|
||||
@State private var error: String?
|
||||
@State private var success = false
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
Form {
|
||||
Section {
|
||||
SecureField("Current password", text: $current)
|
||||
SecureField("New password", text: $newPwd)
|
||||
SecureField("Confirm new password", text: $confirm)
|
||||
}
|
||||
if let error {
|
||||
Text(error).foregroundStyle(.red).font(.caption)
|
||||
}
|
||||
if success {
|
||||
Text("Password changed successfully").foregroundStyle(.green).font(.caption)
|
||||
}
|
||||
}
|
||||
.navigationTitle("Change Password")
|
||||
.navigationBarTitleDisplayMode(.inline)
|
||||
.toolbar {
|
||||
ToolbarItem(placement: .topBarLeading) { Button("Cancel") { dismiss() } }
|
||||
ToolbarItem(placement: .topBarTrailing) {
|
||||
Button("Save") { save() }
|
||||
.disabled(isLoading || newPwd.count < 4 || newPwd != confirm)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func save() {
|
||||
guard newPwd == confirm else { error = "Passwords do not match"; return }
|
||||
isLoading = true
|
||||
error = nil
|
||||
Task {
|
||||
do {
|
||||
struct Body: Encodable { let currentPassword, newPassword: String }
|
||||
let _: EmptyResponse = try await APIClient.shared.fetch(
|
||||
"/api/auth/change-password", method: "POST",
|
||||
body: Body(currentPassword: current, newPassword: newPwd)
|
||||
)
|
||||
success = true
|
||||
try? await Task.sleep(nanoseconds: 1_200_000_000)
|
||||
dismiss()
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
isLoading = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Crop image item (Identifiable wrapper for .sheet(item:))
|
||||
|
||||
private struct CropImageItem: Identifiable {
|
||||
let id = UUID()
|
||||
let image: UIImage
|
||||
}
|
||||
9
ios/LibNovel/LibNovelTests/LibNovelTests.swift
Normal file
9
ios/LibNovel/LibNovelTests/LibNovelTests.swift
Normal file
@@ -0,0 +1,9 @@
|
||||
import XCTest
|
||||
@testable import LibNovel
|
||||
|
||||
final class LibNovelTests: XCTestCase {
|
||||
func testExample() throws {
|
||||
// Placeholder — add real tests here
|
||||
XCTAssert(true)
|
||||
}
|
||||
}
|
||||
36
ios/LibNovel/fastlane/Fastfile
Normal file
36
ios/LibNovel/fastlane/Fastfile
Normal file
@@ -0,0 +1,36 @@
|
||||
default_platform(:ios)
|
||||
|
||||
platform :ios do
|
||||
desc "Build and upload to TestFlight"
|
||||
lane :beta do
|
||||
# Generate Xcode project from project.yml (one level up from fastlane/)
|
||||
sh("cd .. && xcodegen generate --spec project.yml --project .")
|
||||
|
||||
# Set build number from CI run number (passed as env var)
|
||||
increment_build_number(
|
||||
build_number: ENV["BUILD_NUMBER"] || "1",
|
||||
xcodeproj: "LibNovel.xcodeproj"
|
||||
)
|
||||
|
||||
# Build the app - signing settings are in project.yml Release config
|
||||
build_app(
|
||||
scheme: "LibNovel",
|
||||
export_method: "app-store",
|
||||
clean: true,
|
||||
configuration: "Release",
|
||||
export_options: {
|
||||
method: "app-store",
|
||||
teamID: "GHZXC6FVMU",
|
||||
provisioningProfiles: {
|
||||
"com.kalekber.LibNovel" => "LibNovel Distribution"
|
||||
},
|
||||
signingStyle: "manual"
|
||||
}
|
||||
)
|
||||
|
||||
# Upload to TestFlight
|
||||
upload_to_testflight(
|
||||
skip_waiting_for_build_processing: true
|
||||
)
|
||||
end
|
||||
end
|
||||
91
ios/LibNovel/project.yml
Normal file
91
ios/LibNovel/project.yml
Normal file
@@ -0,0 +1,91 @@
|
||||
name: LibNovel
|
||||
options:
|
||||
bundleIdPrefix: com.kalekber
|
||||
deploymentTarget:
|
||||
iOS: "17.0"
|
||||
xcodeVersion: "16.0"
|
||||
generateEmptyDirectories: true
|
||||
indentWidth: 4
|
||||
tabWidth: 4
|
||||
usesTabs: false
|
||||
|
||||
settings:
|
||||
base:
|
||||
SWIFT_VERSION: "5.10"
|
||||
ENABLE_PREVIEWS: YES
|
||||
MARKETING_VERSION: "1.0.0"
|
||||
CURRENT_PROJECT_VERSION: "1"
|
||||
LIBNOVEL_BASE_URL: "https://v2.libnovel.kalekber.cc"
|
||||
configs:
|
||||
Debug:
|
||||
SWIFT_ACTIVE_COMPILATION_CONDITIONS: DEBUG
|
||||
Release:
|
||||
SWIFT_ACTIVE_COMPILATION_CONDITIONS: ""
|
||||
|
||||
packages:
|
||||
# Async image loading with caching
|
||||
Kingfisher:
|
||||
url: https://github.com/onevcat/Kingfisher
|
||||
from: "8.0.0"
|
||||
|
||||
targets:
|
||||
LibNovel:
|
||||
type: application
|
||||
platform: iOS
|
||||
deploymentTarget: "17.0"
|
||||
sources:
|
||||
- path: LibNovel
|
||||
excludes:
|
||||
- "**/.DS_Store"
|
||||
- "Resources/Info.plist"
|
||||
resources:
|
||||
- path: LibNovel/Resources/Assets.xcassets
|
||||
dependencies:
|
||||
- package: Kingfisher
|
||||
settings:
|
||||
base:
|
||||
PRODUCT_BUNDLE_IDENTIFIER: com.kalekber.LibNovel
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon
|
||||
TARGETED_DEVICE_FAMILY: "1,2" # iPhone + iPad
|
||||
GENERATE_INFOPLIST_FILE: NO
|
||||
INFOPLIST_FILE: LibNovel/Resources/Info.plist
|
||||
configs:
|
||||
Release:
|
||||
CODE_SIGN_STYLE: Manual
|
||||
DEVELOPMENT_TEAM: GHZXC6FVMU
|
||||
CODE_SIGN_IDENTITY: "Apple Distribution"
|
||||
PROVISIONING_PROFILE: "af592c3a-f60b-4ac1-a14f-30b8a206017f"
|
||||
|
||||
LibNovelTests:
|
||||
type: bundle.unit-test
|
||||
platform: iOS
|
||||
deploymentTarget: "17.0"
|
||||
sources:
|
||||
- path: LibNovelTests
|
||||
dependencies:
|
||||
- target: LibNovel
|
||||
settings:
|
||||
base:
|
||||
PRODUCT_BUNDLE_IDENTIFIER: com.kalekber.LibNovel.tests
|
||||
|
||||
schemes:
|
||||
LibNovel:
|
||||
build:
|
||||
targets:
|
||||
LibNovel: all
|
||||
run:
|
||||
config: Debug
|
||||
environmentVariables:
|
||||
LIBNOVEL_BASE_URL:
|
||||
value: "https://v2.libnovel.kalekber.cc"
|
||||
isEnabled: true
|
||||
test:
|
||||
config: Debug
|
||||
targets:
|
||||
- LibNovelTests
|
||||
profile:
|
||||
config: Release
|
||||
analyze:
|
||||
config: Debug
|
||||
archive:
|
||||
config: Release
|
||||
32
ios/LibNovel/test-build.sh
Executable file
32
ios/LibNovel/test-build.sh
Executable file
@@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Test script for local iOS build iteration
|
||||
# Run from ios/LibNovel directory
|
||||
|
||||
echo "=== Generating Xcode project ==="
|
||||
xcodegen generate --spec project.yml --project .
|
||||
|
||||
echo ""
|
||||
echo "=== Listing available provisioning profiles ==="
|
||||
ls -la ~/Library/MobileDevice/Provisioning\ Profiles/ || echo "No profiles found"
|
||||
|
||||
echo ""
|
||||
echo "=== Listing available signing identities ==="
|
||||
security find-identity -v -p codesigning
|
||||
|
||||
echo ""
|
||||
echo "=== Attempting archive build ==="
|
||||
xcodebuild archive \
|
||||
-project LibNovel.xcodeproj \
|
||||
-scheme LibNovel \
|
||||
-configuration Release \
|
||||
-destination 'generic/platform=iOS' \
|
||||
-archivePath ./build/LibNovel.xcarchive \
|
||||
-allowProvisioningUpdates \
|
||||
CODE_SIGN_STYLE=Manual \
|
||||
CODE_SIGN_IDENTITY="Apple Distribution" \
|
||||
DEVELOPMENT_TEAM="GHZXC6FVMU"
|
||||
|
||||
echo ""
|
||||
echo "=== Build succeeded! ==="
|
||||
234
justfile
Normal file
234
justfile
Normal file
@@ -0,0 +1,234 @@
|
||||
# justfile — libnovel-v2 task runner
|
||||
# Install just: https://just.systems
|
||||
|
||||
scraper_dir := "scraper"
|
||||
ui_dir := "ui"
|
||||
ios_dir := "ios/LibNovel"
|
||||
ios_scheme := "LibNovel"
|
||||
ios_sim := "platform=iOS Simulator,name=iPhone 17"
|
||||
ios_spm := ".spm-cache"
|
||||
runner_temp := env_var_or_default("RUNNER_TEMP", "/tmp")
|
||||
|
||||
# ─── Build ────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Build the scraper binary
|
||||
build:
|
||||
cd {{scraper_dir}} && go build -o bin/scraper ./cmd/scraper
|
||||
|
||||
# Build and verify all Go packages compile cleanly
|
||||
build-all:
|
||||
cd {{scraper_dir}} && go build ./...
|
||||
|
||||
# ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Run unit tests only (no integration services required)
|
||||
test:
|
||||
cd {{scraper_dir}} && go test -race -count=1 -timeout=60s ./...
|
||||
|
||||
# Run integration tests (requires MinIO, PocketBase, optional Browserless)
|
||||
# Override env vars as needed, e.g.:
|
||||
# just test-integration MINIO_ENDPOINT=localhost:9000
|
||||
test-integration:
|
||||
cd {{scraper_dir}} && go test -v -tags integration -timeout 600s ./...
|
||||
|
||||
# Run unit + integration tests
|
||||
test-all: test test-integration
|
||||
|
||||
# Run a specific package's integration tests, e.g.:
|
||||
# just test-pkg internal/storage
|
||||
test-pkg pkg:
|
||||
cd {{scraper_dir}} && go test -v -tags integration -timeout 600s ./{{pkg}}/...
|
||||
|
||||
# Run end-to-end tests against live services.
|
||||
# All services must be running first (docker compose up -d or just e2e-up).
|
||||
# Override env vars as needed, e.g.:
|
||||
# just test-e2e SCRAPER_URL=http://localhost:8080 KOKORO_VOICE=af_bella
|
||||
test-e2e \
|
||||
browserless_url="http://localhost:3030" \
|
||||
minio_endpoint="localhost:9000" \
|
||||
pocketbase_url="http://localhost:8090" \
|
||||
scraper_url="http://localhost:8080":
|
||||
cd {{scraper_dir}} && \
|
||||
BROWSERLESS_URL={{browserless_url}} \
|
||||
MINIO_ENDPOINT={{minio_endpoint}} \
|
||||
POCKETBASE_URL={{pocketbase_url}} \
|
||||
SCRAPER_URL={{scraper_url}} \
|
||||
go test -v -tags integration -timeout 900s ./internal/e2e/...
|
||||
|
||||
# Start all services required for e2e tests, then run them
|
||||
e2e: up test-e2e
|
||||
|
||||
# ─── Code quality ─────────────────────────────────────────────────────────────
|
||||
|
||||
# Run go vet on all packages (including integration build tag)
|
||||
lint:
|
||||
cd {{scraper_dir}} && go vet ./...
|
||||
cd {{scraper_dir}} && go vet -tags integration ./...
|
||||
|
||||
# ─── UI ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Type-check the SvelteKit UI
|
||||
ui-check:
|
||||
cd {{ui_dir}} && npx svelte-check
|
||||
|
||||
# Start the SvelteKit dev server
|
||||
ui-dev:
|
||||
cd {{ui_dir}} && npm run dev
|
||||
|
||||
# Install UI dependencies
|
||||
ui-install:
|
||||
cd {{ui_dir}} && npm install
|
||||
|
||||
# Build the UI for production
|
||||
ui-build:
|
||||
cd {{ui_dir}} && npm run build
|
||||
|
||||
# ─── iOS ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Regenerate LibNovel.xcodeproj from project.yml (run after structural changes)
|
||||
ios-gen:
|
||||
cd {{ios_dir}} && xcodegen generate --spec project.yml --project .
|
||||
|
||||
# Resolve SPM package dependencies (cached to {{ios_spm}})
|
||||
ios-resolve:
|
||||
cd {{ios_dir}} && xcodebuild \
|
||||
-project {{ios_scheme}}.xcodeproj \
|
||||
-scheme {{ios_scheme}} \
|
||||
-resolvePackageDependencies \
|
||||
-clonedSourcePackagesDirPath {{ios_spm}}
|
||||
|
||||
# Build the iOS app for the simulator (no signing required)
|
||||
# Runs ios-gen first to ensure the project is up to date.
|
||||
ios-build: ios-gen ios-resolve
|
||||
cd {{ios_dir}} && set -o pipefail && xcodebuild \
|
||||
-project {{ios_scheme}}.xcodeproj \
|
||||
-scheme {{ios_scheme}} \
|
||||
-configuration Debug \
|
||||
-destination 'generic/platform=iOS Simulator' \
|
||||
-clonedSourcePackagesDirPath {{ios_spm}} \
|
||||
CODE_SIGNING_ALLOWED=NO \
|
||||
| xcpretty || xcodebuild \
|
||||
-project {{ios_scheme}}.xcodeproj \
|
||||
-scheme {{ios_scheme}} \
|
||||
-configuration Debug \
|
||||
-destination 'generic/platform=iOS Simulator' \
|
||||
-clonedSourcePackagesDirPath {{ios_spm}} \
|
||||
CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
# Run unit tests on the simulator
|
||||
# Runs ios-gen first to ensure the project is up to date.
|
||||
ios-test: ios-gen ios-resolve
|
||||
cd {{ios_dir}} && set -o pipefail && xcodebuild test \
|
||||
-project {{ios_scheme}}.xcodeproj \
|
||||
-scheme {{ios_scheme}} \
|
||||
-configuration Debug \
|
||||
-destination '{{ios_sim}}' \
|
||||
-clonedSourcePackagesDirPath {{ios_spm}} \
|
||||
CODE_SIGNING_ALLOWED=NO \
|
||||
| xcpretty --report junit --output test-results.xml || true
|
||||
|
||||
# Archive a signed Release build (requires valid signing identity in keychain).
|
||||
# Output: {{runner_temp}}/LibNovel.xcarchive
|
||||
# Typically called from CI after importing certificate + provisioning profile.
|
||||
# Usage: just ios-archive <team-id> <profile-uuid>
|
||||
ios-archive team_id profile_uuid: ios-gen ios-resolve
|
||||
cd {{ios_dir}} && xcodebuild archive \
|
||||
-project {{ios_scheme}}.xcodeproj \
|
||||
-scheme {{ios_scheme}} \
|
||||
-configuration Release \
|
||||
-destination 'generic/platform=iOS' \
|
||||
-clonedSourcePackagesDirPath {{ios_spm}} \
|
||||
-archivePath {{runner_temp}}/LibNovel.xcarchive \
|
||||
CODE_SIGN_IDENTITY="Apple Distribution" \
|
||||
"PROVISIONING_PROFILE[sdk=iphoneos*]={{profile_uuid}}" \
|
||||
DEVELOPMENT_TEAM="{{team_id}}"
|
||||
|
||||
# Export an IPA from the archive produced by ios-archive.
|
||||
# Requires ios/LibNovel/ExportOptions.plist.
|
||||
# Output: {{runner_temp}}/ipa/LibNovel.ipa
|
||||
ios-export:
|
||||
cd {{ios_dir}} && xcodebuild -exportArchive \
|
||||
-archivePath {{runner_temp}}/LibNovel.xcarchive \
|
||||
-exportPath {{runner_temp}}/ipa \
|
||||
-exportOptionsPlist ExportOptions.plist
|
||||
|
||||
# Set the build number (CFBundleVersion) in project.yml before archiving.
|
||||
# Usage: just ios-set-build-number 42
|
||||
ios-set-build-number number:
|
||||
cd {{ios_dir}} && sed -i '' \
|
||||
's/CURRENT_PROJECT_VERSION: .*/CURRENT_PROJECT_VERSION: {{number}}/' \
|
||||
project.yml
|
||||
|
||||
# Upload the exported IPA to TestFlight via App Store Connect API.
|
||||
# Requires env vars: ASC_KEY_ID, ASC_ISSUER_ID, ASC_PRIVATE_KEY_PATH
|
||||
# The private key (.p8 file) must be present at ASC_PRIVATE_KEY_PATH.
|
||||
ios-upload:
|
||||
xcrun altool --upload-app \
|
||||
--type ios \
|
||||
--file {{runner_temp}}/ipa/LibNovel.ipa \
|
||||
--apiKey "$ASC_KEY_ID" \
|
||||
--apiIssuer "$ASC_ISSUER_ID"
|
||||
|
||||
# ─── Docker Compose ───────────────────────────────────────────────────────────
|
||||
|
||||
# Start all services (browserless, kokoro, scraper, minio, pocketbase)
|
||||
up:
|
||||
docker compose up -d
|
||||
|
||||
# Stop all services
|
||||
down:
|
||||
docker compose down
|
||||
|
||||
# Tail logs for all services
|
||||
logs:
|
||||
docker compose logs -f
|
||||
|
||||
# Tail logs for a specific service, e.g.: just logs-service scraper
|
||||
logs-service service:
|
||||
docker compose logs -f {{service}}
|
||||
|
||||
# Rebuild and restart a specific service
|
||||
restart service:
|
||||
docker compose up -d --build {{service}}
|
||||
|
||||
# ─── Local dev: individual services ──────────────────────────────────────────
|
||||
|
||||
# Start only PocketBase (for local storage testing)
|
||||
pb-up:
|
||||
docker compose up -d pocketbase
|
||||
|
||||
# Start only MinIO (for local storage testing)
|
||||
minio-up:
|
||||
docker compose up -d minio
|
||||
|
||||
# Start only Browserless (for local scraping tests)
|
||||
browserless-up:
|
||||
docker compose up -d browserless
|
||||
|
||||
# Start storage backends only (MinIO + PocketBase)
|
||||
storage-up:
|
||||
docker compose up -d minio pocketbase
|
||||
|
||||
# ─── Convenience ─────────────────────────────────────────────────────────────
|
||||
|
||||
# Show status of all docker compose services
|
||||
status:
|
||||
docker compose ps
|
||||
|
||||
# Remove all stopped containers and unused images
|
||||
prune:
|
||||
docker compose down --remove-orphans
|
||||
docker image prune -f
|
||||
|
||||
# One-shot scrape of the full catalogue (requires services to be running)
|
||||
scrape-run: build
|
||||
cd {{scraper_dir}} && ./bin/scraper run
|
||||
|
||||
# One-shot scrape of a single book URL, e.g.:
|
||||
# just scrape-book https://novelfire.net/book/my-novel
|
||||
scrape-book url: build
|
||||
cd {{scraper_dir}} && ./bin/scraper run --url {{url}}
|
||||
|
||||
# Start the HTTP server
|
||||
serve: build
|
||||
cd {{scraper_dir}} && ./bin/scraper serve
|
||||
4
scraper/.dockerignore
Normal file
4
scraper/.dockerignore
Normal file
@@ -0,0 +1,4 @@
|
||||
bin/
|
||||
static/
|
||||
*.md
|
||||
.git
|
||||
@@ -13,9 +13,10 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
||||
go build -ldflags="-s -w" -o /scraper ./cmd/scraper
|
||||
|
||||
# ── Runtime stage ──────────────────────────────────────────────────────────────
|
||||
FROM alpine:3.20
|
||||
FROM alpine:3.21
|
||||
|
||||
# ca-certificates is required for HTTPS requests to novelfire.net.
|
||||
# ca-certificates: HTTPS to novelfire.net
|
||||
# tzdata: timezone data
|
||||
RUN apk add --no-cache ca-certificates tzdata
|
||||
|
||||
WORKDIR /app
|
||||
@@ -31,8 +32,6 @@ RUN chown -R scraper:scraper /app
|
||||
USER scraper
|
||||
|
||||
# ── Configuration ─────────────────────────────────────────────────────────────
|
||||
ENV BROWSERLESS_URL=http://browserless:3030
|
||||
ENV BROWSERLESS_STRATEGY=content
|
||||
ENV SCRAPER_WORKERS=0
|
||||
ENV SCRAPER_STATIC_ROOT=/app/static/books
|
||||
ENV SCRAPER_HTTP_ADDR=:8080
|
||||
|
||||
@@ -10,16 +10,20 @@
|
||||
//
|
||||
// Environment variables:
|
||||
//
|
||||
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
||||
// BROWSERLESS_TOKEN Browserless API token (default: "")
|
||||
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
|
||||
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
|
||||
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
|
||||
// SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
|
||||
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
|
||||
// KOKORO_VOICE Default TTS voice (default: af_bella)
|
||||
// LOG_LEVEL debug | info | warn | error (default: info)
|
||||
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
|
||||
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
|
||||
// KOKORO_VOICE Default TTS voice (default: af_bella)
|
||||
// POCKETBASE_URL PocketBase API base URL (default: http://localhost:8090)
|
||||
// POCKETBASE_ADMIN_EMAIL PocketBase admin email (default: admin@libnovel.local)
|
||||
// POCKETBASE_ADMIN_PASSWORD PocketBase admin password (default: changeme123)
|
||||
// MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000)
|
||||
// MINIO_ACCESS_KEY MinIO access key (default: admin)
|
||||
// MINIO_SECRET_KEY MinIO secret key (default: changeme123)
|
||||
// MINIO_USE_SSL Use TLS for MinIO (default: false)
|
||||
// MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters)
|
||||
// MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio)
|
||||
// LOG_LEVEL debug | info | warn | error (default: info)
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -27,6 +31,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"strconv"
|
||||
@@ -37,8 +42,9 @@ import (
|
||||
"github.com/libnovel/scraper/internal/browser"
|
||||
"github.com/libnovel/scraper/internal/novelfire"
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/scraper/htmlutil"
|
||||
"github.com/libnovel/scraper/internal/server"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -67,30 +73,45 @@ func run(log *slog.Logger) error {
|
||||
|
||||
cmd := strings.ToLower(args[0])
|
||||
|
||||
browserCfg := browser.Config{
|
||||
BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3030"),
|
||||
Token: envOr("BROWSERLESS_TOKEN", ""),
|
||||
}
|
||||
browserCfg.MaxConcurrent = 5
|
||||
if s := os.Getenv("BROWSERLESS_MAX_CONCURRENT"); s != "" {
|
||||
// All scraping uses direct HTTP — novelfire.net pages are server-rendered
|
||||
// and do not require a headless browser. A direct HTTP client is faster,
|
||||
// more reliable, and has no Browserless dependency.
|
||||
directCfg := browser.Config{MaxConcurrent: 5}
|
||||
if s := os.Getenv("SCRAPER_TIMEOUT"); s != "" {
|
||||
if n, err := strconv.Atoi(s); err == nil && n > 0 {
|
||||
browserCfg.MaxConcurrent = n
|
||||
directCfg.Timeout = time.Duration(n) * time.Second
|
||||
}
|
||||
}
|
||||
if s := os.Getenv("BROWSERLESS_TIMEOUT"); s != "" {
|
||||
if n, err := strconv.Atoi(s); err == nil && n > 0 {
|
||||
browserCfg.Timeout = time.Duration(n) * time.Second
|
||||
}
|
||||
directClient := browser.NewDirectHTTPClient(directCfg)
|
||||
|
||||
// ── Storage backends ────────────────────────────────────────────────────
|
||||
minioCfg := storage.MinioConfig{
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
PublicEndpoint: envOr("MINIO_PUBLIC_ENDPOINT", ""),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: strings.ToLower(os.Getenv("MINIO_USE_SSL")) == "true",
|
||||
PublicUseSSL: strings.ToLower(os.Getenv("MINIO_PUBLIC_USE_SSL")) != "false",
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
|
||||
BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "libnovel-avatars"),
|
||||
}
|
||||
pbCfg := storage.PocketBaseConfig{
|
||||
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||
}
|
||||
|
||||
strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect))))
|
||||
urlStrategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_URL_STRATEGY", string(browser.StrategyContent))))
|
||||
bc := newBrowserClient(strategy, browserCfg)
|
||||
urlClient := newBrowserClient(urlStrategy, browserCfg)
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books")
|
||||
w := writer.New(staticRoot)
|
||||
nf := novelfire.New(bc, log, urlClient, w)
|
||||
store, err := storage.NewHybridStore(ctx, pbCfg, minioCfg, log)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage init failed: %w", err)
|
||||
}
|
||||
|
||||
nf := novelfire.New(directClient, log, directClient, directClient, store)
|
||||
|
||||
workers := 0
|
||||
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
|
||||
@@ -104,13 +125,9 @@ func run(log *slog.Logger) error {
|
||||
}
|
||||
|
||||
oCfg := orchestrator.Config{
|
||||
Workers: workers,
|
||||
StaticRoot: staticRoot,
|
||||
Workers: workers,
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
switch cmd {
|
||||
case "run":
|
||||
// Optional --url flag.
|
||||
@@ -118,13 +135,13 @@ func run(log *slog.Logger) error {
|
||||
oCfg.SingleBookURL = args[2]
|
||||
}
|
||||
log.Info("starting one-shot scrape",
|
||||
"strategy", strategy,
|
||||
"strategy", "direct",
|
||||
"workers", workers,
|
||||
"max_concurrent", browserCfg.MaxConcurrent,
|
||||
"static_root", oCfg.StaticRoot,
|
||||
"single_book", oCfg.SingleBookURL,
|
||||
"pocketbase_url", pbCfg.BaseURL,
|
||||
"pocketbase_email", pbCfg.AdminEmail,
|
||||
)
|
||||
o := orchestrator.New(oCfg, nf, log)
|
||||
o := orchestrator.New(oCfg, nf, log, store)
|
||||
return o.Run(ctx)
|
||||
|
||||
case "refresh":
|
||||
@@ -133,13 +150,12 @@ func run(log *slog.Logger) error {
|
||||
return fmt.Errorf("refresh command requires a book slug argument")
|
||||
}
|
||||
slug := args[1]
|
||||
w := writer.New(oCfg.StaticRoot)
|
||||
meta, ok, err := w.ReadMetadata(slug)
|
||||
meta, ok, err := store.ReadMetadata(ctx, slug)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read metadata for %s: %w", slug, err)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("book %q not found in %s", slug, oCfg.StaticRoot)
|
||||
return fmt.Errorf("book %q not found in store", slug)
|
||||
}
|
||||
if meta.SourceURL == "" {
|
||||
return fmt.Errorf("book %q has no source_url in metadata", slug)
|
||||
@@ -148,41 +164,301 @@ func run(log *slog.Logger) error {
|
||||
log.Info("refreshing book from source_url",
|
||||
"slug", slug,
|
||||
"source_url", meta.SourceURL,
|
||||
"pocketbase_url", pbCfg.BaseURL,
|
||||
"pocketbase_email", pbCfg.AdminEmail,
|
||||
)
|
||||
o := orchestrator.New(oCfg, nf, log)
|
||||
o := orchestrator.New(oCfg, nf, log, store)
|
||||
return o.Run(ctx)
|
||||
|
||||
case "serve":
|
||||
addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
|
||||
kokoroURL := envOr("KOKORO_URL", "")
|
||||
kokoroURL := envOr("KOKORO_URL", "https://kokoro.kalekber.cc")
|
||||
kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
|
||||
log.Info("starting HTTP server",
|
||||
"addr", addr,
|
||||
"strategy", strategy,
|
||||
"strategy", "direct",
|
||||
"workers", workers,
|
||||
"max_concurrent", browserCfg.MaxConcurrent,
|
||||
"kokoro_url", kokoroURL,
|
||||
"kokoro_voice", kokoroVoice,
|
||||
"pocketbase_url", pbCfg.BaseURL,
|
||||
"pocketbase_email", pbCfg.AdminEmail,
|
||||
)
|
||||
srv := server.New(addr, oCfg, nf, log, kokoroURL, kokoroVoice)
|
||||
srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
|
||||
return srv.ListenAndServe(ctx)
|
||||
|
||||
case "save-browse":
|
||||
return runSaveBrowse(ctx, args[1:], store, log)
|
||||
|
||||
default:
|
||||
return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd)
|
||||
return fmt.Errorf("unknown command %q; use 'run', 'refresh', 'serve', or 'save-browse'", cmd)
|
||||
}
|
||||
}
|
||||
|
||||
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
|
||||
switch strategy {
|
||||
case browser.StrategyScrape:
|
||||
return browser.NewScrapeClient(cfg)
|
||||
case browser.StrategyCDP:
|
||||
return browser.NewCDPClient(cfg)
|
||||
case browser.StrategyDirect:
|
||||
return browser.NewDirectHTTPClient(cfg)
|
||||
default:
|
||||
return browser.NewContentClient(cfg)
|
||||
// runSaveBrowse implements the `save-browse` subcommand.
|
||||
// It iterates over browse pages on novelfire.net, captures each using
|
||||
// SingleFile CLI (connected to the existing Browserless instance), and
|
||||
// stores the resulting self-contained HTML in the MinIO browse bucket.
|
||||
// After storing each page it parses the HTML, upserts ranking records in
|
||||
// PocketBase, and fires background goroutines to download cover images.
|
||||
//
|
||||
// Flags (all optional):
|
||||
//
|
||||
// --genre <value> genre slug (default: all)
|
||||
// --sort <value> sort order (default: popular)
|
||||
// --status <value> status (default: all)
|
||||
// --type <value> novel type (default: all-novel)
|
||||
// --max-pages <n> max pages (default: 5)
|
||||
func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log *slog.Logger) error {
|
||||
// Parse flags manually to avoid importing flag package.
|
||||
genre := "all"
|
||||
sortBy := "popular"
|
||||
status := "all"
|
||||
novelType := "all-novel"
|
||||
maxPages := 5
|
||||
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "--genre":
|
||||
if i+1 < len(args) {
|
||||
genre = args[i+1]
|
||||
i++
|
||||
}
|
||||
case "--sort":
|
||||
if i+1 < len(args) {
|
||||
sortBy = args[i+1]
|
||||
i++
|
||||
}
|
||||
case "--status":
|
||||
if i+1 < len(args) {
|
||||
status = args[i+1]
|
||||
i++
|
||||
}
|
||||
case "--type":
|
||||
if i+1 < len(args) {
|
||||
novelType = args[i+1]
|
||||
i++
|
||||
}
|
||||
case "--max-pages":
|
||||
if i+1 < len(args) {
|
||||
if n, err := strconv.Atoi(args[i+1]); err == nil && n > 0 {
|
||||
maxPages = n
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
singleFilePath := envOr("SINGLEFILE_PATH", "single-file")
|
||||
browserlessURL := envOr("BROWSERLESS_URL", "http://localhost:3030")
|
||||
// SingleFile expects a WebSocket CDP endpoint.
|
||||
// Browserless exposes /chromium at the WS root.
|
||||
wsEndpoint := strings.Replace(browserlessURL, "http://", "ws://", 1)
|
||||
wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)
|
||||
|
||||
log.Info("save-browse: starting",
|
||||
"genre", genre, "sort", sortBy, "status", status,
|
||||
"type", novelType, "max_pages", maxPages,
|
||||
"singlefile", singleFilePath,
|
||||
"browserless_ws", wsEndpoint,
|
||||
)
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "libnovel-browse-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("save-browse: create temp dir: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
const novelFireBase = "https://novelfire.net"
|
||||
const novelFireDomain = "novelfire.net"
|
||||
|
||||
for page := 1; page <= maxPages; page++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
|
||||
novelFireBase, genre, sortBy, status, novelType, page)
|
||||
|
||||
// Use the new domain-based key layout: {domain}/html/page-{n}.html
|
||||
key := store.BrowseHTMLKey(novelFireDomain, page)
|
||||
|
||||
outFile := fmt.Sprintf("%s/page-%d.html", tmpDir, page)
|
||||
|
||||
log.Info("save-browse: capturing page", "page", page, "url", pageURL)
|
||||
|
||||
//nolint:gosec // singleFilePath and pageURL are config/URL values, not user input.
|
||||
cmd := exec.CommandContext(ctx, singleFilePath,
|
||||
pageURL,
|
||||
"--browser-server="+wsEndpoint,
|
||||
"--output="+outFile,
|
||||
)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if runErr := cmd.Run(); runErr != nil {
|
||||
log.Warn("save-browse: SingleFile failed, skipping page",
|
||||
"page", page, "err", runErr)
|
||||
continue
|
||||
}
|
||||
|
||||
htmlBytes, readErr := os.ReadFile(outFile)
|
||||
if readErr != nil {
|
||||
log.Warn("save-browse: failed to read output file",
|
||||
"page", page, "file", outFile, "err", readErr)
|
||||
continue
|
||||
}
|
||||
|
||||
if putErr := store.SaveBrowsePage(ctx, key, string(htmlBytes)); putErr != nil {
|
||||
log.Warn("save-browse: failed to store snapshot in MinIO",
|
||||
"page", page, "key", key, "err", putErr)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Info("save-browse: snapshot stored", "page", page, "key", key,
|
||||
"bytes", len(htmlBytes))
|
||||
|
||||
// Parse the stored HTML and populate the ranking collection.
|
||||
novels := parseSaveBrowseListings(htmlBytes, novelFireBase)
|
||||
for i, novel := range novels {
|
||||
rank := i + 1
|
||||
coverKey := store.BrowseCoverKey(novelFireDomain, novel.slug)
|
||||
|
||||
item := storage.RankingItem{
|
||||
Rank: rank,
|
||||
Slug: novel.slug,
|
||||
Title: novel.title,
|
||||
Cover: coverKey,
|
||||
SourceURL: novel.url,
|
||||
}
|
||||
if werr := store.WriteRankingItem(ctx, item); werr != nil {
|
||||
log.Warn("save-browse: WriteRankingItem failed",
|
||||
"slug", novel.slug, "err", werr)
|
||||
}
|
||||
|
||||
// Download cover image in the background (best-effort).
|
||||
if novel.coverURL != "" {
|
||||
go storage.DownloadAndStoreCover(store, log, coverKey, novel.coverURL)
|
||||
}
|
||||
}
|
||||
if len(novels) > 0 {
|
||||
log.Info("save-browse: ranking populated", "page", page, "count", len(novels))
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("save-browse: done")
|
||||
return nil
|
||||
}
|
||||
|
||||
// novelListingCLI is a minimal novel listing used within the CLI command.
|
||||
type novelListingCLI struct {
|
||||
slug string
|
||||
title string
|
||||
url string
|
||||
coverURL string
|
||||
}
|
||||
|
||||
// parseSaveBrowseListings extracts novel listings from raw HTML bytes.
|
||||
// It reuses the same parsing logic as the server's parseBrowsePage but
|
||||
// operates on []byte to avoid importing the server package.
|
||||
func parseSaveBrowseListings(htmlBytes []byte, novelFireBase string) []novelListingCLI {
|
||||
type listing = novelListingCLI
|
||||
|
||||
// Minimal tokeniser-based walk to find <li class="novel-item"> blocks.
|
||||
// We use the golang.org/x/net/html parser via a local import.
|
||||
// Because main.go already imports golang.org/x/net/html indirectly through
|
||||
// the server package build, we do a simple line-scan here instead to keep
|
||||
// the dependency surface small.
|
||||
//
|
||||
// Strategy: scan for href="/book/{slug}", img data-src/src, h4.novel-title text.
|
||||
var novels []listing
|
||||
|
||||
lines := strings.Split(string(htmlBytes), "\n")
|
||||
var cur listing
|
||||
inNovelItem := false
|
||||
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
|
||||
// Detect start of a novel-item list element.
|
||||
if strings.Contains(trimmed, `class="novel-item"`) || strings.Contains(trimmed, "novel-item") && strings.HasPrefix(trimmed, "<li") {
|
||||
inNovelItem = true
|
||||
cur = listing{}
|
||||
}
|
||||
|
||||
if !inNovelItem {
|
||||
continue
|
||||
}
|
||||
|
||||
// Detect end of list element.
|
||||
if trimmed == "</li>" && cur.slug != "" {
|
||||
novels = append(novels, cur)
|
||||
inNovelItem = false
|
||||
cur = listing{}
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract slug from href="/book/{slug}".
|
||||
if cur.slug == "" {
|
||||
if idx := strings.Index(trimmed, `href="/book/`); idx >= 0 {
|
||||
rest := trimmed[idx+len(`href="/book/`):]
|
||||
if end := strings.IndexAny(rest, `"/ `); end > 0 {
|
||||
cur.slug = rest[:end]
|
||||
cur.url = novelFireBase + "/book/" + cur.slug
|
||||
} else if end := strings.Index(rest, `"`); end > 0 {
|
||||
cur.slug = strings.TrimSuffix(rest[:end], "/")
|
||||
cur.url = novelFireBase + "/book/" + cur.slug
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract cover URL from data-src or src on img tags.
|
||||
if cur.coverURL == "" && strings.Contains(trimmed, "<img") {
|
||||
if src := extractAttr(trimmed, "data-src"); src != "" {
|
||||
cur.coverURL = htmlutil.ResolveURL(novelFireBase, src)
|
||||
} else if src := extractAttr(trimmed, "src"); src != "" && !strings.Contains(src, "data:") {
|
||||
cur.coverURL = htmlutil.ResolveURL(novelFireBase, src)
|
||||
}
|
||||
}
|
||||
|
||||
// Extract title from novel-title element.
|
||||
if cur.title == "" && strings.Contains(trimmed, "novel-title") {
|
||||
// Try to grab inner text: <h4 class="novel-title">Title Here</h4>
|
||||
if start := strings.Index(trimmed, ">"); start >= 0 {
|
||||
rest := trimmed[start+1:]
|
||||
if end := strings.Index(rest, "<"); end > 0 {
|
||||
title := strings.TrimSpace(rest[:end])
|
||||
if title != "" {
|
||||
cur.title = title
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush any open item that wasn't closed by </li> (e.g. last item in file).
|
||||
if inNovelItem && cur.slug != "" {
|
||||
novels = append(novels, cur)
|
||||
}
|
||||
|
||||
return novels
|
||||
}
|
||||
|
||||
// extractAttr extracts an HTML attribute value from a raw tag string.
|
||||
// e.g. extractAttr(`<img data-src="foo.jpg">`, "data-src") → "foo.jpg"
|
||||
func extractAttr(tag, attr string) string {
|
||||
needle := attr + `="`
|
||||
idx := strings.Index(tag, needle)
|
||||
if idx < 0 {
|
||||
return ""
|
||||
}
|
||||
rest := tag[idx+len(needle):]
|
||||
end := strings.Index(rest, `"`)
|
||||
if end < 0 {
|
||||
return ""
|
||||
}
|
||||
return rest[:end]
|
||||
}
|
||||
|
||||
func envOr(key, fallback string) string {
|
||||
@@ -199,19 +475,31 @@ Commands:
|
||||
run [--url <book-url>] One-shot: scrape full catalogue, or a single book
|
||||
refresh <slug> Re-scrape a book from its saved source_url
|
||||
serve Start HTTP server (POST /scrape, POST /scrape/book)
|
||||
save-browse Capture browse pages via SingleFile → MinIO
|
||||
--genre <slug> genre filter (default: all)
|
||||
--sort <value> sort order (default: popular)
|
||||
--status <value> status filter (default: all)
|
||||
--type <value> novel type (default: all-novel)
|
||||
--max-pages <n> pages to capture (default: 5)
|
||||
|
||||
Environment variables:
|
||||
BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
||||
BROWSERLESS_TOKEN API token (default: "")
|
||||
BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct)
|
||||
BROWSERLESS_URL_STRATEGY Strategy for URL retrieval (default: content)
|
||||
BROWSERLESS_MAX_CONCURRENT Max simultaneous sessions (default: 5)
|
||||
BROWSERLESS_TIMEOUT HTTP request timeout sec (default: 90)
|
||||
SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d)
|
||||
SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
|
||||
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||
SCRAPER_TIMEOUT HTTP request timeout sec (default: 90)
|
||||
KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled)
|
||||
KOKORO_VOICE Default TTS voice (default: af_bella)
|
||||
POCKETBASE_URL PocketBase base URL (default: http://localhost:8090)
|
||||
POCKETBASE_ADMIN_EMAIL PocketBase admin email (default: admin@libnovel.local)
|
||||
POCKETBASE_ADMIN_PASSWORD PocketBase admin password (default: changeme123)
|
||||
MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000)
|
||||
MINIO_ACCESS_KEY MinIO access key (default: admin)
|
||||
MINIO_SECRET_KEY MinIO secret key (default: changeme123)
|
||||
MINIO_USE_SSL MinIO TLS (default: false)
|
||||
MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters)
|
||||
MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio)
|
||||
MINIO_BUCKET_BROWSE Browse snapshots bucket (default: libnovel-browse)
|
||||
BROWSERLESS_URL Browserless WS endpoint (default: http://localhost:3030)
|
||||
SINGLEFILE_PATH Path to single-file CLI (default: single-file)
|
||||
LOG_LEVEL debug|info|warn|error (default: info)
|
||||
`, runtime.NumCPU())
|
||||
}
|
||||
|
||||
@@ -3,8 +3,35 @@ module github.com/libnovel/scraper
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/yuin/goldmark v1.7.16 // indirect
|
||||
golang.org/x/net v0.51.0 // indirect
|
||||
github.com/minio/minio-go/v7 v7.0.98
|
||||
golang.org/x/net v0.51.0
|
||||
honnef.co/go/tools v0.7.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
|
||||
golang.org/x/mod v0.32.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
golang.org/x/tools v0.41.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
tool honnef.co/go/tools/cmd/staticcheck
|
||||
|
||||
@@ -1,9 +1,61 @@
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE=
|
||||
github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
|
||||
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
|
||||
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ=
|
||||
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
|
||||
golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
|
||||
golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
|
||||
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
|
||||
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
|
||||
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
|
||||
golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
|
||||
golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU=
|
||||
honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc=
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// cdpClient implements BrowserClient using the CDP WebSocket endpoint.
|
||||
type cdpClient struct {
|
||||
cfg Config
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
// NewCDPClient returns a BrowserClient that uses CDP WebSocket sessions.
|
||||
func NewCDPClient(cfg Config) BrowserClient {
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = 60 * time.Second
|
||||
}
|
||||
return &cdpClient{cfg: cfg, sem: makeSem(cfg.MaxConcurrent)}
|
||||
}
|
||||
|
||||
func (c *cdpClient) Strategy() Strategy { return StrategyCDP }
|
||||
|
||||
func (c *cdpClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
|
||||
return "", fmt.Errorf("CDP client does not support /content; use NewContentClient")
|
||||
}
|
||||
|
||||
func (c *cdpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
|
||||
return ScrapeResponse{}, fmt.Errorf("CDP client does not support /scrape; use NewScrapeClient")
|
||||
}
|
||||
|
||||
// CDPSession opens a WebSocket to the Browserless /devtools/browser endpoint,
|
||||
// navigates to pageURL, and invokes fn with a live CDPConn.
|
||||
func (c *cdpClient) CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error {
|
||||
if err := acquire(ctx, c.sem); err != nil {
|
||||
return fmt.Errorf("cdp: semaphore: %w", err)
|
||||
}
|
||||
defer release(c.sem)
|
||||
|
||||
// Build WebSocket URL: ws://host:port/devtools/browser?token=...&url=...
|
||||
wsURL := strings.Replace(c.cfg.BaseURL, "http://", "ws://", 1)
|
||||
wsURL = strings.Replace(wsURL, "https://", "wss://", 1)
|
||||
wsURL += "/devtools/browser"
|
||||
sep := "?"
|
||||
if c.cfg.Token != "" {
|
||||
wsURL += sep + "token=" + c.cfg.Token
|
||||
sep = "&"
|
||||
}
|
||||
wsURL += sep + "url=" + pageURL
|
||||
|
||||
dialer := websocket.Dialer{
|
||||
HandshakeTimeout: 15 * time.Second,
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
}
|
||||
|
||||
conn, _, err := dialer.DialContext(ctx, wsURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cdp: dial %s: %w", wsURL, err)
|
||||
}
|
||||
|
||||
cdp := &cdpConn{ws: conn}
|
||||
defer cdp.Close()
|
||||
|
||||
return fn(ctx, cdp)
|
||||
}
|
||||
|
||||
// ─── cdpConn ─────────────────────────────────────────────────────────────────
|
||||
|
||||
type cdpConn struct {
|
||||
ws *websocket.Conn
|
||||
counter atomic.Int64
|
||||
}
|
||||
|
||||
type cdpRequest struct {
|
||||
ID int64 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params map[string]any `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
type cdpResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
Result map[string]any `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (c *cdpConn) Send(ctx context.Context, method string, params map[string]any) (map[string]any, error) {
|
||||
id := c.counter.Add(1)
|
||||
|
||||
req := cdpRequest{ID: id, Method: method, Params: params}
|
||||
data, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdp send: marshal: %w", err)
|
||||
}
|
||||
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.ws.SetWriteDeadline(dl)
|
||||
}
|
||||
if err := c.ws.WriteMessage(websocket.TextMessage, data); err != nil {
|
||||
return nil, fmt.Errorf("cdp send: write: %w", err)
|
||||
}
|
||||
|
||||
// Read messages until we find the response matching our id.
|
||||
for {
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.ws.SetReadDeadline(dl)
|
||||
}
|
||||
_, msg, err := c.ws.ReadMessage()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdp send: read: %w", err)
|
||||
}
|
||||
var resp cdpResponse
|
||||
if err := json.Unmarshal(msg, &resp); err != nil {
|
||||
continue // skip non-JSON frames (events etc.)
|
||||
}
|
||||
if resp.ID != id {
|
||||
continue // event or different command reply
|
||||
}
|
||||
if resp.Error != nil {
|
||||
return nil, fmt.Errorf("cdp error %d: %s", resp.Error.Code, resp.Error.Message)
|
||||
}
|
||||
return resp.Result, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cdpConn) Close() error {
|
||||
return c.ws.Close()
|
||||
}
|
||||
@@ -55,6 +55,8 @@ func release(sem chan struct{}) {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── /content client ──────────────────────────────────────────────────────────
|
||||
|
||||
// contentClient implements BrowserClient using the /content endpoint.
|
||||
type contentClient struct {
|
||||
cfg Config
|
||||
@@ -121,75 +123,5 @@ func (c *contentClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeRe
|
||||
}
|
||||
|
||||
func (c *contentClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
|
||||
return fmt.Errorf("content client does not support CDP; use NewCDPClient")
|
||||
}
|
||||
|
||||
// ─── /scrape client ───────────────────────────────────────────────────────────
|
||||
|
||||
type scrapeClient struct {
|
||||
cfg Config
|
||||
http *http.Client
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
// NewScrapeClient returns a BrowserClient that uses POST /scrape.
|
||||
func NewScrapeClient(cfg Config) BrowserClient {
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = 90 * time.Second
|
||||
}
|
||||
return &scrapeClient{
|
||||
cfg: cfg,
|
||||
http: &http.Client{Timeout: cfg.Timeout},
|
||||
sem: makeSem(cfg.MaxConcurrent),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *scrapeClient) Strategy() Strategy { return StrategyScrape }
|
||||
|
||||
func (c *scrapeClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
|
||||
return "", fmt.Errorf("scrape client does not support /content; use NewContentClient")
|
||||
}
|
||||
|
||||
func (c *scrapeClient) ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) {
|
||||
if err := acquire(ctx, c.sem); err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: semaphore: %w", err)
|
||||
}
|
||||
defer release(c.sem)
|
||||
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: marshal request: %w", err)
|
||||
}
|
||||
|
||||
url := c.cfg.BaseURL + "/scrape"
|
||||
if c.cfg.Token != "" {
|
||||
url += "?token=" + c.cfg.Token
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: build request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.http.Do(httpReq)
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: unexpected status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
|
||||
var result ScrapeResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: decode response: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *scrapeClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
|
||||
return fmt.Errorf("scrape client does not support CDP; use NewCDPClient")
|
||||
return fmt.Errorf("content client does not support CDP")
|
||||
}
|
||||
818
scraper/internal/e2e/e2e_test.go
Normal file
818
scraper/internal/e2e/e2e_test.go
Normal file
@@ -0,0 +1,818 @@
|
||||
//go:build integration
|
||||
|
||||
// End-to-end integration test for libnovel.
|
||||
//
|
||||
// Scenario (executed in order):
|
||||
// 1. Health-check all Docker services (PocketBase, MinIO, Browserless, scraper).
|
||||
// 2. Register a test user in the app_users PocketBase collection.
|
||||
// 3. Scrape the popular-ranking page 1 and capture the first book.
|
||||
// 4. Scrape full metadata for that book and persist it; verify in PocketBase.
|
||||
// 5. Scrape chapters 1–3 and persist them; verify in MinIO + PocketBase.
|
||||
// 6. Generate TTS audio for the first 100 chars of each chapter via the scraper
|
||||
// HTTP API; verify MinIO object + PocketBase audio_cache entry.
|
||||
// 7. Fetch presigned URLs for each chapter's markdown and audio; verify HTTP 200.
|
||||
//
|
||||
// Prerequisites (all must be running):
|
||||
//
|
||||
// docker-compose up -d minio pocketbase browserless scraper
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// BROWSERLESS_URL=http://localhost:3030 \
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// SCRAPER_URL=http://localhost:8080 \
|
||||
// go test -v -tags integration -timeout 900s \
|
||||
// github.com/libnovel/scraper/internal/e2e
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/browser"
|
||||
"github.com/libnovel/scraper/internal/novelfire"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// ─── env helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// ─── fixture ─────────────────────────────────────────────────────────────────
|
||||
|
||||
type e2eFixture struct {
|
||||
sc *novelfire.Scraper
|
||||
hs *storage.HybridStore
|
||||
scraperURL string // base URL of the running scraper HTTP server
|
||||
pbBaseURL string
|
||||
pbEmail string
|
||||
pbPassword string
|
||||
}
|
||||
|
||||
func newE2EFixture(t *testing.T) *e2eFixture {
|
||||
t.Helper()
|
||||
|
||||
browserlessURL := envOr("BROWSERLESS_URL", "")
|
||||
if browserlessURL == "" {
|
||||
t.Skip("BROWSERLESS_URL not set — skipping e2e test")
|
||||
}
|
||||
if os.Getenv("MINIO_ENDPOINT") == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping e2e test")
|
||||
}
|
||||
if os.Getenv("POCKETBASE_URL") == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping e2e test")
|
||||
}
|
||||
scraperURL := envOr("SCRAPER_URL", "http://localhost:8080")
|
||||
|
||||
pbBaseURL := envOr("POCKETBASE_URL", "http://localhost:8090")
|
||||
pbEmail := envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local")
|
||||
pbPassword := envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123")
|
||||
|
||||
pbCfg := storage.PocketBaseConfig{
|
||||
BaseURL: pbBaseURL,
|
||||
AdminEmail: pbEmail,
|
||||
AdminPassword: pbPassword,
|
||||
}
|
||||
minioCfg := storage.MinioConfig{
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: envOr("MINIO_USE_SSL", "false") == "true",
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
hs, err := storage.NewHybridStore(ctx, pbCfg, minioCfg, slog.Default())
|
||||
if err != nil {
|
||||
t.Fatalf("NewHybridStore: %v", err)
|
||||
}
|
||||
|
||||
// directClient: plain HTTP GET — used for chapter text, metadata, and ranking
|
||||
// (novelfire.net serves these pages server-side; no JS rendering needed).
|
||||
directClient := browser.NewDirectHTTPClient(browser.Config{
|
||||
Timeout: 60 * time.Second,
|
||||
MaxConcurrent: 2,
|
||||
})
|
||||
// urlClient: Browserless content strategy — used only for chapter-list
|
||||
// pagination pages which require JS rendering to populate the list.
|
||||
urlClient := browser.NewContentClient(browser.Config{
|
||||
BaseURL: browserlessURL,
|
||||
Token: os.Getenv("BROWSERLESS_TOKEN"),
|
||||
Timeout: 120 * time.Second,
|
||||
MaxConcurrent: 2,
|
||||
})
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
sc := novelfire.New(directClient, log, urlClient, directClient, nil)
|
||||
|
||||
return &e2eFixture{
|
||||
sc: sc,
|
||||
hs: hs,
|
||||
scraperURL: scraperURL,
|
||||
pbBaseURL: pbBaseURL,
|
||||
pbEmail: pbEmail,
|
||||
pbPassword: pbPassword,
|
||||
}
|
||||
}
|
||||
|
||||
// ─── The single end-to-end test ───────────────────────────────────────────────
|
||||
|
||||
// TestE2E_FullScenario executes the complete end-to-end scenario in order.
|
||||
func TestE2E_FullScenario(t *testing.T) {
|
||||
f := newE2EFixture(t)
|
||||
|
||||
// ── Step 1: Health-check services ────────────────────────────────────────
|
||||
t.Run("step1_health_checks", func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// PocketBase health
|
||||
pbHealth := f.pbBaseURL + "/api/health"
|
||||
checkHTTP(t, ctx, pbHealth, "PocketBase")
|
||||
|
||||
// MinIO health — the MinIO console liveness endpoint
|
||||
minioEndpoint := envOr("MINIO_ENDPOINT", "localhost:9000")
|
||||
scheme := "http"
|
||||
if envOr("MINIO_USE_SSL", "false") == "true" {
|
||||
scheme = "https"
|
||||
}
|
||||
minioHealth := fmt.Sprintf("%s://%s/minio/health/live", scheme, minioEndpoint)
|
||||
checkHTTP(t, ctx, minioHealth, "MinIO")
|
||||
|
||||
// Browserless health — /pressure is the liveness endpoint
|
||||
browserlessURL := envOr("BROWSERLESS_URL", "http://localhost:3030")
|
||||
blHealth := browserlessURL + "/pressure"
|
||||
checkHTTP(t, ctx, blHealth, "Browserless")
|
||||
|
||||
// Scraper server health — wait up to 10 s for it to be ready
|
||||
scraperHealth := f.scraperURL + "/health"
|
||||
waitForHTTP(t, ctx, scraperHealth, "scraper server", 10*time.Second)
|
||||
})
|
||||
|
||||
// ── Step 2: Register test user ────────────────────────────────────────────
|
||||
var testUsername string
|
||||
t.Run("step2_register_user", func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
testUsername = fmt.Sprintf("e2euser-%d", time.Now().UnixMilli()%100000)
|
||||
passwordHash := "pbkdf2:sha256:dummy-hash-for-test"
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cleanCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cleanCancel()
|
||||
deleteAppUser(t, f, cleanCtx, testUsername)
|
||||
})
|
||||
|
||||
if err := createAppUser(ctx, f, testUsername, passwordHash, "reader"); err != nil {
|
||||
t.Fatalf("createAppUser: %v", err)
|
||||
}
|
||||
t.Logf("created user %q", testUsername)
|
||||
|
||||
// Verify the user exists in PocketBase.
|
||||
rec, err := getAppUserByUsername(ctx, f, testUsername)
|
||||
if err != nil {
|
||||
t.Fatalf("getAppUserByUsername: %v", err)
|
||||
}
|
||||
if rec == nil {
|
||||
t.Fatal("user not found in app_users after creation")
|
||||
}
|
||||
if rec["username"] != testUsername {
|
||||
t.Errorf("username = %q, want %q", rec["username"], testUsername)
|
||||
}
|
||||
t.Logf("user verified in PocketBase: id=%v username=%v role=%v", rec["id"], rec["username"], rec["role"])
|
||||
})
|
||||
|
||||
// ── Step 3: Scrape ranking page 1, capture first book ────────────────────
|
||||
var firstBook scraper.BookMeta
|
||||
t.Run("step3_scrape_ranking", func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
|
||||
entries, errs := f.sc.ScrapeRanking(ctx, 1) // maxPages=1 → only page 1
|
||||
|
||||
select {
|
||||
case meta, ok := <-entries:
|
||||
if !ok {
|
||||
t.Fatal("ranking channel closed without any entry")
|
||||
}
|
||||
firstBook = meta
|
||||
case err := <-errs:
|
||||
t.Fatalf("ScrapeRanking error: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("ScrapeRanking timed out waiting for first entry")
|
||||
}
|
||||
|
||||
// Drain remaining entries and errors.
|
||||
for range entries {
|
||||
}
|
||||
for range errs {
|
||||
}
|
||||
|
||||
if firstBook.Slug == "" {
|
||||
t.Fatal("first book has empty slug")
|
||||
}
|
||||
if firstBook.Title == "" {
|
||||
t.Fatal("first book has empty title")
|
||||
}
|
||||
if firstBook.SourceURL == "" {
|
||||
t.Fatal("first book has empty SourceURL")
|
||||
}
|
||||
t.Logf("first ranked book: slug=%q title=%q rank=%d url=%s",
|
||||
firstBook.Slug, firstBook.Title, firstBook.Ranking, firstBook.SourceURL)
|
||||
})
|
||||
|
||||
if firstBook.Slug == "" || firstBook.SourceURL == "" {
|
||||
t.Fatal("cannot continue: step3 did not produce a valid first book")
|
||||
}
|
||||
|
||||
// Use a unique slug for the test to avoid colliding with real scraped data.
|
||||
testSlug := fmt.Sprintf("%s-e2e-%d", firstBook.Slug, time.Now().UnixMilli()%100000)
|
||||
t.Logf("using test slug: %q", testSlug)
|
||||
|
||||
// Register cleanup for all data written by subsequent steps.
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cleanCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cleanCancel()
|
||||
cleanupTestData(t, f, cleanCtx, testSlug)
|
||||
})
|
||||
|
||||
// ── Step 4: Scrape book metadata and persist ──────────────────────────────
|
||||
var fullMeta scraper.BookMeta
|
||||
t.Run("step4_scrape_metadata", func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
|
||||
meta, err := f.sc.ScrapeMetadata(ctx, firstBook.SourceURL)
|
||||
if err != nil {
|
||||
t.Fatalf("ScrapeMetadata: %v", err)
|
||||
}
|
||||
t.Logf("scraped metadata: title=%q author=%q totalChapters=%d",
|
||||
meta.Title, meta.Author, meta.TotalChapters)
|
||||
|
||||
// Override slug so data lands under our test slug.
|
||||
meta.Slug = testSlug
|
||||
fullMeta = meta
|
||||
|
||||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer storeCancel()
|
||||
|
||||
if err := f.hs.WriteMetadata(storeCtx, meta); err != nil {
|
||||
t.Fatalf("WriteMetadata: %v", err)
|
||||
}
|
||||
|
||||
// Verify in PocketBase.
|
||||
got, found, err := f.hs.ReadMetadata(storeCtx, testSlug)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMetadata: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("book not found in PocketBase after WriteMetadata")
|
||||
}
|
||||
if got.Title == "" {
|
||||
t.Error("book title is empty after round-trip")
|
||||
}
|
||||
if got.Author == "" {
|
||||
t.Logf("WARNING: book author is empty after round-trip (site may not expose author for this book)")
|
||||
}
|
||||
t.Logf("PocketBase verified: title=%q author=%q totalChapters=%d", got.Title, got.Author, got.TotalChapters)
|
||||
})
|
||||
|
||||
if fullMeta.SourceURL == "" {
|
||||
fullMeta.SourceURL = firstBook.SourceURL
|
||||
}
|
||||
|
||||
// ── Step 5: Scrape first 3 chapters and persist ───────────────────────────
|
||||
var chapterRefs []scraper.ChapterRef
|
||||
t.Run("step5_scrape_chapters", func(t *testing.T) {
|
||||
// Fetch only page 1 of the chapter list from
|
||||
// https://novelfire.net/book/{slug}/chapters?page=1
|
||||
// to avoid paginating through hundreds of pages for popular books.
|
||||
listCtx, listCancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer listCancel()
|
||||
|
||||
chaptersPageURL := firstBook.SourceURL + "/chapters?page=1"
|
||||
refs, err := scrapeChapterListPage1(listCtx, f, chaptersPageURL)
|
||||
if err != nil {
|
||||
t.Fatalf("scrapeChapterListPage1: %v", err)
|
||||
}
|
||||
if len(refs) == 0 {
|
||||
t.Fatal("chapter list page 1 returned no chapters")
|
||||
}
|
||||
t.Logf("chapter list page 1: %d chapters found", len(refs))
|
||||
|
||||
// Take the first 3 (or fewer if page 1 has < 3 chapters).
|
||||
n := 3
|
||||
if len(refs) < n {
|
||||
n = len(refs)
|
||||
}
|
||||
chapterRefs = refs[:n]
|
||||
t.Logf("will scrape first %d chapters: %v", n, chapterNumbers(chapterRefs))
|
||||
|
||||
for _, ref := range chapterRefs {
|
||||
ref := ref
|
||||
t.Run(fmt.Sprintf("chapter-%d", ref.Number), func(t *testing.T) {
|
||||
scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 180*time.Second)
|
||||
defer scrapeCancel()
|
||||
|
||||
ch, err := f.sc.ScrapeChapterText(scrapeCtx, ref)
|
||||
if err != nil {
|
||||
t.Fatalf("ScrapeChapterText(%d): %v", ref.Number, err)
|
||||
}
|
||||
t.Logf("scraped chapter %d: %d bytes", ref.Number, len(ch.Text))
|
||||
if len(ch.Text) < 50 {
|
||||
t.Errorf("chapter %d text too short (%d bytes)", ref.Number, len(ch.Text))
|
||||
}
|
||||
|
||||
// Override ref slug with our test slug.
|
||||
ch.Ref.Number = ref.Number
|
||||
ch.Ref.Title = ref.Title
|
||||
|
||||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer storeCancel()
|
||||
|
||||
if err := f.hs.WriteChapter(storeCtx, testSlug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter(%d): %v", ref.Number, err)
|
||||
}
|
||||
|
||||
// Verify in MinIO via ReadChapter.
|
||||
got, err := f.hs.ReadChapter(storeCtx, testSlug, ref.Number)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadChapter(%d): %v", ref.Number, err)
|
||||
}
|
||||
if got == "" {
|
||||
t.Errorf("chapter %d: ReadChapter returned empty content", ref.Number)
|
||||
}
|
||||
if !strings.HasPrefix(got, "# ") {
|
||||
t.Errorf("chapter %d: stored content missing markdown header (got %q)", ref.Number, got[:min(len(got), 80)])
|
||||
}
|
||||
|
||||
// Verify PocketBase chapters_idx entry.
|
||||
idxCtx, idxCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer idxCancel()
|
||||
count := f.hs.CountChapters(idxCtx, testSlug)
|
||||
if count == 0 {
|
||||
t.Errorf("chapter %d: chapters_idx count = 0 after WriteChapter", ref.Number)
|
||||
}
|
||||
t.Logf("chapter %d stored; chapters_idx count=%d", ref.Number, count)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
if len(chapterRefs) == 0 {
|
||||
t.Fatal("cannot continue: step5 produced no chapter refs")
|
||||
}
|
||||
|
||||
// ── Step 6: Generate TTS audio via scraper HTTP API ───────────────────────
|
||||
t.Run("step6_tts_audio", func(t *testing.T) {
|
||||
if os.Getenv("SCRAPER_URL") == "" {
|
||||
t.Skip("SCRAPER_URL not set — skipping TTS step")
|
||||
}
|
||||
|
||||
voice := envOr("KOKORO_VOICE", "af_bella")
|
||||
|
||||
for _, ref := range chapterRefs {
|
||||
ref := ref
|
||||
t.Run(fmt.Sprintf("audio-chapter-%d", ref.Number), func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
audioURL := fmt.Sprintf("%s/api/audio/%s/%d", f.scraperURL, testSlug, ref.Number)
|
||||
body, _ := json.Marshal(map[string]interface{}{
|
||||
"voice": voice,
|
||||
"speed": 1.0,
|
||||
"max_chars": 200,
|
||||
})
|
||||
|
||||
audioReq, _ := http.NewRequestWithContext(ctx, http.MethodPost, audioURL, bytes.NewReader(body))
|
||||
audioReq.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(audioReq)
|
||||
if err != nil {
|
||||
t.Fatalf("POST %s: %v", audioURL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("audio generation status=%d body=%s", resp.StatusCode, raw)
|
||||
}
|
||||
|
||||
var audioResp struct {
|
||||
URL string `json:"url"`
|
||||
Filename string `json:"filename"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&audioResp); err != nil {
|
||||
t.Fatalf("decode audio response: %v", err)
|
||||
}
|
||||
if audioResp.URL == "" {
|
||||
t.Error("audio response has empty url field")
|
||||
}
|
||||
if audioResp.Filename == "" {
|
||||
t.Error("audio response has empty filename field")
|
||||
}
|
||||
t.Logf("chapter %d audio: url=%s filename=%s", ref.Number, audioResp.URL, audioResp.Filename)
|
||||
|
||||
// Verify audio_cache entry exists in PocketBase.
|
||||
pbCtx, pbCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer pbCancel()
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s/1.00", testSlug, ref.Number, voice)
|
||||
filename, found := f.hs.GetAudioCache(pbCtx, cacheKey)
|
||||
if !found {
|
||||
t.Errorf("audio_cache entry not found for key=%q", cacheKey)
|
||||
} else {
|
||||
t.Logf("audio_cache[%q] = %q", cacheKey, filename)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
// ── Step 7: Presigned URLs ────────────────────────────────────────────────
|
||||
t.Run("step7_presigned_urls", func(t *testing.T) {
|
||||
if os.Getenv("SCRAPER_URL") == "" {
|
||||
t.Skip("SCRAPER_URL not set — skipping presign step")
|
||||
}
|
||||
|
||||
// Give the background MinIO upload goroutines (launched by handleAudioGenerate)
|
||||
// a moment to complete before we attempt to access the presigned URLs.
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
voice := envOr("KOKORO_VOICE", "af_bella")
|
||||
|
||||
for _, ref := range chapterRefs {
|
||||
ref := ref
|
||||
t.Run(fmt.Sprintf("presign-chapter-%d", ref.Number), func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Chapter markdown presign.
|
||||
chPresignURL := fmt.Sprintf("%s/api/presign/chapter/%s/%d",
|
||||
f.scraperURL, testSlug, ref.Number)
|
||||
chPresigned := fetchPresignedURL(t, ctx, chPresignURL, "chapter presign")
|
||||
if chPresigned != "" {
|
||||
assertURLAccessible(t, ctx, chPresigned, fmt.Sprintf("chapter %d presigned URL", ref.Number))
|
||||
}
|
||||
|
||||
// Audio presign — poll with retries to allow background MinIO upload to finish.
|
||||
auPresignURL := fmt.Sprintf("%s/api/presign/audio/%s/%d?voice=%s&speed=1.0",
|
||||
f.scraperURL, testSlug, ref.Number, voice)
|
||||
auPresigned := fetchPresignedURL(t, ctx, auPresignURL, "audio presign")
|
||||
if auPresigned != "" {
|
||||
assertURLAccessibleWithRetry(t, ctx, auPresigned, fmt.Sprintf("chapter %d audio presigned URL", ref.Number), 6, 5*time.Second)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ─── PocketBase admin helpers ─────────────────────────────────────────────────
|
||||
|
||||
// pbAuthToken obtains a PocketBase superuser JWT.
|
||||
func pbAuthToken(ctx context.Context, f *e2eFixture) (string, error) {
|
||||
body, _ := json.Marshal(map[string]string{
|
||||
"identity": f.pbEmail,
|
||||
"password": f.pbPassword,
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
f.pbBaseURL+"/api/collections/_superusers/auth-with-password",
|
||||
bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pbAuthToken: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return "", fmt.Errorf("pbAuthToken status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
var result struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return "", fmt.Errorf("pbAuthToken decode: %w", err)
|
||||
}
|
||||
return result.Token, nil
|
||||
}
|
||||
|
||||
// createAppUser inserts a record into app_users via PocketBase admin API.
|
||||
func createAppUser(ctx context.Context, f *e2eFixture, username, passwordHash, role string) error {
|
||||
tok, err := pbAuthToken(ctx, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]interface{}{
|
||||
"username": username,
|
||||
"password_hash": passwordHash,
|
||||
"role": role,
|
||||
"created": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
f.pbBaseURL+"/api/collections/app_users/records",
|
||||
bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", tok)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("createAppUser: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("createAppUser status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getAppUserByUsername fetches an app_users record by username.
|
||||
// Returns nil, nil when not found.
|
||||
func getAppUserByUsername(ctx context.Context, f *e2eFixture, username string) (map[string]interface{}, error) {
|
||||
tok, err := pbAuthToken(ctx, f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
url := fmt.Sprintf("%s/api/collections/app_users/records?filter=username%%3D%%22%s%%22&perPage=1",
|
||||
f.pbBaseURL, username)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", tok)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getAppUserByUsername: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var result struct {
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("getAppUserByUsername decode: %w", err)
|
||||
}
|
||||
if len(result.Items) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return result.Items[0], nil
|
||||
}
|
||||
|
||||
// deleteAppUser removes app_users records matching username.
|
||||
func deleteAppUser(t *testing.T, f *e2eFixture, ctx context.Context, username string) {
|
||||
t.Helper()
|
||||
tok, err := pbAuthToken(ctx, f)
|
||||
if err != nil {
|
||||
t.Logf("deleteAppUser: pbAuthToken error: %v", err)
|
||||
return
|
||||
}
|
||||
// List matching records.
|
||||
url := fmt.Sprintf("%s/api/collections/app_users/records?filter=username%%3D%%22%s%%22&perPage=10",
|
||||
f.pbBaseURL, username)
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
req.Header.Set("Authorization", tok)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Logf("deleteAppUser list error: %v", err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var result struct {
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
_ = json.NewDecoder(resp.Body).Decode(&result)
|
||||
for _, item := range result.Items {
|
||||
id, _ := item["id"].(string)
|
||||
delURL := fmt.Sprintf("%s/api/collections/app_users/records/%s", f.pbBaseURL, id)
|
||||
delReq, _ := http.NewRequestWithContext(ctx, http.MethodDelete, delURL, nil)
|
||||
delReq.Header.Set("Authorization", tok)
|
||||
delResp, _ := http.DefaultClient.Do(delReq)
|
||||
if delResp != nil {
|
||||
delResp.Body.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupTestData removes all PocketBase + MinIO data for the given slug.
|
||||
func cleanupTestData(t *testing.T, f *e2eFixture, ctx context.Context, slug string) {
|
||||
t.Helper()
|
||||
tok, err := pbAuthToken(ctx, f)
|
||||
if err != nil {
|
||||
t.Logf("cleanupTestData: pbAuthToken error: %v", err)
|
||||
return
|
||||
}
|
||||
pbDelete := func(collection, filter string) {
|
||||
listURL := fmt.Sprintf("%s/api/collections/%s/records?filter=%s&perPage=500",
|
||||
f.pbBaseURL, collection, filter)
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, listURL, nil)
|
||||
req.Header.Set("Authorization", tok)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Logf("cleanupTestData list %s error: %v", collection, err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var result struct {
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
_ = json.NewDecoder(resp.Body).Decode(&result)
|
||||
for _, item := range result.Items {
|
||||
id, _ := item["id"].(string)
|
||||
delURL := fmt.Sprintf("%s/api/collections/%s/records/%s", f.pbBaseURL, collection, id)
|
||||
delReq, _ := http.NewRequestWithContext(ctx, http.MethodDelete, delURL, nil)
|
||||
delReq.Header.Set("Authorization", tok)
|
||||
delResp, _ := http.DefaultClient.Do(delReq)
|
||||
if delResp != nil {
|
||||
delResp.Body.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
slugFilter := fmt.Sprintf("slug%%3D%%22%s%%22", slug)
|
||||
ckFilter := fmt.Sprintf("cache_key%%7E%%22%s%%2F%%22", slug) // cache_key ~ "slug/"
|
||||
pbDelete("books", slugFilter)
|
||||
pbDelete("chapters_idx", slugFilter)
|
||||
pbDelete("audio_cache", ckFilter)
|
||||
t.Logf("cleanup complete for slug=%q", slug)
|
||||
}
|
||||
|
||||
// ─── HTTP assertion helpers ───────────────────────────────────────────────────
|
||||
|
||||
// checkHTTP asserts that a GET to url returns 2xx within the context deadline.
|
||||
func checkHTTP(t *testing.T, ctx context.Context, url, name string) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
t.Errorf("%s health check: build request: %v", name, err)
|
||||
return
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Errorf("%s health check failed: %v", name, err)
|
||||
return
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
t.Errorf("%s health check: status %d, want 2xx", name, resp.StatusCode)
|
||||
return
|
||||
}
|
||||
t.Logf("%s health OK (HTTP %d)", name, resp.StatusCode)
|
||||
}
|
||||
|
||||
// waitForHTTP retries GET url until a 2xx is received or timeout is reached.
|
||||
func waitForHTTP(t *testing.T, ctx context.Context, url, name string, timeout time.Duration) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
var lastErr error
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Errorf("%s: context cancelled while waiting for health", name)
|
||||
return
|
||||
default:
|
||||
}
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err == nil && resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
resp.Body.Close()
|
||||
t.Logf("%s health OK (HTTP %d)", name, resp.StatusCode)
|
||||
return
|
||||
}
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
lastErr = fmt.Errorf("status %d", resp.StatusCode)
|
||||
} else {
|
||||
lastErr = err
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
t.Errorf("%s not healthy after %s: %v", name, timeout, lastErr)
|
||||
}
|
||||
|
||||
// fetchPresignedURL calls the presign endpoint and returns the presigned URL.
|
||||
// It logs and returns "" on failure (non-fatal) so the caller can decide.
|
||||
func fetchPresignedURL(t *testing.T, ctx context.Context, presignEndpoint, label string) string {
|
||||
t.Helper()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, presignEndpoint, nil)
|
||||
if err != nil {
|
||||
t.Errorf("fetchPresignedURL %s: %v", label, err)
|
||||
return ""
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Errorf("fetchPresignedURL %s: %v", label, err)
|
||||
return ""
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Errorf("fetchPresignedURL %s: status %d body=%s", label, resp.StatusCode, b)
|
||||
return ""
|
||||
}
|
||||
var body struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
t.Errorf("fetchPresignedURL %s decode: %v", label, err)
|
||||
return ""
|
||||
}
|
||||
if body.URL == "" {
|
||||
t.Errorf("fetchPresignedURL %s: empty url in response", label)
|
||||
return ""
|
||||
}
|
||||
t.Logf("%s presigned URL: %s", label, body.URL)
|
||||
return body.URL
|
||||
}
|
||||
|
||||
// assertURLAccessible does a GET to url and asserts HTTP 200.
|
||||
func assertURLAccessible(t *testing.T, ctx context.Context, url, label string) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
t.Errorf("%s: build request: %v", label, err)
|
||||
return
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Errorf("%s: GET error: %v", label, err)
|
||||
return
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("%s: status %d, want 200", label, resp.StatusCode)
|
||||
return
|
||||
}
|
||||
t.Logf("%s: HTTP 200 OK", label)
|
||||
}
|
||||
|
||||
// assertURLAccessibleWithRetry retries GET url up to maxAttempts times with
|
||||
// interval between attempts, asserting HTTP 200 on any success.
|
||||
func assertURLAccessibleWithRetry(t *testing.T, ctx context.Context, url, label string, maxAttempts int, interval time.Duration) {
|
||||
t.Helper()
|
||||
var lastStatus int
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
t.Errorf("%s: build request: %v", label, err)
|
||||
return
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Logf("%s: attempt %d GET error: %v", label, attempt, err)
|
||||
} else {
|
||||
resp.Body.Close()
|
||||
lastStatus = resp.StatusCode
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
t.Logf("%s: HTTP 200 OK (attempt %d)", label, attempt)
|
||||
return
|
||||
}
|
||||
t.Logf("%s: attempt %d status %d", label, attempt, resp.StatusCode)
|
||||
}
|
||||
if attempt < maxAttempts {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Errorf("%s: context cancelled before success", label)
|
||||
return
|
||||
case <-time.After(interval):
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Errorf("%s: status %d after %d attempts, want 200", label, lastStatus, maxAttempts)
|
||||
}
|
||||
|
||||
// ─── stdlib helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
func chapterNumbers(refs []scraper.ChapterRef) []int {
|
||||
ns := make([]int, len(refs))
|
||||
for i, r := range refs {
|
||||
ns[i] = r.Number
|
||||
}
|
||||
return ns
|
||||
}
|
||||
|
||||
// scrapeChapterListPage1 fetches a single chapter-list page URL via Browserless
|
||||
// and returns the chapter refs found on that page (no pagination).
|
||||
// URL should be: https://novelfire.net/book/{slug}/chapters?page=1
|
||||
func scrapeChapterListPage1(ctx context.Context, f *e2eFixture, pageURL string) ([]scraper.ChapterRef, error) {
|
||||
return f.sc.ScrapeChapterListPage(ctx, pageURL)
|
||||
}
|
||||
@@ -21,6 +21,7 @@ package novelfire
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -51,7 +52,8 @@ func newIntegrationScraper(t *testing.T) *Scraper {
|
||||
Timeout: 120 * time.Second,
|
||||
MaxConcurrent: 1,
|
||||
})
|
||||
return New(client, nil)
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
return New(client, log, client, nil, nil)
|
||||
}
|
||||
|
||||
// ── Metadata ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -2,14 +2,9 @@ package novelfire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/libnovel/scraper/internal/browser"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
)
|
||||
|
||||
// rankingPage1HTML is a realistic mock of the popular genre listing page
|
||||
@@ -116,7 +111,7 @@ func TestScrapeRanking_MultiPage(t *testing.T) {
|
||||
// Use pagedStubClient for s.client so each GetContent call returns the
|
||||
// next page. ScrapeRanking now calls s.client directly.
|
||||
urlClient := &pagedStubClient{pages: []string{rankingPage1HTML(), rankingPage2HTML()}}
|
||||
s := New(urlClient, nil, nil, nil) // nil cache — no disk I/O in tests
|
||||
s := New(urlClient, nil, nil, nil, nil) // nil cache — no disk I/O in tests
|
||||
|
||||
entryCh, errCh := s.ScrapeRanking(context.Background(), 0) // 0 = all pages
|
||||
entries := drainRanking(t, entryCh, errCh)
|
||||
@@ -151,146 +146,3 @@ func TestScrapeRanking_EmptyPage(t *testing.T) {
|
||||
t.Errorf("expected 0 entries for empty page, got %d", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteRanking_RoundTrip verifies WriteRanking → ReadRankingItems
|
||||
// faithfully reconstructs the original slice.
|
||||
func TestWriteRanking_RoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
w := writer.New(dir)
|
||||
|
||||
items := []writer.RankingItem{
|
||||
{Rank: 1, Slug: "the-iron-throne", Title: "The Iron Throne", Status: "Ongoing",
|
||||
Genres: []string{"Fantasy", "Action"}, SourceURL: "https://novelfire.net/book/the-iron-throne"},
|
||||
{Rank: 2, Slug: "shadow-mage", Title: "Shadow Mage", Status: "Completed",
|
||||
Genres: []string{"Magic"}, SourceURL: "https://novelfire.net/book/shadow-mage"},
|
||||
}
|
||||
|
||||
if err := w.WriteRanking(items); err != nil {
|
||||
t.Fatalf("WriteRanking failed: %v", err)
|
||||
}
|
||||
|
||||
rankingFile := filepath.Join(dir, "ranking.json")
|
||||
if _, err := os.Stat(rankingFile); err != nil {
|
||||
t.Fatalf("ranking.json not created: %v", err)
|
||||
}
|
||||
|
||||
got, err := w.ReadRankingItems()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadRankingItems failed: %v", err)
|
||||
}
|
||||
if len(got) != len(items) {
|
||||
t.Fatalf("expected %d items, got %d", len(items), len(got))
|
||||
}
|
||||
for i, want := range items {
|
||||
if got[i].Rank != want.Rank {
|
||||
t.Errorf("item[%d].Rank = %d, want %d", i, got[i].Rank, want.Rank)
|
||||
}
|
||||
if got[i].Slug != want.Slug {
|
||||
t.Errorf("item[%d].Slug = %q, want %q", i, got[i].Slug, want.Slug)
|
||||
}
|
||||
if got[i].Title != want.Title {
|
||||
t.Errorf("item[%d].Title = %q, want %q", i, got[i].Title, want.Title)
|
||||
}
|
||||
if got[i].Status != want.Status {
|
||||
t.Errorf("item[%d].Status = %q, want %q", i, got[i].Status, want.Status)
|
||||
}
|
||||
if len(got[i].Genres) != len(want.Genres) {
|
||||
t.Errorf("item[%d].Genres len = %d, want %d", i, len(got[i].Genres), len(want.Genres))
|
||||
} else {
|
||||
for j, g := range want.Genres {
|
||||
if got[i].Genres[j] != g {
|
||||
t.Errorf("item[%d].Genres[%d] = %q, want %q", i, j, got[i].Genres[j], g)
|
||||
}
|
||||
}
|
||||
}
|
||||
if got[i].SourceURL != want.SourceURL {
|
||||
t.Errorf("item[%d].SourceURL = %q, want %q", i, got[i].SourceURL, want.SourceURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── in-memory page cacher ─────────────────────────────────────────────────────
|
||||
|
||||
// memPageCacher is a RankingPageCacher backed by an in-memory map.
|
||||
// It records how many times each page was written and exposes the stored HTML.
|
||||
type memPageCacher struct {
|
||||
pages map[int]string
|
||||
writes map[int]int
|
||||
}
|
||||
|
||||
func newMemPageCacher() *memPageCacher {
|
||||
return &memPageCacher{pages: make(map[int]string), writes: make(map[int]int)}
|
||||
}
|
||||
|
||||
func (c *memPageCacher) WriteRankingPageCache(page int, html string) error {
|
||||
c.pages[page] = html
|
||||
c.writes[page]++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *memPageCacher) ReadRankingPageCache(page int) (string, error) {
|
||||
return c.pages[page], nil // returns "" on miss, satisfying the interface contract
|
||||
}
|
||||
|
||||
var _ scraper.RankingPageCacher = (*memPageCacher)(nil) // compile-time check
|
||||
|
||||
// TestScrapeRanking_CacheHit verifies that when a page is already in the cache
|
||||
// ScrapeRanking serves from cache and does NOT call the browser client.
|
||||
func TestScrapeRanking_CacheHit(t *testing.T) {
|
||||
cache := newMemPageCacher()
|
||||
// Pre-populate the cache with page 1 HTML.
|
||||
if err := cache.WriteRankingPageCache(1, rankingPage1HTML()); err != nil {
|
||||
t.Fatalf("cache write: %v", err)
|
||||
}
|
||||
cache.writes[1] = 0 // reset write counter — we only care about fetches
|
||||
|
||||
// The stub client panics on any GetContent call so we can prove it is not used.
|
||||
panicClient := &panicOnGetContent{}
|
||||
s := New(panicClient, nil, panicClient, cache)
|
||||
|
||||
entryCh, errCh := s.ScrapeRanking(context.Background(), 1)
|
||||
entries := drainRanking(t, entryCh, errCh)
|
||||
|
||||
if len(entries) != 2 {
|
||||
t.Fatalf("expected 2 entries from cache, got %d", len(entries))
|
||||
}
|
||||
// Cache should not have been written again (we served from cache).
|
||||
if cache.writes[1] != 0 {
|
||||
t.Errorf("expected 0 cache writes on a hit, got %d", cache.writes[1])
|
||||
}
|
||||
}
|
||||
|
||||
// TestScrapeRanking_CacheMiss verifies that on a cache miss the page is fetched
|
||||
// from the network and the result is written to the cache.
|
||||
func TestScrapeRanking_CacheMiss(t *testing.T) {
|
||||
cache := newMemPageCacher() // empty cache
|
||||
s := New(&stubClient{html: rankingPage1HTML()}, nil, nil, cache)
|
||||
|
||||
entryCh, errCh := s.ScrapeRanking(context.Background(), 1)
|
||||
entries := drainRanking(t, entryCh, errCh)
|
||||
|
||||
if len(entries) != 2 {
|
||||
t.Fatalf("expected 2 entries, got %d", len(entries))
|
||||
}
|
||||
if cache.writes[1] != 1 {
|
||||
t.Errorf("expected 1 cache write on a miss, got %d", cache.writes[1])
|
||||
}
|
||||
if cache.pages[1] == "" {
|
||||
t.Error("expected page 1 to be stored in cache after miss")
|
||||
}
|
||||
}
|
||||
|
||||
// panicOnGetContent is a BrowserClient whose GetContent panics, letting tests
|
||||
// assert that it is never called (i.e. the cache was used instead).
|
||||
type panicOnGetContent struct{}
|
||||
|
||||
func (p *panicOnGetContent) Strategy() browser.Strategy { return browser.StrategyContent }
|
||||
func (p *panicOnGetContent) GetContent(_ context.Context, req browser.ContentRequest) (string, error) {
|
||||
panic(fmt.Sprintf("unexpected GetContent call for URL %s — should have been served from cache", req.URL))
|
||||
}
|
||||
func (p *panicOnGetContent) ScrapePage(_ context.Context, _ browser.ScrapeRequest) (browser.ScrapeResponse, error) {
|
||||
return browser.ScrapeResponse{}, nil
|
||||
}
|
||||
func (p *panicOnGetContent) CDPSession(_ context.Context, _ string, _ browser.CDPSessionFunc) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -30,46 +30,38 @@ const (
|
||||
rankingPath = "/genre-all/sort-popular/status-all/all-novel"
|
||||
)
|
||||
|
||||
// rejectResourceTypes lists Browserless resource types to block on every request.
|
||||
// We keep: document (the page), script (JS renders the DOM), fetch/xhr (JS data calls).
|
||||
// Everything else is safe to drop for HTML-only scraping.
|
||||
var rejectResourceTypes = []string{
|
||||
"cspviolationreport",
|
||||
"eventsource",
|
||||
"fedcm",
|
||||
"font",
|
||||
"image",
|
||||
"manifest",
|
||||
"media",
|
||||
"other",
|
||||
"ping",
|
||||
"signedexchange",
|
||||
"stylesheet",
|
||||
"texttrack",
|
||||
"websocket",
|
||||
// RankingStore is the subset of storage.Store consumed by ScrapeRanking.
|
||||
type RankingStore interface {
|
||||
WriteRankingItem(ctx context.Context, item scraper.RankingItem) error
|
||||
RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error)
|
||||
}
|
||||
|
||||
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
|
||||
// It uses the /content strategy by default (rendered HTML via Browserless).
|
||||
// It uses direct HTTP requests (no headless browser required).
|
||||
type Scraper struct {
|
||||
client browser.BrowserClient
|
||||
urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy)
|
||||
pageCache scraper.RankingPageCacher
|
||||
log *slog.Logger
|
||||
client browser.BrowserClient
|
||||
urlClient browser.BrowserClient // used for chapter list pagination
|
||||
chapterClient browser.BrowserClient // used for chapter text fetching
|
||||
rankingStore RankingStore
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// New returns a new novelfire Scraper.
|
||||
// client is used for content fetching, urlClient is used for URL retrieval (chapter list).
|
||||
// If urlClient is nil, client will be used for both.
|
||||
// pageCache is optional; pass nil to disable ranking page caching.
|
||||
func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, pageCache scraper.RankingPageCacher) *Scraper {
|
||||
// client is used for catalogue/metadata/ranking fetching (direct HTTP).
|
||||
// urlClient is used for chapter list pagination; falls back to client if nil.
|
||||
// chapterClient is used for chapter text fetching; falls back to client if nil.
|
||||
// rankingStore is optional; pass nil to disable freshness checks and per-item persistence.
|
||||
func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, chapterClient browser.BrowserClient, rankingStore RankingStore) *Scraper {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
if urlClient == nil {
|
||||
urlClient = client
|
||||
}
|
||||
return &Scraper{client: client, urlClient: urlClient, pageCache: pageCache, log: log}
|
||||
if chapterClient == nil {
|
||||
chapterClient = client
|
||||
}
|
||||
return &Scraper{client: client, urlClient: urlClient, chapterClient: chapterClient, rankingStore: rankingStore, log: log}
|
||||
}
|
||||
|
||||
// SourceName implements NovelScraper.
|
||||
@@ -97,18 +89,9 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
|
||||
}
|
||||
|
||||
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
|
||||
s.log.Debug("catalogue page fetch starting",
|
||||
"page", page,
|
||||
"payload_url", pageURL,
|
||||
"payload_wait_selector", ".novel-item",
|
||||
"payload_wait_selector_timeout_ms", 5000,
|
||||
)
|
||||
|
||||
html, err := s.client.GetContent(ctx, browser.ContentRequest{
|
||||
URL: pageURL,
|
||||
WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
|
||||
RejectResourceTypes: rejectResourceTypes,
|
||||
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
||||
URL: pageURL,
|
||||
})
|
||||
if err != nil {
|
||||
s.log.Debug("catalogue page fetch failed",
|
||||
@@ -131,24 +114,28 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
|
||||
return
|
||||
}
|
||||
|
||||
// Extract novel cards: <div class="novel-item">
|
||||
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true})
|
||||
// Extract novel cards: <li class="novel-item">
|
||||
// <a href="/book/slug" title="Title">
|
||||
// <figure class="novel-cover"><img data-src="..."></figure>
|
||||
// <h4 class="novel-title text2row">Title</h4>
|
||||
// </a>
|
||||
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
|
||||
if len(cards) == 0 {
|
||||
s.log.Warn("no novel cards found, stopping pagination", "page", page)
|
||||
return
|
||||
}
|
||||
|
||||
for _, card := range cards {
|
||||
// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
|
||||
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"})
|
||||
// The outer <a> carries the href; <h4 class="novel-title"> has the title text.
|
||||
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a", Attr: "href"})
|
||||
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})
|
||||
|
||||
var title, href string
|
||||
if linkNode != nil {
|
||||
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
||||
}
|
||||
if titleNode != nil {
|
||||
linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
|
||||
if linkNode != nil {
|
||||
title = htmlutil.ExtractText(linkNode, scraper.Selector{})
|
||||
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
||||
}
|
||||
title = strings.TrimSpace(htmlutil.ExtractText(titleNode, scraper.Selector{}))
|
||||
}
|
||||
if href == "" || title == "" {
|
||||
continue
|
||||
@@ -162,8 +149,17 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
|
||||
}
|
||||
}
|
||||
|
||||
// Find next page link: <a class="next" href="...">
|
||||
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
|
||||
// Find next page link: <a rel="next" href="..."> (same structure as ranking pages)
|
||||
if !hasNextPageLink(root) {
|
||||
break
|
||||
}
|
||||
nextHref := ""
|
||||
for _, a := range htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) {
|
||||
if htmlutil.AttrVal(a, "rel") == "next" {
|
||||
nextHref = htmlutil.AttrVal(a, "href")
|
||||
break
|
||||
}
|
||||
}
|
||||
if nextHref == "" {
|
||||
break
|
||||
}
|
||||
@@ -178,17 +174,10 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
|
||||
// ─── MetadataProvider ────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) {
|
||||
s.log.Debug("metadata fetch starting",
|
||||
"payload_url", bookURL,
|
||||
"payload_wait_selector", ".novel-title",
|
||||
"payload_wait_selector_timeout_ms", 5000,
|
||||
)
|
||||
s.log.Debug("metadata fetch starting", "url", bookURL)
|
||||
|
||||
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
|
||||
URL: bookURL,
|
||||
WaitFor: &browser.WaitForSelector{Selector: ".novel-title", Timeout: 5000},
|
||||
RejectResourceTypes: rejectResourceTypes,
|
||||
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
||||
URL: bookURL,
|
||||
})
|
||||
if err != nil {
|
||||
s.log.Debug("metadata fetch failed", "url", bookURL, "err", err)
|
||||
@@ -209,6 +198,9 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.B
|
||||
var cover string
|
||||
if figureCover := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); figureCover != nil {
|
||||
cover = htmlutil.ExtractFirst(figureCover, scraper.Selector{Tag: "img", Attr: "src"})
|
||||
if cover != "" && !strings.HasPrefix(cover, "http") {
|
||||
cover = baseURL + cover
|
||||
}
|
||||
}
|
||||
// <span class="status">Ongoing</span>
|
||||
status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"})
|
||||
@@ -272,24 +264,11 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
|
||||
s.log.Debug("chapter list fetch starting",
|
||||
"page", page,
|
||||
"payload_url", pageURL,
|
||||
"payload_wait_selector", ".chapter-list",
|
||||
"payload_wait_selector_timeout_ms", 15000,
|
||||
"payload_wait_timeout_ms", 2000,
|
||||
"strategy", s.urlClient.Strategy(),
|
||||
)
|
||||
|
||||
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
|
||||
URL: pageURL,
|
||||
// Wait up to 15 s for the chapter list container to appear in the DOM.
|
||||
WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000},
|
||||
// After the selector is found, wait an additional 2 s for any
|
||||
// deferred JS rendering (lazy-loaded links, infinite-scroll hydration).
|
||||
WaitForTimeout: 2000,
|
||||
RejectResourceTypes: rejectResourceTypes,
|
||||
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
||||
// Do NOT use BestAttempt — we want a complete page or a clear error,
|
||||
// not silently partial HTML that looks like "no more chapters".
|
||||
BestAttempt: false,
|
||||
})
|
||||
if err != nil {
|
||||
s.log.Debug("chapter list fetch failed",
|
||||
@@ -366,6 +345,57 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
|
||||
return refs, nil
|
||||
}
|
||||
|
||||
// ScrapeChapterListPage fetches and parses a single chapter-list page URL and
|
||||
// returns all chapter refs found on that page without following pagination.
|
||||
// pageURL should be the full URL including query params, e.g.:
|
||||
//
|
||||
// https://novelfire.net/book/shadow-slave/chapters?page=1
|
||||
func (s *Scraper) ScrapeChapterListPage(ctx context.Context, pageURL string) ([]scraper.ChapterRef, error) {
|
||||
s.log.Info("scraping chapter list page (single)", "url", pageURL)
|
||||
|
||||
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
|
||||
URL: pageURL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chapter list page fetch: %w", err)
|
||||
}
|
||||
|
||||
root, err := htmlutil.ParseHTML(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chapter list page parse: %w", err)
|
||||
}
|
||||
|
||||
chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
|
||||
if chapterList == nil {
|
||||
return nil, fmt.Errorf("chapter list container not found in %s", pageURL)
|
||||
}
|
||||
|
||||
items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
|
||||
var refs []scraper.ChapterRef
|
||||
for _, item := range items {
|
||||
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
|
||||
if linkNode == nil {
|
||||
continue
|
||||
}
|
||||
href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
|
||||
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
|
||||
if href == "" {
|
||||
continue
|
||||
}
|
||||
chURL := resolveURL(baseURL, href)
|
||||
num := chapterNumberFromURL(chURL)
|
||||
if num <= 0 {
|
||||
num = len(refs) + 1
|
||||
}
|
||||
refs = append(refs, scraper.ChapterRef{
|
||||
Number: num,
|
||||
Title: strings.TrimSpace(chTitle),
|
||||
URL: chURL,
|
||||
})
|
||||
}
|
||||
return refs, nil
|
||||
}
|
||||
|
||||
// ─── RankingProvider ───────────────────────────────────────────────────────────
|
||||
|
||||
// hasNextPageLink returns true if the HTML document contains a pagination link
|
||||
@@ -388,6 +418,9 @@ func hasNextPageLink(root *html.Node) bool {
|
||||
// listing on novelfire.net (/genre-all/sort-popular/status-all/all-novel).
|
||||
// Pages are fetched one at a time, strictly sequentially.
|
||||
// maxPages <= 0 means "fetch all pages until no more are found".
|
||||
//
|
||||
// If a RankingStore was provided and the stored ranking is fresh (< 24 hours old),
|
||||
// both channels are closed immediately without any network traffic.
|
||||
func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scraper.BookMeta, <-chan error) {
|
||||
entries := make(chan scraper.BookMeta, 32)
|
||||
errs := make(chan error, 16)
|
||||
@@ -396,6 +429,17 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
|
||||
defer close(entries)
|
||||
defer close(errs)
|
||||
|
||||
// Freshness check: skip scraping if data is recent enough.
|
||||
if s.rankingStore != nil {
|
||||
fresh, err := s.rankingStore.RankingFreshEnough(ctx, 24*time.Hour)
|
||||
if err != nil {
|
||||
s.log.Warn("ranking freshness check failed, proceeding with scrape", "err", err)
|
||||
} else if fresh {
|
||||
s.log.Info("ranking data is fresh, skipping scrape")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
rank := 1
|
||||
|
||||
for page := 1; maxPages <= 0 || page <= maxPages; page++ {
|
||||
@@ -407,38 +451,14 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
|
||||
|
||||
pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
|
||||
|
||||
// Try to serve from disk cache before hitting the network.
|
||||
var raw string
|
||||
if s.pageCache != nil {
|
||||
if cached, err := s.pageCache.ReadRankingPageCache(page); err != nil {
|
||||
s.log.Warn("ranking page cache read error", "page", page, "err", err)
|
||||
} else if cached != "" {
|
||||
s.log.Info("serving ranking page from cache", "page", page)
|
||||
raw = cached
|
||||
}
|
||||
}
|
||||
|
||||
if raw == "" {
|
||||
s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)
|
||||
fetched, err := s.client.GetContent(ctx, browser.ContentRequest{
|
||||
URL: pageURL,
|
||||
WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
|
||||
RejectResourceTypes: rejectResourceTypes,
|
||||
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
||||
})
|
||||
if err != nil {
|
||||
s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)
|
||||
errs <- fmt.Errorf("ranking page %d: %w", page, err)
|
||||
return
|
||||
}
|
||||
raw = fetched
|
||||
|
||||
// Persist to cache for future runs.
|
||||
if s.pageCache != nil {
|
||||
if werr := s.pageCache.WriteRankingPageCache(page, raw); werr != nil {
|
||||
s.log.Warn("ranking page cache write error", "page", page, "err", werr)
|
||||
}
|
||||
}
|
||||
s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)
|
||||
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
|
||||
URL: pageURL,
|
||||
})
|
||||
if err != nil {
|
||||
s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)
|
||||
errs <- fmt.Errorf("ranking page %d: %w", page, err)
|
||||
return
|
||||
}
|
||||
|
||||
root, err := htmlutil.ParseHTML(raw)
|
||||
@@ -497,10 +517,10 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
|
||||
}
|
||||
}
|
||||
|
||||
slug := slugFromURL(bookURL)
|
||||
bookSlug := slugFromURL(bookURL)
|
||||
|
||||
meta := scraper.BookMeta{
|
||||
Slug: slug,
|
||||
Slug: bookSlug,
|
||||
Title: title,
|
||||
Cover: cover,
|
||||
SourceURL: bookURL,
|
||||
@@ -508,6 +528,20 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
|
||||
}
|
||||
rank++
|
||||
|
||||
// Persist item to store immediately.
|
||||
if s.rankingStore != nil {
|
||||
item := scraper.RankingItem{
|
||||
Rank: meta.Ranking,
|
||||
Slug: meta.Slug,
|
||||
Title: meta.Title,
|
||||
Cover: meta.Cover,
|
||||
SourceURL: meta.SourceURL,
|
||||
}
|
||||
if werr := s.rankingStore.WriteRankingItem(ctx, item); werr != nil {
|
||||
s.log.Warn("ranking item write failed", "slug", meta.Slug, "err", werr)
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
@@ -583,12 +617,8 @@ func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef)
|
||||
"payload_wait_selector_timeout_ms", 5000,
|
||||
)
|
||||
|
||||
raw, err := retryGetContent(ctx, s.log, s.client, browser.ContentRequest{
|
||||
URL: ref.URL,
|
||||
WaitFor: &browser.WaitForSelector{Selector: "#content", Timeout: 5000},
|
||||
RejectResourceTypes: rejectResourceTypes,
|
||||
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
||||
BestAttempt: true,
|
||||
raw, err := retryGetContent(ctx, s.log, s.chapterClient, browser.ContentRequest{
|
||||
URL: ref.URL,
|
||||
}, 9, 6*time.Second)
|
||||
if err != nil {
|
||||
s.log.Debug("chapter text fetch failed",
|
||||
@@ -643,20 +673,8 @@ func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef)
|
||||
|
||||
// ─── helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func resolveURL(base, href string) string {
|
||||
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||||
return href
|
||||
}
|
||||
b, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
ref, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
return b.ResolveReference(ref).String()
|
||||
}
|
||||
// resolveURL is a thin alias over htmlutil.ResolveURL kept for readability.
|
||||
func resolveURL(base, href string) string { return htmlutil.ResolveURL(base, href) }
|
||||
|
||||
func slugFromURL(bookURL string) string {
|
||||
u, err := url.Parse(bookURL)
|
||||
|
||||
@@ -62,12 +62,12 @@ func (c *pagedStubClient) CDPSession(_ context.Context, _ string, _ browser.CDPS
|
||||
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func newScraper(html string) *Scraper {
|
||||
return New(&stubClient{html: html}, nil, &stubClient{html: html}, nil)
|
||||
return New(&stubClient{html: html}, nil, &stubClient{html: html}, nil, nil)
|
||||
}
|
||||
|
||||
func newPagedScraper(pages ...string) *Scraper {
|
||||
urlClient := &pagedStubClient{pages: pages}
|
||||
return New(&stubClient{}, nil, urlClient, nil)
|
||||
return New(&stubClient{}, nil, urlClient, nil, nil)
|
||||
}
|
||||
|
||||
// ── ScrapeChapterText ─────────────────────────────────────────────────────────
|
||||
@@ -141,6 +141,84 @@ func TestChapterNumberFromURL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// ── ScrapeMetadata ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestScrapeMetadata_ParsesFields(t *testing.T) {
|
||||
html := `<!DOCTYPE html><html><body>
|
||||
<h1 class="novel-title">The Iron Throne</h1>
|
||||
<span class="author"><a>Jane Doe</a></span>
|
||||
<figure class="cover"><img src="https://cdn.example.com/cover.jpg"></figure>
|
||||
<span class="status">Ongoing</span>
|
||||
<div class="genres"><a>Fantasy</a><a>Action</a></div>
|
||||
<div class="summary"><p>A sweeping epic set in a magical world.</p></div>
|
||||
<span class="chapter-count">42 Chapters</span>
|
||||
</body></html>`
|
||||
|
||||
s := newScraper(html)
|
||||
meta, err := s.ScrapeMetadata(context.Background(), "https://novelfire.net/book/the-iron-throne")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if meta.Slug != "the-iron-throne" {
|
||||
t.Errorf("Slug = %q, want %q", meta.Slug, "the-iron-throne")
|
||||
}
|
||||
if meta.Title != "The Iron Throne" {
|
||||
t.Errorf("Title = %q, want %q", meta.Title, "The Iron Throne")
|
||||
}
|
||||
if meta.Author != "Jane Doe" {
|
||||
t.Errorf("Author = %q, want %q", meta.Author, "Jane Doe")
|
||||
}
|
||||
if meta.Cover != "https://cdn.example.com/cover.jpg" {
|
||||
t.Errorf("Cover = %q, want %q", meta.Cover, "https://cdn.example.com/cover.jpg")
|
||||
}
|
||||
if meta.Status != "Ongoing" {
|
||||
t.Errorf("Status = %q, want %q", meta.Status, "Ongoing")
|
||||
}
|
||||
if len(meta.Genres) != 2 || meta.Genres[0] != "Fantasy" || meta.Genres[1] != "Action" {
|
||||
t.Errorf("Genres = %v, want [Fantasy Action]", meta.Genres)
|
||||
}
|
||||
if !strings.Contains(meta.Summary, "sweeping epic") {
|
||||
t.Errorf("Summary = %q, want it to contain 'sweeping epic'", meta.Summary)
|
||||
}
|
||||
if meta.TotalChapters != 42 {
|
||||
t.Errorf("TotalChapters = %d, want 42", meta.TotalChapters)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrapeMetadata_RelativeCoverURL(t *testing.T) {
|
||||
html := `<!DOCTYPE html><html><body>
|
||||
<h1 class="novel-title">Relative Cover</h1>
|
||||
<figure class="cover"><img src="/images/cover.jpg"></figure>
|
||||
</body></html>`
|
||||
|
||||
s := newScraper(html)
|
||||
meta, err := s.ScrapeMetadata(context.Background(), "https://novelfire.net/book/relative-cover")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
// Relative cover URL should be resolved against the base domain.
|
||||
if !strings.HasPrefix(meta.Cover, "https://novelfire.net") {
|
||||
t.Errorf("Cover = %q, expected it to be resolved to an absolute URL", meta.Cover)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrapeMetadata_MissingFields(t *testing.T) {
|
||||
// Minimal page — everything absent; should succeed without panicking.
|
||||
html := `<!DOCTYPE html><html><body></body></html>`
|
||||
|
||||
s := newScraper(html)
|
||||
meta, err := s.ScrapeMetadata(context.Background(), "https://novelfire.net/book/empty-novel")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if meta.Slug != "empty-novel" {
|
||||
t.Errorf("Slug = %q, want %q", meta.Slug, "empty-novel")
|
||||
}
|
||||
if meta.TotalChapters != 0 {
|
||||
t.Errorf("TotalChapters = %d, want 0 for missing chapter-count", meta.TotalChapters)
|
||||
}
|
||||
}
|
||||
|
||||
// ── ScrapeChapterList (position vs URL numbering) ─────────────────────────────
|
||||
|
||||
// TestScrapeChapterList_NumbersFromURL verifies that when the chapter list HTML
|
||||
|
||||
@@ -17,36 +17,58 @@ import (
|
||||
"log/slog"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// Progress is a snapshot of counters at a point in time.
|
||||
type Progress struct {
|
||||
BooksFound int
|
||||
ChaptersScraped int
|
||||
ChaptersSkipped int
|
||||
Errors int
|
||||
}
|
||||
|
||||
// Config holds tunable parameters for the orchestrator.
|
||||
type Config struct {
|
||||
// Workers is the number of goroutines used to scrape chapters in parallel.
|
||||
// Defaults to runtime.NumCPU() when 0.
|
||||
Workers int
|
||||
|
||||
// StaticRoot is the path to the static/books output directory.
|
||||
// StaticRoot is kept for backwards-compatibility but is no longer used
|
||||
// when a Store is provided.
|
||||
StaticRoot string
|
||||
|
||||
// SingleBookURL when non-empty causes the orchestrator to scrape only
|
||||
// that one book instead of walking the full catalogue.
|
||||
SingleBookURL string
|
||||
|
||||
// FromChapter, when > 0, skips chapters with number < FromChapter.
|
||||
// Only effective in single-book mode.
|
||||
FromChapter int
|
||||
|
||||
// ToChapter, when > 0, skips chapters with number > ToChapter.
|
||||
// Only effective in single-book mode. 0 means "no upper limit".
|
||||
ToChapter int
|
||||
|
||||
// OnProgress is called periodically with the current progress counters.
|
||||
// It is always called on completion (success or failure). May be nil.
|
||||
OnProgress func(p Progress)
|
||||
}
|
||||
|
||||
// Orchestrator coordinates the full scrape pipeline.
|
||||
type Orchestrator struct {
|
||||
cfg Config
|
||||
novel scraper.NovelScraper
|
||||
writer *writer.Writer
|
||||
store storage.Store
|
||||
log *slog.Logger
|
||||
workers int
|
||||
}
|
||||
|
||||
// New returns a new Orchestrator.
|
||||
func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator {
|
||||
// New returns a new Orchestrator backed by the provided Store.
|
||||
func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store) *Orchestrator {
|
||||
workers := cfg.Workers
|
||||
if workers <= 0 {
|
||||
workers = runtime.NumCPU()
|
||||
@@ -54,7 +76,7 @@ func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator
|
||||
return &Orchestrator{
|
||||
cfg: cfg,
|
||||
novel: novel,
|
||||
writer: writer.New(cfg.StaticRoot),
|
||||
store: store,
|
||||
log: log,
|
||||
workers: workers,
|
||||
}
|
||||
@@ -66,9 +88,31 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
o.log.Info("orchestrator starting",
|
||||
"source", o.novel.SourceName(),
|
||||
"workers", o.workers,
|
||||
"static_root", o.cfg.StaticRoot,
|
||||
)
|
||||
|
||||
// Atomic counters updated by concurrent goroutines.
|
||||
var (
|
||||
booksFound atomic.Int64
|
||||
chaptersScraped atomic.Int64
|
||||
chaptersSkipped atomic.Int64
|
||||
errors atomic.Int64
|
||||
)
|
||||
|
||||
snapshot := func() Progress {
|
||||
return Progress{
|
||||
BooksFound: int(booksFound.Load()),
|
||||
ChaptersScraped: int(chaptersScraped.Load()),
|
||||
ChaptersSkipped: int(chaptersSkipped.Load()),
|
||||
Errors: int(errors.Load()),
|
||||
}
|
||||
}
|
||||
|
||||
notify := func() {
|
||||
if o.cfg.OnProgress != nil {
|
||||
o.cfg.OnProgress(snapshot())
|
||||
}
|
||||
}
|
||||
|
||||
// chapterWork is the shared queue consumed by chapter worker goroutines.
|
||||
type chapterJob struct {
|
||||
slug string
|
||||
@@ -89,10 +133,12 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
default:
|
||||
}
|
||||
|
||||
// Skip if already on disk.
|
||||
if o.writer.ChapterExists(job.slug, job.ref) {
|
||||
// Skip if already stored.
|
||||
if o.store.ChapterExists(ctx, job.slug, job.ref) {
|
||||
o.log.Debug("chapter already exists, skipping",
|
||||
"book", job.slug, "chapter", job.ref.Number)
|
||||
chaptersSkipped.Add(1)
|
||||
notify()
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -104,18 +150,24 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
"url", job.ref.URL,
|
||||
"err", err,
|
||||
)
|
||||
errors.Add(1)
|
||||
notify()
|
||||
continue
|
||||
}
|
||||
|
||||
if err := o.writer.WriteChapter(job.slug, chapter); err != nil {
|
||||
if err := o.store.WriteChapter(ctx, job.slug, chapter); err != nil {
|
||||
o.log.Error("chapter write failed",
|
||||
"book", job.slug,
|
||||
"chapter", job.ref.Number,
|
||||
"err", err,
|
||||
)
|
||||
errors.Add(1)
|
||||
notify()
|
||||
continue
|
||||
}
|
||||
|
||||
chaptersScraped.Add(1)
|
||||
notify()
|
||||
o.log.Info("chapter saved",
|
||||
"book", job.slug,
|
||||
"chapter", job.ref.Number,
|
||||
@@ -132,21 +184,27 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
meta, err := o.novel.ScrapeMetadata(ctx, bookURL)
|
||||
if err != nil {
|
||||
o.log.Error("metadata scrape failed", "url", bookURL, "err", err)
|
||||
errors.Add(1)
|
||||
notify()
|
||||
return
|
||||
}
|
||||
|
||||
// Persist / update metadata.yaml.
|
||||
if err := o.writer.WriteMetadata(meta); err != nil {
|
||||
// Persist / update metadata.
|
||||
if err := o.store.WriteMetadata(ctx, meta); err != nil {
|
||||
o.log.Error("metadata write failed", "slug", meta.Slug, "err", err)
|
||||
// Continue — chapters can still be scraped.
|
||||
}
|
||||
|
||||
booksFound.Add(1)
|
||||
notify()
|
||||
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
|
||||
|
||||
// Fetch chapter list.
|
||||
refs, err := o.novel.ScrapeChapterList(ctx, bookURL)
|
||||
if err != nil {
|
||||
o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err)
|
||||
errors.Add(1)
|
||||
notify()
|
||||
return
|
||||
}
|
||||
|
||||
@@ -154,6 +212,15 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
|
||||
// Enqueue chapter jobs.
|
||||
for _, ref := range refs {
|
||||
// Apply chapter range filter (only in single-book mode when set).
|
||||
if o.cfg.FromChapter > 0 && ref.Number < o.cfg.FromChapter {
|
||||
chaptersSkipped.Add(1)
|
||||
continue
|
||||
}
|
||||
if o.cfg.ToChapter > 0 && ref.Number > o.cfg.ToChapter {
|
||||
chaptersSkipped.Add(1)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
@@ -174,14 +241,17 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
go func() {
|
||||
for err := range catErrs {
|
||||
o.log.Error("catalogue error", "err", err)
|
||||
errors.Add(1)
|
||||
notify()
|
||||
}
|
||||
}()
|
||||
|
||||
var bookWG sync.WaitGroup
|
||||
bookLoop:
|
||||
for entry := range entries {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break
|
||||
break bookLoop
|
||||
default:
|
||||
}
|
||||
|
||||
@@ -204,6 +274,9 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
// Wait for all in-flight chapter scrapes to finish.
|
||||
chapterWG.Wait()
|
||||
|
||||
// Final progress notification.
|
||||
notify()
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return fmt.Errorf("orchestrator: context cancelled: %w", ctx.Err())
|
||||
}
|
||||
|
||||
336
scraper/internal/orchestrator/orchestrator_test.go
Normal file
336
scraper/internal/orchestrator/orchestrator_test.go
Normal file
@@ -0,0 +1,336 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
"io"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// ── mock NovelScraper ─────────────────────────────────────────────────────────
|
||||
|
||||
type mockScraper struct {
|
||||
catalogue []scraper.CatalogueEntry
|
||||
meta scraper.BookMeta
|
||||
metaErr error
|
||||
chapters []scraper.ChapterRef
|
||||
chapterTextFn func(ref scraper.ChapterRef) (scraper.Chapter, error)
|
||||
}
|
||||
|
||||
func (m *mockScraper) SourceName() string { return "mock" }
|
||||
|
||||
func (m *mockScraper) ScrapeCatalogue(_ context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
|
||||
entries := make(chan scraper.CatalogueEntry, len(m.catalogue))
|
||||
errs := make(chan error, 1)
|
||||
for _, e := range m.catalogue {
|
||||
entries <- e
|
||||
}
|
||||
close(entries)
|
||||
close(errs)
|
||||
return entries, errs
|
||||
}
|
||||
|
||||
func (m *mockScraper) ScrapeMetadata(_ context.Context, _ string) (scraper.BookMeta, error) {
|
||||
return m.meta, m.metaErr
|
||||
}
|
||||
|
||||
func (m *mockScraper) ScrapeChapterList(_ context.Context, _ string) ([]scraper.ChapterRef, error) {
|
||||
return m.chapters, nil
|
||||
}
|
||||
|
||||
func (m *mockScraper) ScrapeChapterText(_ context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
|
||||
if m.chapterTextFn != nil {
|
||||
return m.chapterTextFn(ref)
|
||||
}
|
||||
return scraper.Chapter{Ref: ref, Text: "stub text"}, nil
|
||||
}
|
||||
|
||||
func (m *mockScraper) ScrapeRanking(_ context.Context, _ int) (<-chan scraper.BookMeta, <-chan error) {
|
||||
ch := make(chan scraper.BookMeta)
|
||||
errs := make(chan error)
|
||||
close(ch)
|
||||
close(errs)
|
||||
return ch, errs
|
||||
}
|
||||
|
||||
// ── mock Store ────────────────────────────────────────────────────────────────
|
||||
|
||||
// mockStore records which methods were called; only implements what the
|
||||
// orchestrator touches. All other methods panic so unexpected calls surface
|
||||
// as test failures rather than silent no-ops.
|
||||
type mockStore struct {
|
||||
mu sync.Mutex
|
||||
writtenMeta []scraper.BookMeta
|
||||
writtenChapters []scraper.Chapter
|
||||
existingSlugs map[string]map[int]bool // slug → chapterNum → exists
|
||||
}
|
||||
|
||||
func newMockStore() *mockStore {
|
||||
return &mockStore{existingSlugs: make(map[string]map[int]bool)}
|
||||
}
|
||||
|
||||
func (s *mockStore) ChapterExists(_ context.Context, slug string, ref scraper.ChapterRef) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if m, ok := s.existingSlugs[slug]; ok {
|
||||
return m[ref.Number]
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *mockStore) WriteChapter(_ context.Context, slug string, ch scraper.Chapter) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.writtenChapters = append(s.writtenChapters, ch)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *mockStore) WriteChapterRefs(_ context.Context, _ string, _ []scraper.ChapterRef) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *mockStore) WriteMetadata(_ context.Context, meta scraper.BookMeta) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.writtenMeta = append(s.writtenMeta, meta)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unimplemented Store methods — panic so accidental calls surface immediately.
|
||||
func (s *mockStore) ReadMetadata(_ context.Context, _ string) (scraper.BookMeta, bool, error) {
|
||||
panic("ReadMetadata not expected")
|
||||
}
|
||||
func (s *mockStore) ListBooks(_ context.Context) ([]scraper.BookMeta, error) {
|
||||
panic("ListBooks not expected")
|
||||
}
|
||||
func (s *mockStore) LocalSlugs(_ context.Context) (map[string]bool, error) {
|
||||
panic("LocalSlugs not expected")
|
||||
}
|
||||
func (s *mockStore) MetadataMtime(_ context.Context, _ string) int64 { return 0 }
|
||||
func (s *mockStore) ReadChapter(_ context.Context, _ string, _ int) (string, error) {
|
||||
panic("ReadChapter not expected")
|
||||
}
|
||||
func (s *mockStore) ListChapters(_ context.Context, _ string) ([]storage.ChapterInfo, error) {
|
||||
panic("ListChapters not expected")
|
||||
}
|
||||
func (s *mockStore) CountChapters(_ context.Context, _ string) int { return 0 }
|
||||
func (s *mockStore) ReindexChapters(_ context.Context, _ string) (int, error) {
|
||||
panic("ReindexChapters not expected")
|
||||
}
|
||||
func (s *mockStore) WriteRankingItem(_ context.Context, _ storage.RankingItem) error { return nil }
|
||||
func (s *mockStore) ReadRankingItems(_ context.Context) ([]storage.RankingItem, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *mockStore) RankingFreshEnough(_ context.Context, _ time.Duration) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
func (s *mockStore) GetAudioCache(_ context.Context, _ string) (string, bool) { return "", false }
|
||||
func (s *mockStore) SetAudioCache(_ context.Context, _, _ string) error { return nil }
|
||||
func (s *mockStore) PutAudio(_ context.Context, _ string, _ []byte) error { return nil }
|
||||
func (s *mockStore) GetProgress(_ context.Context, _, _ string) (storage.ReadingProgress, bool) {
|
||||
return storage.ReadingProgress{}, false
|
||||
}
|
||||
func (s *mockStore) SetProgress(_ context.Context, _ string, _ storage.ReadingProgress) error {
|
||||
return nil
|
||||
}
|
||||
func (s *mockStore) AllProgress(_ context.Context, _ string) ([]storage.ReadingProgress, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *mockStore) DeleteProgress(_ context.Context, _, _ string) error { return nil }
|
||||
func (s *mockStore) AudioObjectKey(_ string, _ int, _ string) string { return "" }
|
||||
|
||||
func (s *mockStore) AudioExists(_ context.Context, _ string) bool { return false }
|
||||
func (s *mockStore) PresignChapter(_ context.Context, _ string, _ int, _ time.Duration) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
func (s *mockStore) PresignAudio(_ context.Context, _ string, _ time.Duration) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
func (s *mockStore) PresignAvatarUpload(_ context.Context, _, _ string) (string, string, error) {
|
||||
return "", "", nil
|
||||
}
|
||||
func (s *mockStore) PresignAvatarURL(_ context.Context, _ string) (string, bool, error) {
|
||||
return "", false, nil
|
||||
}
|
||||
func (s *mockStore) DeleteAvatar(_ context.Context, _ string) error { return nil }
|
||||
func (s *mockStore) SaveBrowsePage(_ context.Context, _, _ string) error { return nil }
|
||||
func (s *mockStore) GetBrowsePage(_ context.Context, _ string) (string, bool, error) {
|
||||
return "", false, nil
|
||||
}
|
||||
func (s *mockStore) BrowseHTMLKey(_ string, _ int) string { return "" }
|
||||
func (s *mockStore) BrowseFilteredHTMLKey(_ string, _ int, _, _, _ string) string { return "" }
|
||||
func (s *mockStore) BrowseCoverKey(_, _ string) string { return "" }
|
||||
func (s *mockStore) SaveBrowseAsset(_ context.Context, _ string, _ []byte, _ string) error {
|
||||
return nil
|
||||
}
|
||||
func (s *mockStore) GetBrowseAsset(_ context.Context, _ string) ([]byte, string, bool, error) {
|
||||
return nil, "", false, nil
|
||||
}
|
||||
func (s *mockStore) CreateScrapeTask(_ context.Context, _, _ string) (string, error) {
|
||||
return "task-id", nil
|
||||
}
|
||||
func (s *mockStore) UpdateScrapeTask(_ context.Context, _ string, _ storage.ScrapeTaskUpdate) error {
|
||||
return nil
|
||||
}
|
||||
func (s *mockStore) ListScrapeTasks(_ context.Context) ([]storage.ScrapeTask, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *mockStore) CreateAudioJob(_ context.Context, _ string, _ int, _ string) (string, error) {
|
||||
return "audio-job-id", nil
|
||||
}
|
||||
func (s *mockStore) UpdateAudioJob(_ context.Context, _, _, _ string, _ time.Time) error {
|
||||
return nil
|
||||
}
|
||||
func (s *mockStore) GetAudioJob(_ context.Context, _ string) (storage.AudioJob, bool, error) {
|
||||
return storage.AudioJob{}, false, nil
|
||||
}
|
||||
func (s *mockStore) ListAudioJobs(_ context.Context) ([]storage.AudioJob, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func discardLogger() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
}
|
||||
|
||||
// ── tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
// TestRun_SingleBook verifies the happy-path single-book scrape: metadata is
|
||||
// persisted and all chapters are written to the store.
|
||||
func TestRun_SingleBook(t *testing.T) {
|
||||
novel := &mockScraper{
|
||||
meta: scraper.BookMeta{Slug: "the-iron-throne", Title: "The Iron Throne"},
|
||||
chapters: []scraper.ChapterRef{
|
||||
{Number: 1, Title: "Chapter 1", URL: "https://example.com/book/ch-1"},
|
||||
{Number: 2, Title: "Chapter 2", URL: "https://example.com/book/ch-2"},
|
||||
{Number: 3, Title: "Chapter 3", URL: "https://example.com/book/ch-3"},
|
||||
},
|
||||
}
|
||||
store := newMockStore()
|
||||
|
||||
o := New(Config{Workers: 2, SingleBookURL: "https://example.com/book/the-iron-throne"}, novel, discardLogger(), store)
|
||||
if err := o.Run(context.Background()); err != nil {
|
||||
t.Fatalf("Run() returned error: %v", err)
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
if len(store.writtenMeta) != 1 {
|
||||
t.Errorf("writtenMeta count = %d, want 1", len(store.writtenMeta))
|
||||
}
|
||||
if len(store.writtenChapters) != 3 {
|
||||
t.Errorf("writtenChapters count = %d, want 3", len(store.writtenChapters))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_SingleBook_SkipsExistingChapters verifies that chapters already in
|
||||
// the store are not re-scraped.
|
||||
func TestRun_SingleBook_SkipsExistingChapters(t *testing.T) {
|
||||
novel := &mockScraper{
|
||||
meta: scraper.BookMeta{Slug: "test-novel", Title: "Test Novel"},
|
||||
chapters: []scraper.ChapterRef{
|
||||
{Number: 1, Title: "Chapter 1"},
|
||||
{Number: 2, Title: "Chapter 2"},
|
||||
},
|
||||
}
|
||||
store := newMockStore()
|
||||
// Mark chapter 1 as already existing.
|
||||
store.existingSlugs["test-novel"] = map[int]bool{1: true}
|
||||
|
||||
o := New(Config{Workers: 1, SingleBookURL: "https://example.com/book/test-novel"}, novel, discardLogger(), store)
|
||||
if err := o.Run(context.Background()); err != nil {
|
||||
t.Fatalf("Run() returned error: %v", err)
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
// Only chapter 2 should have been written; chapter 1 was skipped.
|
||||
if len(store.writtenChapters) != 1 {
|
||||
t.Errorf("writtenChapters count = %d, want 1 (skipped ch1)", len(store.writtenChapters))
|
||||
}
|
||||
if store.writtenChapters[0].Ref.Number != 2 {
|
||||
t.Errorf("expected chapter 2 to be written, got chapter %d", store.writtenChapters[0].Ref.Number)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_CatalogueMode verifies that catalogue mode processes all books.
|
||||
func TestRun_CatalogueMode(t *testing.T) {
|
||||
novel := &mockScraper{
|
||||
catalogue: []scraper.CatalogueEntry{
|
||||
{Title: "Book A", URL: "https://example.com/book/a"},
|
||||
{Title: "Book B", URL: "https://example.com/book/b"},
|
||||
},
|
||||
meta: scraper.BookMeta{Slug: "book-slug", Title: "A Book"},
|
||||
chapters: []scraper.ChapterRef{{Number: 1, Title: "Chapter 1"}},
|
||||
}
|
||||
store := newMockStore()
|
||||
|
||||
o := New(Config{Workers: 2}, novel, discardLogger(), store)
|
||||
if err := o.Run(context.Background()); err != nil {
|
||||
t.Fatalf("Run() returned error: %v", err)
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
// 2 books → 2 metadata writes, 2 chapter writes (one chapter per book).
|
||||
if len(store.writtenMeta) != 2 {
|
||||
t.Errorf("writtenMeta count = %d, want 2", len(store.writtenMeta))
|
||||
}
|
||||
if len(store.writtenChapters) != 2 {
|
||||
t.Errorf("writtenChapters count = %d, want 2", len(store.writtenChapters))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_OnProgress_Called verifies that the OnProgress callback fires at
|
||||
// least once upon completion.
|
||||
func TestRun_OnProgress_Called(t *testing.T) {
|
||||
novel := &mockScraper{
|
||||
meta: scraper.BookMeta{Slug: "progress-book", Title: "Progress Book"},
|
||||
chapters: []scraper.ChapterRef{{Number: 1, Title: "Chapter 1"}},
|
||||
}
|
||||
store := newMockStore()
|
||||
|
||||
var callCount int
|
||||
o := New(Config{
|
||||
Workers: 1,
|
||||
SingleBookURL: "https://example.com/book/progress-book",
|
||||
OnProgress: func(_ Progress) {
|
||||
callCount++
|
||||
},
|
||||
}, novel, discardLogger(), store)
|
||||
|
||||
if err := o.Run(context.Background()); err != nil {
|
||||
t.Fatalf("Run() returned error: %v", err)
|
||||
}
|
||||
if callCount == 0 {
|
||||
t.Error("OnProgress was never called")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_ContextCancelled verifies that Run returns a non-nil error when the
|
||||
// context is cancelled before work completes.
|
||||
func TestRun_ContextCancelled(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel immediately
|
||||
|
||||
novel := &mockScraper{
|
||||
meta: scraper.BookMeta{Slug: "cancel-book", Title: "Cancel Book"},
|
||||
chapters: []scraper.ChapterRef{{Number: 1}},
|
||||
}
|
||||
store := newMockStore()
|
||||
|
||||
o := New(Config{Workers: 1, SingleBookURL: "https://example.com/book/cancel-book"}, novel, discardLogger(), store)
|
||||
err := o.Run(ctx)
|
||||
if err == nil {
|
||||
t.Error("expected non-nil error when context is cancelled, got nil")
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
package htmlutil
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
@@ -10,6 +11,24 @@ import (
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// ResolveURL returns an absolute URL. If href is already absolute it is
|
||||
// returned unchanged. Otherwise it is resolved against base using standard
|
||||
// URL resolution (handles relative paths, absolute paths, etc.).
|
||||
func ResolveURL(base, href string) string {
|
||||
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||||
return href
|
||||
}
|
||||
b, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
ref, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
return b.ResolveReference(ref).String()
|
||||
}
|
||||
|
||||
// ParseHTML parses raw HTML and returns the root node.
|
||||
func ParseHTML(raw string) (*html.Node, error) {
|
||||
return html.Parse(strings.NewReader(raw))
|
||||
@@ -48,8 +67,8 @@ matched:
|
||||
return true
|
||||
}
|
||||
|
||||
// attrVal returns the value of attribute key from node n.
|
||||
func attrVal(n *html.Node, key string) string {
|
||||
// AttrVal returns the value of attribute key from node n.
|
||||
func AttrVal(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return a.Val
|
||||
@@ -58,8 +77,11 @@ func attrVal(n *html.Node, key string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// textContent returns the concatenated text content of all descendant text nodes.
|
||||
func textContent(n *html.Node) string {
|
||||
// attrVal is an unexported alias kept for internal use within this package.
|
||||
func attrVal(n *html.Node, key string) string { return AttrVal(n, key) }
|
||||
|
||||
// TextContent returns the concatenated text content of all descendant text nodes.
|
||||
func TextContent(n *html.Node) string {
|
||||
var sb strings.Builder
|
||||
var walk func(*html.Node)
|
||||
walk = func(cur *html.Node) {
|
||||
@@ -74,6 +96,9 @@ func textContent(n *html.Node) string {
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// textContent is an unexported alias kept for internal use within this package.
|
||||
func textContent(n *html.Node) string { return TextContent(n) }
|
||||
|
||||
// FindFirst returns the first node matching sel within root.
|
||||
func FindFirst(root *html.Node, sel scraper.Selector) *html.Node {
|
||||
var found *html.Node
|
||||
|
||||
221
scraper/internal/scraper/htmlutil/htmlutil_test.go
Normal file
221
scraper/internal/scraper/htmlutil/htmlutil_test.go
Normal file
@@ -0,0 +1,221 @@
|
||||
package htmlutil
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// ── ResolveURL ────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestResolveURL(t *testing.T) {
|
||||
cases := []struct{ base, href, want string }{
|
||||
// Already absolute → unchanged.
|
||||
{"https://example.com", "https://other.com/page", "https://other.com/page"},
|
||||
{"https://example.com", "http://other.com/page", "http://other.com/page"},
|
||||
// Absolute path.
|
||||
{"https://example.com", "/book/slug", "https://example.com/book/slug"},
|
||||
// Relative path.
|
||||
{"https://example.com/genre/all", "page?p=2", "https://example.com/genre/page?p=2"},
|
||||
// Empty href → base itself.
|
||||
{"https://example.com", "", "https://example.com"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := ResolveURL(c.base, c.href)
|
||||
if got != c.want {
|
||||
t.Errorf("ResolveURL(%q, %q) = %q, want %q", c.base, c.href, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── AttrVal ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestAttrVal(t *testing.T) {
|
||||
root, err := ParseHTML(`<html><body><a href="/book/slug" class="link">text</a></body></html>`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := FindFirst(root, scraper.Selector{Tag: "a"})
|
||||
if a == nil {
|
||||
t.Fatal("expected to find <a>")
|
||||
}
|
||||
if got := AttrVal(a, "href"); got != "/book/slug" {
|
||||
t.Errorf("AttrVal href = %q, want %q", got, "/book/slug")
|
||||
}
|
||||
if got := AttrVal(a, "class"); got != "link" {
|
||||
t.Errorf("AttrVal class = %q, want %q", got, "link")
|
||||
}
|
||||
if got := AttrVal(a, "missing"); got != "" {
|
||||
t.Errorf("AttrVal missing = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ── TextContent ───────────────────────────────────────────────────────────────
|
||||
|
||||
func TestTextContent(t *testing.T) {
|
||||
root, err := ParseHTML(`<html><body><p>Hello <b>world</b></p></body></html>`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := FindFirst(root, scraper.Selector{Tag: "p"})
|
||||
if p == nil {
|
||||
t.Fatal("expected to find <p>")
|
||||
}
|
||||
if got := TextContent(p); got != "Hello world" {
|
||||
t.Errorf("TextContent = %q, want %q", got, "Hello world")
|
||||
}
|
||||
}
|
||||
|
||||
// ── FindFirst / FindAll ───────────────────────────────────────────────────────
|
||||
|
||||
func TestFindFirst_ByTag(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><h1>Title</h1><h2>Sub</h2></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{Tag: "h1"})
|
||||
if n == nil {
|
||||
t.Fatal("expected to find <h1>")
|
||||
}
|
||||
if TextContent(n) != "Title" {
|
||||
t.Errorf("h1 text = %q, want %q", TextContent(n), "Title")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFirst_ByClass(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><span class="author foo">JR</span></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{Tag: "span", Class: "author"})
|
||||
if n == nil {
|
||||
t.Fatal("expected to find span.author")
|
||||
}
|
||||
if TextContent(n) != "JR" {
|
||||
t.Errorf("author text = %q, want %q", TextContent(n), "JR")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFirst_ByID(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content"><p>text</p></div></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
if n == nil {
|
||||
t.Fatal("expected to find #content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFirst_NoMatch(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><p>nothing</p></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{Tag: "h1"})
|
||||
if n != nil {
|
||||
t.Errorf("expected nil for missing tag, got %v", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindAll_Multiple(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body>
|
||||
<li class="novel-item">A</li>
|
||||
<li class="novel-item">B</li>
|
||||
<li class="other">C</li>
|
||||
</body></html>`)
|
||||
nodes := FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item"})
|
||||
if len(nodes) != 2 {
|
||||
t.Errorf("FindAll novel-item = %d, want 2", len(nodes))
|
||||
}
|
||||
}
|
||||
|
||||
// ── ExtractFirst / ExtractAll ─────────────────────────────────────────────────
|
||||
|
||||
func TestExtractFirst_TextNode(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><h1 class="novel-title">Shadow Slave</h1></body></html>`)
|
||||
got := ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
|
||||
if got != "Shadow Slave" {
|
||||
t.Errorf("ExtractFirst title = %q, want %q", got, "Shadow Slave")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFirst_AttrNode(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><img src="/covers/slug.jpg"></body></html>`)
|
||||
got := ExtractFirst(root, scraper.Selector{Tag: "img", Attr: "src"})
|
||||
if got != "/covers/slug.jpg" {
|
||||
t.Errorf("ExtractFirst img src = %q, want %q", got, "/covers/slug.jpg")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFirst_Missing(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body></body></html>`)
|
||||
got := ExtractFirst(root, scraper.Selector{Tag: "h1"})
|
||||
if got != "" {
|
||||
t.Errorf("ExtractFirst missing = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractAll_Genres(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body>
|
||||
<div class="genres">
|
||||
<a href="/genre/action">Action</a>
|
||||
<a href="/genre/fantasy">Fantasy</a>
|
||||
</div>
|
||||
</body></html>`)
|
||||
genresNode := FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
|
||||
if genresNode == nil {
|
||||
t.Fatal("expected genres div")
|
||||
}
|
||||
genres := ExtractAll(genresNode, scraper.Selector{Tag: "a"})
|
||||
if len(genres) != 2 {
|
||||
t.Fatalf("genres = %v, want 2", genres)
|
||||
}
|
||||
if genres[0] != "Action" || genres[1] != "Fantasy" {
|
||||
t.Errorf("genres = %v, want [Action Fantasy]", genres)
|
||||
}
|
||||
}
|
||||
|
||||
// ── NodeToMarkdown ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestNodeToMarkdown_Paragraphs(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content">
|
||||
<p>First paragraph.</p>
|
||||
<p>Second paragraph.</p>
|
||||
</div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
if container == nil {
|
||||
t.Fatal("missing #content")
|
||||
}
|
||||
md := NodeToMarkdown(container)
|
||||
if md == "" {
|
||||
t.Fatal("NodeToMarkdown returned empty string")
|
||||
}
|
||||
for _, want := range []string{"First paragraph", "Second paragraph"} {
|
||||
if !strings.Contains(md, want) {
|
||||
t.Errorf("NodeToMarkdown missing %q in:\n%s", want, md)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeToMarkdown_Bold(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content"><p>He was <strong>very</strong> strong.</p></div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
md := NodeToMarkdown(container)
|
||||
if !strings.Contains(md, "**very**") {
|
||||
t.Errorf("NodeToMarkdown should wrap <strong> in **, got:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeToMarkdown_ScriptStripped(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content"><p>Good</p><script>alert(1)</script></div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
md := NodeToMarkdown(container)
|
||||
if strings.Contains(md, "alert") {
|
||||
t.Errorf("NodeToMarkdown should strip <script> content, got:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeToMarkdown_CollapseBlankLines(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content">
|
||||
<p>A</p>
|
||||
<p></p>
|
||||
<p></p>
|
||||
<p>B</p>
|
||||
</div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
md := NodeToMarkdown(container)
|
||||
// Should not have more than one consecutive blank line.
|
||||
if strings.Contains(md, "\n\n\n") {
|
||||
t.Errorf("NodeToMarkdown should collapse triple newlines, got:\n%q", md)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,10 @@
|
||||
// wires them together without knowing anything about the concrete provider.
|
||||
package scraper
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ─── Domain types ────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -58,6 +61,19 @@ type Chapter struct {
|
||||
Text string
|
||||
}
|
||||
|
||||
// RankingItem represents a single entry in the novel ranking list.
|
||||
type RankingItem struct {
|
||||
Rank int `json:"rank"`
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Cover string `json:"cover,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Genres []string `json:"genres,omitempty"`
|
||||
SourceURL string `json:"source_url,omitempty"`
|
||||
Updated time.Time `json:"updated,omitempty"`
|
||||
}
|
||||
|
||||
// ─── Scraping selector descriptors ───────────────────────────────────────────
|
||||
|
||||
// Selector describes how to locate an element in an HTML document.
|
||||
@@ -120,16 +136,6 @@ type RankingProvider interface {
|
||||
ScrapeRanking(ctx context.Context, maxPages int) (<-chan BookMeta, <-chan error)
|
||||
}
|
||||
|
||||
// RankingPageCacher persists and retrieves raw HTML for individual ranking pages.
|
||||
// Implementations (e.g. writer.Writer) store files on disk so that a
|
||||
// subsequent ScrapeRanking call can serve cached HTML without a network round-trip.
|
||||
type RankingPageCacher interface {
|
||||
// WriteRankingPageCache stores the raw HTML string for the given page number.
|
||||
WriteRankingPageCache(page int, html string) error
|
||||
// ReadRankingPageCache returns the cached HTML for page, or ("", nil) on a miss.
|
||||
ReadRankingPageCache(page int) (string, error)
|
||||
}
|
||||
|
||||
// NovelScraper is the full interface that a concrete novel source must implement.
|
||||
// It composes all four provider interfaces.
|
||||
type NovelScraper interface {
|
||||
|
||||
712
scraper/internal/server/handlers_audio.go
Normal file
712
scraper/internal/server/handlers_audio.go
Normal file
@@ -0,0 +1,712 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ─── Audio generation via Kokoro /v1/audio/speech ────────────────────────────
|
||||
//
|
||||
// handleAudioGenerate handles POST /api/audio/{slug}/{n}.
|
||||
//
|
||||
// The handler is non-blocking: it creates an audio_jobs record in PocketBase
|
||||
// with status="pending", then fires a background goroutine to call Kokoro.
|
||||
// The caller should poll GET /api/audio/status/{slug}/{n} to track progress.
|
||||
//
|
||||
// If audio is already cached (audio_cache hit) the handler returns
|
||||
// status=200 with the proxy URL immediately — no job is created.
|
||||
//
|
||||
// Concurrent requests for the same key are deduplicated via audioJobIDs:
|
||||
// the second caller gets a 202 with the existing job_id immediately.
|
||||
func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 {
|
||||
http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse optional voice from JSON body.
|
||||
voice := s.kokoroVoice
|
||||
var body struct {
|
||||
Voice string `json:"voice"`
|
||||
MaxChars int `json:"max_chars"`
|
||||
}
|
||||
if r.Body != nil {
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
}
|
||||
if body.Voice != "" {
|
||||
voice = body.Voice
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
|
||||
|
||||
// Fast path: already generated (check persistent store first).
|
||||
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
||||
s.writeAudioResponse(w, slug, n, voice, filename)
|
||||
return
|
||||
}
|
||||
|
||||
// Deduplicate concurrent generation for the same key.
|
||||
// If a goroutine is already running for this key, return the existing job_id.
|
||||
s.audioMu.Lock()
|
||||
if jobID, ok := s.audioJobIDs[cacheKey]; ok {
|
||||
s.audioMu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"job_id": jobID, "status": "generating"})
|
||||
return
|
||||
}
|
||||
|
||||
// Create the PocketBase job record.
|
||||
jobID, createErr := s.store.CreateAudioJob(r.Context(), slug, n, voice)
|
||||
if createErr != nil {
|
||||
s.audioMu.Unlock()
|
||||
s.log.Warn("audio: failed to create job record", "slug", slug, "chapter", n, "err", createErr)
|
||||
// Non-fatal: still proceed, just won't have a persistent job record.
|
||||
jobID = ""
|
||||
}
|
||||
|
||||
s.audioJobIDs[cacheKey] = jobID
|
||||
s.audioMu.Unlock()
|
||||
|
||||
// Fire background goroutine — request context must NOT be used here since
|
||||
// the handler returns immediately.
|
||||
maxChars := body.MaxChars
|
||||
go func() {
|
||||
defer func() {
|
||||
s.audioMu.Lock()
|
||||
delete(s.audioJobIDs, cacheKey)
|
||||
s.audioMu.Unlock()
|
||||
}()
|
||||
|
||||
bgCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
s.runAudioGeneration(bgCtx, jobID, slug, n, voice, maxChars, cacheKey)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"job_id": jobID, "status": "pending"})
|
||||
}
|
||||
|
||||
// runAudioGeneration performs the actual Kokoro TTS work in a goroutine.
|
||||
// It updates the audio_jobs record as it progresses and writes to audio_cache
|
||||
// and MinIO on success.
|
||||
func (s *Server) runAudioGeneration(ctx context.Context, jobID, slug string, n int, voice string, maxChars int, cacheKey string) {
|
||||
markFailed := func(msg string) {
|
||||
if jobID == "" {
|
||||
return
|
||||
}
|
||||
if err := s.store.UpdateAudioJob(ctx, jobID, "failed", msg, time.Now()); err != nil {
|
||||
s.log.Warn("audio: failed to update job to failed", "job_id", jobID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Transition to "generating".
|
||||
if jobID != "" {
|
||||
if err := s.store.UpdateAudioJob(ctx, jobID, "generating", "", time.Time{}); err != nil {
|
||||
s.log.Warn("audio: failed to mark job generating", "job_id", jobID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Load and validate chapter text.
|
||||
raw, err := s.store.ReadChapter(ctx, slug, n)
|
||||
if err != nil {
|
||||
s.log.Error("audio: chapter not found", "slug", slug, "chapter", n, "err", err)
|
||||
markFailed("chapter not found")
|
||||
return
|
||||
}
|
||||
text := stripMarkdown(raw)
|
||||
if text == "" {
|
||||
markFailed("chapter text is empty")
|
||||
return
|
||||
}
|
||||
if maxChars > 0 && len([]rune(text)) > maxChars {
|
||||
text = string([]rune(text)[:maxChars])
|
||||
}
|
||||
if s.kokoroURL == "" {
|
||||
markFailed("kokoro not configured")
|
||||
return
|
||||
}
|
||||
|
||||
// Call Kokoro.
|
||||
filename, err := s.generateSpeech(ctx, text, voice, 1.0)
|
||||
if err != nil {
|
||||
s.log.Error("audio: kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
|
||||
markFailed(err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.store.SetAudioCache(ctx, cacheKey, filename); err != nil {
|
||||
s.log.Warn("audio: cache write failed", "slug", slug, "chapter", n, "err", err)
|
||||
}
|
||||
|
||||
// Download from Kokoro and persist to MinIO.
|
||||
minioKey := s.store.AudioObjectKey(slug, n, voice)
|
||||
audioData, dlErr := s.downloadFromKokoro(ctx, filename)
|
||||
if dlErr != nil {
|
||||
s.log.Warn("audio: MinIO upload skipped: kokoro download failed",
|
||||
"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
|
||||
} else if putErr := s.store.PutAudio(ctx, minioKey, audioData); putErr != nil {
|
||||
s.log.Warn("audio: MinIO upload failed",
|
||||
"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
|
||||
} else {
|
||||
s.log.Info("audio: uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
|
||||
}
|
||||
|
||||
// Mark job done.
|
||||
if jobID != "" {
|
||||
if err := s.store.UpdateAudioJob(ctx, jobID, "done", "", time.Now()); err != nil {
|
||||
s.log.Warn("audio: failed to mark job done", "job_id", jobID, "err", err)
|
||||
}
|
||||
}
|
||||
s.log.Info("audio: generation complete", "slug", slug, "chapter", n, "filename", filename)
|
||||
}
|
||||
|
||||
// handleAudioStatus handles GET /api/audio/status/{slug}/{n}.
|
||||
// Returns the current generation status for the given chapter + voice.
|
||||
//
|
||||
// Query params: voice (optional, defaults to server default).
|
||||
//
|
||||
// Possible responses:
|
||||
// - 200 {"status":"done","url":"/api/audio-proxy/..."} — audio ready
|
||||
// - 200 {"status":"pending"|"generating","job_id":"..."} — in progress
|
||||
// - 200 {"status":"idle"} — no job yet
|
||||
// - 200 {"status":"failed","error":"..."} — last job failed
|
||||
func (s *Server) handleAudioStatus(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
voice := r.URL.Query().Get("voice")
|
||||
if voice == "" {
|
||||
voice = s.kokoroVoice
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
// Fast path: audio already in audio_cache → done.
|
||||
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
||||
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"status": "done",
|
||||
"url": proxyURL,
|
||||
"filename": filename,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Check in-flight map for live job ID.
|
||||
s.audioMu.Lock()
|
||||
liveJobID, inFlight := s.audioJobIDs[cacheKey]
|
||||
s.audioMu.Unlock()
|
||||
|
||||
if inFlight {
|
||||
// Look up persistent record for richer status.
|
||||
if job, ok, _ := s.store.GetAudioJob(r.Context(), cacheKey); ok {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"status": job.Status,
|
||||
"job_id": liveJobID,
|
||||
})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"status": "generating",
|
||||
"job_id": liveJobID,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Not in-flight: check persistent record for last known result.
|
||||
job, ok, _ := s.store.GetAudioJob(r.Context(), cacheKey)
|
||||
if !ok {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"status": "idle"})
|
||||
return
|
||||
}
|
||||
|
||||
resp := map[string]string{
|
||||
"status": job.Status,
|
||||
"job_id": job.ID,
|
||||
}
|
||||
if job.Status == "failed" && job.ErrorMessage != "" {
|
||||
resp["error"] = job.ErrorMessage
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
// generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true
|
||||
// and returns the filename from the X-Download-Path response header.
|
||||
func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) {
|
||||
reqBody, _ := json.Marshal(map[string]interface{}{
|
||||
"model": "kokoro",
|
||||
"input": text,
|
||||
"voice": voice,
|
||||
"response_format": "mp3",
|
||||
"speed": speed,
|
||||
"stream": false,
|
||||
"return_download_link": true,
|
||||
})
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("kokoro request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
// Drain body so the connection can be reused.
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("kokoro status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// X-Download-Path is e.g. "/download/speech_abc123.mp3"
|
||||
dlPath := resp.Header.Get("X-Download-Path")
|
||||
if dlPath == "" {
|
||||
return "", fmt.Errorf("kokoro did not return X-Download-Path header")
|
||||
}
|
||||
|
||||
// Extract just the filename from the path.
|
||||
filename := dlPath
|
||||
if idx := strings.LastIndex(dlPath, "/"); idx >= 0 {
|
||||
filename = dlPath[idx+1:]
|
||||
}
|
||||
if filename == "" {
|
||||
return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath)
|
||||
}
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
// downloadFromKokoro downloads a generated audio file from Kokoro's temp storage
|
||||
// using GET /v1/download/{filename} and returns the raw bytes.
|
||||
func (s *Server) downloadFromKokoro(ctx context.Context, filename string) ([]byte, error) {
|
||||
url := s.kokoroURL + "/v1/download/" + filename
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build download request: %w", err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kokoro download request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("kokoro download status %d", resp.StatusCode)
|
||||
}
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read kokoro download body: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// writeAudioResponse writes the JSON response for an already-cached audio chapter.
|
||||
// The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
|
||||
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, filename string) {
|
||||
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"url": proxyURL,
|
||||
"filename": filename,
|
||||
})
|
||||
}
|
||||
|
||||
// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}.
|
||||
// It looks up the Kokoro download filename for this chapter (voice) and
|
||||
// proxies GET /v1/download/{filename} from the Kokoro server back to the browser.
|
||||
func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
voice := r.URL.Query().Get("voice")
|
||||
if voice == "" {
|
||||
voice = s.kokoroVoice
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
|
||||
filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
|
||||
if !ok {
|
||||
http.Error(w, "audio not generated yet", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
kokoroURL := s.kokoroURL + "/v1/download/" + filename
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to build proxy request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
http.Error(w, "kokoro download failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "audio/mpeg")
|
||||
w.Header().Set("Cache-Control", "public, max-age=3600")
|
||||
if cl := resp.Header.Get("Content-Length"); cl != "" {
|
||||
w.Header().Set("Content-Length", cl)
|
||||
}
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// ─── Presigned URL handlers ───────────────────────────────────────────────────
|
||||
|
||||
// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}.
|
||||
// Returns a short-lived presigned MinIO URL for the chapter markdown object.
|
||||
// The SvelteKit server uses this to fetch chapter content server-side.
|
||||
func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
url, err := s.store.PresignChapter(r.Context(), slug, n, 15*time.Minute)
|
||||
if err != nil {
|
||||
s.log.Error("presign chapter failed", "slug", slug, "n", n, "err", err)
|
||||
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
||||
}
|
||||
|
||||
// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}.
|
||||
// Returns a presigned MinIO URL for the audio object (if it has been generated).
|
||||
// Query params: voice (optional, defaults to server default).
|
||||
func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
voice := r.URL.Query().Get("voice")
|
||||
if voice == "" {
|
||||
voice = s.kokoroVoice
|
||||
}
|
||||
|
||||
key := s.store.AudioObjectKey(slug, n, voice)
|
||||
|
||||
// Return 404 when the object hasn't been uploaded yet — the client treats
|
||||
// this as "audio not ready" and will either poll or trigger generation.
|
||||
if !s.store.AudioExists(r.Context(), key) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
|
||||
if err != nil {
|
||||
s.log.Error("presign audio failed", "slug", slug, "n", n, "err", err)
|
||||
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
||||
}
|
||||
|
||||
// ─── Voices API ───────────────────────────────────────────────────────────────
|
||||
|
||||
// handleVoices handles GET /api/voices.
|
||||
// Returns the list of available Kokoro voices as JSON: {"voices": [...]}
|
||||
func (s *Server) handleVoices(w http.ResponseWriter, _ *http.Request) {
|
||||
voices := s.voices()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{"voices": voices})
|
||||
}
|
||||
|
||||
// ─── Voice sample generation ──────────────────────────────────────────────────
|
||||
|
||||
// voiceSampleText is the short passage used for voice sample previews.
|
||||
const voiceSampleText = "The ancient library held secrets older than memory itself, its dust-laden shelves stretching upward into shadow. She reached for the worn leather spine, fingers trembling with anticipation."
|
||||
|
||||
// voiceSampleKey returns the MinIO object key for a voice sample.
|
||||
// Key: _voice-samples/{voice}.mp3
|
||||
func voiceSampleKey(voice string) string {
|
||||
safe := strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
|
||||
(r >= '0' && r <= '9') || r == '_' || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '_'
|
||||
}, voice)
|
||||
return fmt.Sprintf("_voice-samples/%s.mp3", safe)
|
||||
}
|
||||
|
||||
// warmVoiceSamples runs at startup in a background goroutine.
|
||||
// It generates a short audio sample for every available Kokoro voice that
|
||||
// doesn't already have one in MinIO, so the UI voice selector has playable
|
||||
// previews without requiring a manual trigger.
|
||||
// It respects ctx cancellation and waits up to 30 s for Kokoro to become
|
||||
// reachable before giving up.
|
||||
func (s *Server) warmVoiceSamples(ctx context.Context) {
|
||||
if s.kokoroURL == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// Wait for Kokoro to be reachable (it may still be starting up).
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
break
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
voices := s.voices()
|
||||
s.log.Info("warming voice samples", "voices", len(voices))
|
||||
|
||||
generated, skipped, failed := 0, 0, 0
|
||||
for _, voice := range voices {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
key := voiceSampleKey(voice)
|
||||
if s.store.AudioExists(ctx, key) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
filename, err := s.generateSpeech(ctx, voiceSampleText, voice, 1.0)
|
||||
if err != nil {
|
||||
s.log.Warn("voice sample warmup: generation failed", "voice", voice, "err", err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
|
||||
audioData, err := s.downloadFromKokoro(ctx, filename)
|
||||
if err != nil {
|
||||
s.log.Warn("voice sample warmup: download failed", "voice", voice, "err", err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
|
||||
if err := s.store.PutAudio(ctx, key, audioData); err != nil {
|
||||
s.log.Warn("voice sample warmup: upload failed", "voice", voice, "key", key, "err", err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
|
||||
s.log.Debug("voice sample warmed", "voice", voice)
|
||||
generated++
|
||||
}
|
||||
|
||||
s.log.Info("voice sample warmup complete",
|
||||
"generated", generated, "skipped", skipped, "failed", failed)
|
||||
}
|
||||
|
||||
// handleGenerateVoiceSamples handles POST /api/audio/voice-samples.
|
||||
// It generates short audio samples for each available voice and stores them
|
||||
// in the audio MinIO bucket so the UI can play them during voice selection.
|
||||
// Already-generated samples are skipped (idempotent).
|
||||
// Optional JSON body: {"voices": ["af_bella", ...]} to generate a subset.
|
||||
// Returns: {"generated": [...], "skipped": [...], "failed": [...]}
|
||||
func (s *Server) handleGenerateVoiceSamples(w http.ResponseWriter, r *http.Request) {
|
||||
if s.kokoroURL == "" {
|
||||
http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse optional voice list from body.
|
||||
var body struct {
|
||||
Voices []string `json:"voices"`
|
||||
}
|
||||
if r.Body != nil {
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
}
|
||||
|
||||
targetVoices := body.Voices
|
||||
if len(targetVoices) == 0 {
|
||||
targetVoices = s.voices()
|
||||
}
|
||||
|
||||
type result struct {
|
||||
Generated []string `json:"generated"`
|
||||
Skipped []string `json:"skipped"`
|
||||
Failed []string `json:"failed"`
|
||||
}
|
||||
var res result
|
||||
|
||||
for _, voice := range targetVoices {
|
||||
key := voiceSampleKey(voice)
|
||||
|
||||
// Skip if already uploaded.
|
||||
if s.store.AudioExists(r.Context(), key) {
|
||||
res.Skipped = append(res.Skipped, voice)
|
||||
s.log.Debug("voice sample already exists, skipping", "voice", voice)
|
||||
continue
|
||||
}
|
||||
|
||||
// Generate via Kokoro (speed 1.0 for samples).
|
||||
filename, err := s.generateSpeech(r.Context(), voiceSampleText, voice, 1.0)
|
||||
if err != nil {
|
||||
s.log.Warn("voice sample generation failed", "voice", voice, "err", err)
|
||||
res.Failed = append(res.Failed, voice)
|
||||
continue
|
||||
}
|
||||
|
||||
// Download from Kokoro and upload to MinIO.
|
||||
audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
|
||||
if dlErr != nil {
|
||||
s.log.Warn("voice sample kokoro download failed", "voice", voice, "err", dlErr)
|
||||
res.Failed = append(res.Failed, voice)
|
||||
continue
|
||||
}
|
||||
|
||||
if putErr := s.store.PutAudio(r.Context(), key, audioData); putErr != nil {
|
||||
s.log.Warn("voice sample MinIO upload failed", "voice", voice, "key", key, "err", putErr)
|
||||
res.Failed = append(res.Failed, voice)
|
||||
continue
|
||||
}
|
||||
|
||||
s.log.Info("voice sample generated", "voice", voice, "key", key)
|
||||
res.Generated = append(res.Generated, voice)
|
||||
}
|
||||
|
||||
if res.Generated == nil {
|
||||
res.Generated = []string{}
|
||||
}
|
||||
if res.Skipped == nil {
|
||||
res.Skipped = []string{}
|
||||
}
|
||||
if res.Failed == nil {
|
||||
res.Failed = []string{}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(res)
|
||||
}
|
||||
|
||||
// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}.
|
||||
// Returns a presigned URL for the voice sample audio file stored in MinIO.
|
||||
// Returns 404 if the sample has not been generated yet.
|
||||
func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) {
|
||||
voice := r.PathValue("voice")
|
||||
if voice == "" {
|
||||
http.Error(w, `{"error":"missing voice"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
key := voiceSampleKey(voice)
|
||||
|
||||
if !s.store.AudioExists(r.Context(), key) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
|
||||
if err != nil {
|
||||
s.log.Error("presign voice sample failed", "voice", voice, "err", err)
|
||||
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
||||
}
|
||||
|
||||
// handlePresignAvatarUpload handles GET /api/presign/avatar-upload/{userId}.
|
||||
// Returns a short-lived presigned PUT URL for uploading an avatar image directly
|
||||
// to MinIO, along with the object key to record in PocketBase after the upload.
|
||||
// Query param: ext — image extension (jpg, png, webp). Defaults to "jpg".
|
||||
func (s *Server) handlePresignAvatarUpload(w http.ResponseWriter, r *http.Request) {
|
||||
userID := r.PathValue("userId")
|
||||
if userID == "" {
|
||||
http.Error(w, `{"error":"missing userId"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
ext := r.URL.Query().Get("ext")
|
||||
switch ext {
|
||||
case "jpg", "jpeg":
|
||||
ext = "jpg"
|
||||
case "png":
|
||||
ext = "png"
|
||||
case "webp":
|
||||
ext = "webp"
|
||||
default:
|
||||
ext = "jpg"
|
||||
}
|
||||
|
||||
uploadURL, key, err := s.store.PresignAvatarUpload(r.Context(), userID, ext)
|
||||
if err != nil {
|
||||
s.log.Error("presign avatar upload failed", "userId", userID, "err", err)
|
||||
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"upload_url": uploadURL,
|
||||
"key": key,
|
||||
})
|
||||
}
|
||||
|
||||
// handlePresignAvatar handles GET /api/presign/avatar/{userId}.
|
||||
// Returns a presigned GET URL for a user's existing avatar, or 404 if none.
|
||||
func (s *Server) handlePresignAvatar(w http.ResponseWriter, r *http.Request) {
|
||||
userID := r.PathValue("userId")
|
||||
if userID == "" {
|
||||
http.Error(w, `{"error":"missing userId"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
url, found, err := s.store.PresignAvatarURL(r.Context(), userID)
|
||||
if err != nil {
|
||||
s.log.Error("presign avatar failed", "userId", userID, "err", err)
|
||||
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
||||
}
|
||||
575
scraper/internal/server/handlers_browse.go
Normal file
575
scraper/internal/server/handlers_browse.go
Normal file
@@ -0,0 +1,575 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
"golang.org/x/net/html"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper/htmlutil"
|
||||
)
|
||||
|
||||
// ─── Browse API ───────────────────────────────────────────────────────────────
|
||||
|
||||
// NovelListing represents a single novel entry from the novelfire browse page.
|
||||
type NovelListing struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Cover string `json:"cover"`
|
||||
Rank string `json:"rank"`
|
||||
Rating string `json:"rating"`
|
||||
Chapters string `json:"chapters"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
const novelFireBase = "https://novelfire.net"
|
||||
const novelFireDomain = "novelfire.net"
|
||||
|
||||
// handleBrowse handles GET /api/browse.
|
||||
// Query params:
|
||||
//
|
||||
// page (default 1)
|
||||
// genre (default "all")
|
||||
// sort (default "popular")
|
||||
// status (default "all")
|
||||
// type (default "all-novel")
|
||||
//
|
||||
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
|
||||
//
|
||||
// Cache strategy: check MinIO browse bucket first (key: {domain}/html/page-N.html);
|
||||
// if a snapshot exists, parse it and return structured JSON.
|
||||
// On a cache miss, fetch live from novelfire.net, return the result, and
|
||||
// trigger a background SingleFile snapshot + ranking population.
|
||||
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
page := q.Get("page")
|
||||
if page == "" {
|
||||
page = "1"
|
||||
}
|
||||
genre := q.Get("genre")
|
||||
if genre == "" {
|
||||
genre = "all"
|
||||
}
|
||||
sortBy := q.Get("sort")
|
||||
if sortBy == "" {
|
||||
sortBy = "popular"
|
||||
}
|
||||
status := q.Get("status")
|
||||
if status == "" {
|
||||
status = "all"
|
||||
}
|
||||
novelType := q.Get("type")
|
||||
if novelType == "" {
|
||||
novelType = "all-novel"
|
||||
}
|
||||
|
||||
pageNum, _ := strconv.Atoi(page)
|
||||
if pageNum <= 0 {
|
||||
pageNum = 1
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
|
||||
cacheKey := s.store.BrowseFilteredHTMLKey(novelFireDomain, pageNum, sortBy, genre, status)
|
||||
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok && len(html) > 0 {
|
||||
novels, hasNext := parseBrowsePage(strings.NewReader(html))
|
||||
s.log.Debug("browse: served from cache", "key", cacheKey)
|
||||
// Still fire background ranking population in case PocketBase ranking
|
||||
// records are missing (e.g. after a schema reset / fresh deploy).
|
||||
targetURLForRanking := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
|
||||
novelFireBase, genre, sortBy, status, novelType, page)
|
||||
s.triggerDirectScrape(cacheKey, targetURLForRanking)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"novels": novels,
|
||||
"page": pageNum,
|
||||
"hasNext": hasNext,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// ── Live fallback: direct fetch from novelfire.net ───────────────────
|
||||
// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
|
||||
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
|
||||
novelFireBase, genre, sortBy, status, novelType, page)
|
||||
|
||||
var novels []NovelListing
|
||||
var hasNext bool
|
||||
var fetchErr error
|
||||
for attempt := 1; attempt <= 3; attempt++ {
|
||||
if attempt > 1 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
case <-time.After(time.Duration(attempt) * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
var req *http.Request
|
||||
req, fetchErr = http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
|
||||
if fetchErr != nil {
|
||||
http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
// Do NOT set Accept-Encoding manually: Go's http.Transport handles
|
||||
// transparent gzip decompression only when it adds the header itself.
|
||||
// If we set it explicitly, Transport disables auto-decompression and
|
||||
// parseBrowsePage receives raw gzip bytes instead of HTML.
|
||||
req.Header.Set("Cache-Control", "no-cache")
|
||||
req.Header.Set("Pragma", "no-cache")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
fetchErr = err
|
||||
s.log.Warn("browse fetch failed, retrying", "url", targetURL, "attempt", attempt, "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
fetchErr = fmt.Errorf("upstream returned %d", resp.StatusCode)
|
||||
s.log.Warn("browse upstream error, retrying", "url", targetURL, "attempt", attempt, "status", resp.StatusCode)
|
||||
continue
|
||||
}
|
||||
|
||||
novels, hasNext = parseBrowsePage(resp.Body)
|
||||
resp.Body.Close()
|
||||
fetchErr = nil
|
||||
break
|
||||
}
|
||||
if fetchErr != nil {
|
||||
s.log.Error("browse fetch failed after retries", "url", targetURL, "err", fetchErr)
|
||||
// ── In-memory fallback: use cached result from a prior successful fetch ──
|
||||
s.browseMemCacheMu.RLock()
|
||||
entry, memHit := s.browseMemCache[cacheKey]
|
||||
s.browseMemCacheMu.RUnlock()
|
||||
if memHit {
|
||||
s.log.Warn("browse: upstream unavailable, serving stale in-memory cache",
|
||||
"key", cacheKey, "age", time.Since(entry.cachedAt).Round(time.Second))
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=60")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"novels": entry.novels,
|
||||
"page": pageNum,
|
||||
"hasNext": entry.hasNext,
|
||||
})
|
||||
return
|
||||
}
|
||||
http.Error(w, fmt.Sprintf(`{"error":"%s"}`, fetchErr.Error()), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
// ── Populate in-memory cache with the fresh upstream result ──────────
|
||||
if len(novels) > 0 {
|
||||
s.browseMemCacheMu.Lock()
|
||||
s.browseMemCache[cacheKey] = browseCacheEntry{
|
||||
novels: novels,
|
||||
hasNext: hasNext,
|
||||
cachedAt: time.Now(),
|
||||
}
|
||||
s.browseMemCacheMu.Unlock()
|
||||
}
|
||||
|
||||
// ── Background: fetch and cache page directly from novelfire.net ─────
|
||||
// Fire-and-forget: stores raw HTML in MinIO and populates the ranking
|
||||
// collection in PocketBase (no browser/SingleFile needed).
|
||||
s.triggerDirectScrape(cacheKey, targetURL)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"novels": novels,
|
||||
"page": pageNum,
|
||||
"hasNext": hasNext,
|
||||
})
|
||||
}
|
||||
|
||||
// triggerDirectScrape fires a background goroutine that:
|
||||
// 1. Fetches pageURL directly from novelfire.net using Go's HTTP client
|
||||
// (no browser/SingleFile needed — the page is server-rendered HTML).
|
||||
// 2. Stores the raw HTML in MinIO at cacheKey so future requests are served
|
||||
// from cache without hitting the origin.
|
||||
// 3. Parses the HTML to extract novel listings.
|
||||
// 4. For each listing, upserts a ranking record in PocketBase (rank, slug,
|
||||
// title, cover key, source_url).
|
||||
// 5. Fires a separate goroutine per cover image to download and store it at
|
||||
// {domain}/assets/book-covers/{slug}.jpg in MinIO.
|
||||
//
|
||||
// It is a no-op when a refresh for this cache key is already in progress.
|
||||
// The goroutine uses a fresh context so it outlives the HTTP request.
|
||||
func (s *Server) triggerDirectScrape(cacheKey, pageURL string) {
|
||||
s.browseMu.Lock()
|
||||
if _, inflight := s.browseInFlight[cacheKey]; inflight {
|
||||
s.browseMu.Unlock()
|
||||
return
|
||||
}
|
||||
s.browseInFlight[cacheKey] = struct{}{}
|
||||
s.browseMu.Unlock()
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
s.browseMu.Lock()
|
||||
delete(s.browseInFlight, cacheKey)
|
||||
s.browseMu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
|
||||
if err != nil {
|
||||
s.log.Warn("triggerDirectScrape: build request failed", "key", cacheKey, "err", err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
s.log.Warn("triggerDirectScrape: fetch failed", "key", cacheKey, "err", err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
s.log.Warn("triggerDirectScrape: non-200 response", "key", cacheKey, "status", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
htmlBytes, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
s.log.Warn("triggerDirectScrape: read body failed", "key", cacheKey, "err", readErr)
|
||||
return
|
||||
}
|
||||
if len(htmlBytes) == 0 {
|
||||
s.log.Warn("triggerDirectScrape: empty response body", "key", cacheKey)
|
||||
return
|
||||
}
|
||||
|
||||
// Store the HTML in MinIO so subsequent requests are cache-hits.
|
||||
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
|
||||
s.log.Warn("triggerDirectScrape: SaveBrowsePage failed", "key", cacheKey, "err", putErr)
|
||||
// Non-fatal: continue to populate PocketBase/covers even if MinIO write fails.
|
||||
} else {
|
||||
s.log.Info("triggerDirectScrape: cached browse page", "key", cacheKey, "bytes", len(htmlBytes))
|
||||
}
|
||||
|
||||
// Parse to extract novel listings.
|
||||
novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
|
||||
if len(novels) == 0 {
|
||||
s.log.Warn("triggerDirectScrape: no novels parsed", "key", cacheKey)
|
||||
return
|
||||
}
|
||||
|
||||
// Upsert each novel into PocketBase ranking and kick off cover downloads.
|
||||
for i, novel := range novels {
|
||||
rank := i + 1
|
||||
coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)
|
||||
|
||||
item := storage.RankingItem{
|
||||
Rank: rank,
|
||||
Slug: novel.Slug,
|
||||
Title: novel.Title,
|
||||
Cover: coverKey, // stored as MinIO key; UI fetches via /api/cover/...
|
||||
SourceURL: novel.URL,
|
||||
}
|
||||
if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
|
||||
s.log.Warn("triggerDirectScrape: WriteRankingItem failed",
|
||||
"slug", novel.Slug, "err", werr)
|
||||
}
|
||||
|
||||
if novel.Cover != "" {
|
||||
go s.downloadAndStoreCover(coverKey, novel.Cover)
|
||||
}
|
||||
}
|
||||
|
||||
s.log.Info("triggerDirectScrape: ranking populated", "count", len(novels), "key", cacheKey)
|
||||
}()
|
||||
}
|
||||
|
||||
// warmBrowseCache checks whether the browse cache for page 1 is populated in
|
||||
// MinIO and, if not, triggers a background direct scrape. This is called
|
||||
// once on server startup so the first user request is likely served from cache.
|
||||
func (s *Server) warmBrowseCache() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cacheKey := s.store.BrowseHTMLKey(novelFireDomain, 1)
|
||||
if _, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
|
||||
s.log.Debug("warmBrowseCache: page 1 already cached, skipping")
|
||||
return
|
||||
}
|
||||
|
||||
targetURL := fmt.Sprintf("%s/genre-all/sort-popular/status-all/all-novel?page=1", novelFireBase)
|
||||
s.log.Info("warmBrowseCache: page 1 not cached, triggering background scrape")
|
||||
s.triggerDirectScrape(cacheKey, targetURL)
|
||||
}
|
||||
|
||||
// downloadAndStoreCover delegates to storage.DownloadAndStoreCover.
|
||||
func (s *Server) downloadAndStoreCover(key, imageURL string) {
|
||||
storage.DownloadAndStoreCover(s.store, s.log, key, imageURL)
|
||||
}
|
||||
|
||||
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
|
||||
// Returns novels and whether a "next page" link was found.
|
||||
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
|
||||
doc, err := html.Parse(r)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
var novels []NovelListing
|
||||
hasNext := false
|
||||
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.Data {
|
||||
case "li":
|
||||
if hasClass(n, "novel-item") {
|
||||
if novel, ok := parseNovelItem(n); ok {
|
||||
novels = append(novels, novel)
|
||||
}
|
||||
}
|
||||
// pagination li with class "next"
|
||||
if hasClass(n, "next") {
|
||||
hasNext = true
|
||||
}
|
||||
case "a":
|
||||
// Detect "next" pagination link
|
||||
if hasClass(n, "next") || attrVal(n, "rel") == "next" {
|
||||
hasNext = true
|
||||
}
|
||||
// Also check aria-label="Next"
|
||||
if attrVal(n, "aria-label") == "Next" {
|
||||
hasNext = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(doc)
|
||||
return novels, hasNext
|
||||
}
|
||||
|
||||
// parseNovelItem extracts a NovelListing from a <li class="novel-item"> node.
|
||||
func parseNovelItem(li *html.Node) (NovelListing, bool) {
|
||||
var novel NovelListing
|
||||
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.Data {
|
||||
case "a":
|
||||
href := attrVal(n, "href")
|
||||
if strings.HasPrefix(href, "/book/") {
|
||||
slug := strings.TrimPrefix(href, "/book/")
|
||||
slug = strings.TrimSuffix(slug, "/")
|
||||
if novel.Slug == "" {
|
||||
novel.Slug = slug
|
||||
novel.URL = novelFireBase + href
|
||||
}
|
||||
}
|
||||
case "img":
|
||||
// lazy-loaded covers use data-src
|
||||
src := attrVal(n, "data-src")
|
||||
if src == "" {
|
||||
src = attrVal(n, "src")
|
||||
}
|
||||
if src != "" && novel.Cover == "" {
|
||||
if !strings.HasPrefix(src, "http") {
|
||||
src = novelFireBase + src
|
||||
}
|
||||
novel.Cover = src
|
||||
}
|
||||
case "h4":
|
||||
if hasClass(n, "novel-title") && novel.Title == "" {
|
||||
novel.Title = strings.TrimSpace(textContent(n))
|
||||
}
|
||||
case "span":
|
||||
cls := attrVal(n, "class")
|
||||
if strings.Contains(cls, "_bl") && novel.Rank == "" {
|
||||
novel.Rank = strings.TrimSpace(textContent(n))
|
||||
}
|
||||
if strings.Contains(cls, "_br") && novel.Rating == "" {
|
||||
novel.Rating = strings.TrimSpace(textContent(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(li)
|
||||
|
||||
// Extract chapter count from the novel stats text (contains "N Chapters")
|
||||
novel.Chapters = extractChapters(li)
|
||||
|
||||
if novel.Slug == "" || novel.Title == "" {
|
||||
return novel, false
|
||||
}
|
||||
return novel, true
|
||||
}
|
||||
|
||||
// extractChapters finds the chapter count text within a novel-item node.
|
||||
func extractChapters(n *html.Node) string {
|
||||
var result string
|
||||
var walk func(*html.Node)
|
||||
walk = func(node *html.Node) {
|
||||
if node.Type == html.ElementNode {
|
||||
cls := attrVal(node, "class")
|
||||
if strings.Contains(cls, "novel-stats") || strings.Contains(cls, "chapter") {
|
||||
txt := strings.TrimSpace(textContent(node))
|
||||
if strings.Contains(txt, "Chapter") || strings.Contains(txt, "chapter") {
|
||||
// Extract just the numeric part if possible
|
||||
result = txt
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := node.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return result
|
||||
}
|
||||
|
||||
// hasClass reports whether an HTML node has the given CSS class.
|
||||
func hasClass(n *html.Node, cls string) bool {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == "class" {
|
||||
for _, c := range strings.Fields(a.Val) {
|
||||
if c == cls {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// attrVal returns the value of an attribute on an HTML node, or "".
|
||||
// Delegates to htmlutil.AttrVal.
|
||||
func attrVal(n *html.Node, key string) string { return htmlutil.AttrVal(n, key) }
|
||||
|
||||
// textContent returns the concatenated text content of a node and its descendants.
|
||||
// Delegates to htmlutil.TextContent.
|
||||
func textContent(n *html.Node) string { return htmlutil.TextContent(n) }
|
||||
|
||||
// ─── Search API ───────────────────────────────────────────────────────────────
|
||||
|
||||
// handleSearch handles GET /api/search.
|
||||
//
|
||||
// Query params:
|
||||
//
|
||||
// q — search query string (required, min 2 chars)
|
||||
// source — "local" | "remote" | "all" (default: "all")
|
||||
//
|
||||
// When source includes "local", it searches books already in the local store
|
||||
// by title substring match. When source includes "remote", it fetches the
|
||||
// novelfire.net search page and parses results. Results from both sources
|
||||
// are merged with local results first (de-duplicated by slug).
|
||||
//
|
||||
// Returns JSON: {"results": [...NovelListing], "local_count": N, "remote_count": N}
|
||||
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query().Get("q")
|
||||
if len([]rune(q)) < 2 {
|
||||
http.Error(w, `{"error":"query must be at least 2 characters"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
source := r.URL.Query().Get("source")
|
||||
if source == "" {
|
||||
source = "all"
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var localResults []NovelListing
|
||||
var remoteResults []NovelListing
|
||||
|
||||
// ── Local search (PocketBase books) ──────────────────────────────────
|
||||
if source == "local" || source == "all" {
|
||||
books, err := s.store.ListBooks(ctx)
|
||||
if err != nil {
|
||||
s.log.Warn("search: ListBooks failed", "err", err)
|
||||
} else {
|
||||
qLower := strings.ToLower(q)
|
||||
for _, b := range books {
|
||||
if strings.Contains(strings.ToLower(b.Title), qLower) ||
|
||||
strings.Contains(strings.ToLower(b.Author), qLower) {
|
||||
listing := NovelListing{
|
||||
Slug: b.Slug,
|
||||
Title: b.Title,
|
||||
Cover: b.Cover,
|
||||
URL: b.SourceURL,
|
||||
}
|
||||
localResults = append(localResults, listing)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Remote search (novelfire.net /search?keyword=...) ─────────────────
|
||||
if source == "remote" || source == "all" {
|
||||
searchURL := novelFireBase + "/search?keyword=" + url.QueryEscape(q)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
|
||||
if err == nil {
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
if resp, fetchErr := http.DefaultClient.Do(req); fetchErr == nil {
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
parsed, _ := parseBrowsePage(resp.Body)
|
||||
remoteResults = parsed
|
||||
} else {
|
||||
s.log.Warn("search: remote returned non-200", "status", resp.StatusCode, "url", searchURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Merge: de-duplicate remote results already in local ───────────────
|
||||
localSlugs := make(map[string]bool, len(localResults))
|
||||
for _, item := range localResults {
|
||||
localSlugs[item.Slug] = true
|
||||
}
|
||||
|
||||
combined := make([]NovelListing, 0, len(localResults)+len(remoteResults))
|
||||
combined = append(combined, localResults...)
|
||||
for _, item := range remoteResults {
|
||||
if !localSlugs[item.Slug] {
|
||||
combined = append(combined, item)
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"results": combined,
|
||||
"local_count": len(localResults),
|
||||
"remote_count": len(remoteResults),
|
||||
})
|
||||
}
|
||||
168
scraper/internal/server/handlers_preview.go
Normal file
168
scraper/internal/server/handlers_preview.go
Normal file
@@ -0,0 +1,168 @@
|
||||
package server
|
||||
|
||||
// handlers_preview.go — on-demand preview endpoints for books not yet in PocketBase.
|
||||
//
|
||||
// These endpoints allow the UI to display a book's metadata and chapter list
|
||||
// (scraped live from novelfire.net) without requiring a full scrape to have
|
||||
// been run first. They are read-only: nothing is persisted to PocketBase or
|
||||
// MinIO.
|
||||
//
|
||||
// Endpoints:
|
||||
//
|
||||
// GET /api/book-preview/{slug} — scrape book metadata + chapter list live
|
||||
// GET /api/chapter-text-preview/{slug}/{n} — scrape a single chapter text live
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// BookPreviewResponse is the JSON response for /api/book-preview/{slug}.
|
||||
type BookPreviewResponse struct {
|
||||
InLib bool `json:"in_lib"`
|
||||
Meta scraper.BookMeta `json:"meta"`
|
||||
Chapters []scraper.ChapterRef `json:"chapters"`
|
||||
}
|
||||
|
||||
// handleBookPreview handles GET /api/book-preview/{slug}.
|
||||
//
|
||||
// It scrapes book metadata and the full chapter list live from novelfire.net.
|
||||
// It also checks whether the book exists in the local store (PocketBase) and
|
||||
// sets the InLib flag accordingly. Nothing is written to any store.
|
||||
//
|
||||
// Query param: source_url (optional) — if provided, uses that URL instead of
|
||||
// constructing one from the slug.
|
||||
func (s *Server) handleBookPreview(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Determine the book URL: prefer explicit source_url query param.
|
||||
bookURL := r.URL.Query().Get("source_url")
|
||||
if bookURL == "" {
|
||||
bookURL = fmt.Sprintf("%s/book/%s", novelFireBase, slug)
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
|
||||
// Check whether the book is already in the local library.
|
||||
_, inLib, err := s.store.ReadMetadata(ctx, slug)
|
||||
if err != nil {
|
||||
// Non-fatal: we can still serve the preview.
|
||||
s.log.Warn("book-preview: ReadMetadata failed", "slug", slug, "err", err)
|
||||
inLib = false
|
||||
}
|
||||
|
||||
// Scrape live metadata.
|
||||
meta, err := s.novel.ScrapeMetadata(ctx, bookURL)
|
||||
if err != nil {
|
||||
s.log.Error("book-preview: ScrapeMetadata failed", "slug", slug, "url", bookURL, "err", err)
|
||||
http.Error(w, fmt.Sprintf(`{"error":"metadata scrape failed: %s"}`, err.Error()), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
// Scrape live chapter list.
|
||||
chapters, err := s.novel.ScrapeChapterList(ctx, bookURL)
|
||||
if err != nil {
|
||||
s.log.Error("book-preview: ScrapeChapterList failed", "slug", slug, "url", bookURL, "err", err)
|
||||
// Return partial response with metadata only — chapters are non-critical.
|
||||
chapters = []scraper.ChapterRef{}
|
||||
}
|
||||
|
||||
// If the book was not already in the library, persist the metadata and
|
||||
// chapter list skeleton to PocketBase now so that subsequent visits load
|
||||
// from the local store rather than scraping live again. Chapter text is
|
||||
// NOT fetched here — that still requires an explicit scrape job.
|
||||
if !inLib {
|
||||
go func() {
|
||||
bgCtx := context.Background()
|
||||
if werr := s.store.WriteMetadata(bgCtx, meta); werr != nil {
|
||||
s.log.Warn("book-preview: WriteMetadata failed (non-fatal)", "slug", slug, "err", werr)
|
||||
}
|
||||
if len(chapters) > 0 {
|
||||
if werr := s.store.WriteChapterRefs(bgCtx, slug, chapters); werr != nil {
|
||||
s.log.Warn("book-preview: WriteChapterRefs failed (non-fatal)", "slug", slug, "err", werr)
|
||||
}
|
||||
}
|
||||
s.log.Info("book-preview: metadata+chapter list persisted", "slug", slug, "chapters", len(chapters))
|
||||
}()
|
||||
inLib = true // will be true by the time the client navigates back
|
||||
}
|
||||
|
||||
resp := BookPreviewResponse{
|
||||
InLib: inLib,
|
||||
Meta: meta,
|
||||
Chapters: chapters,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
// ChapterPreviewResponse is the JSON response for /api/chapter-text-preview/{slug}/{n}.
|
||||
type ChapterPreviewResponse struct {
|
||||
Slug string `json:"slug"`
|
||||
Number int `json:"number"`
|
||||
Title string `json:"title"`
|
||||
Text string `json:"text"` // plain text (markdown stripped)
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// handleChapterTextPreview handles GET /api/chapter-text-preview/{slug}/{n}.
|
||||
//
|
||||
// It scrapes a single chapter from novelfire.net live without storing anything.
|
||||
// The chapter URL is determined from either:
|
||||
// - the "chapter_url" query param (preferred — used when the UI knows it from
|
||||
// a prior book-preview call), or
|
||||
// - a best-effort construction: {novelFireBase}/book/{slug}/chapter-{n}
|
||||
//
|
||||
// Returns plain text (markdown stripped) suitable for TTS or display.
|
||||
func (s *Server) handleChapterTextPreview(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
nStr := r.PathValue("n")
|
||||
n, err := strconv.Atoi(nStr)
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Chapter URL: prefer explicit query param.
|
||||
chapterURL := r.URL.Query().Get("chapter_url")
|
||||
if chapterURL == "" {
|
||||
chapterURL = fmt.Sprintf("%s/book/%s/chapter-%d", novelFireBase, slug, n)
|
||||
}
|
||||
|
||||
title := r.URL.Query().Get("title")
|
||||
|
||||
ref := scraper.ChapterRef{
|
||||
Number: n,
|
||||
Title: title,
|
||||
URL: chapterURL,
|
||||
}
|
||||
|
||||
chapter, err := s.novel.ScrapeChapterText(r.Context(), ref)
|
||||
if err != nil {
|
||||
s.log.Error("chapter-text-preview: ScrapeChapterText failed",
|
||||
"slug", slug, "n", n, "url", chapterURL, "err", err)
|
||||
http.Error(w, fmt.Sprintf(`{"error":"chapter scrape failed: %s"}`, err.Error()), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
resp := ChapterPreviewResponse{
|
||||
Slug: slug,
|
||||
Number: n,
|
||||
Title: chapter.Ref.Title,
|
||||
Text: stripMarkdown(chapter.Text),
|
||||
URL: chapterURL,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
103
scraper/internal/server/handlers_progress.go
Normal file
103
scraper/internal/server/handlers_progress.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// ─── Reading progress API ─────────────────────────────────────────────────────
|
||||
|
||||
// handleGetProgress handles GET /api/progress.
|
||||
// Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...}
|
||||
func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
entries, err := s.store.AllProgress(r.Context(), sid)
|
||||
if err != nil {
|
||||
s.log.Error("AllProgress failed", "err", err)
|
||||
entries = nil
|
||||
}
|
||||
|
||||
progress := make(map[string]interface{}, len(entries)*2)
|
||||
for _, p := range entries {
|
||||
progress[p.Slug] = p.Chapter
|
||||
progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(progress)
|
||||
}
|
||||
|
||||
// handleSetProgress handles POST /api/progress/{slug}.
|
||||
// Body: {"chapter": N}
|
||||
func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var body struct {
|
||||
Chapter int `json:"chapter"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
|
||||
http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
p := storage.ReadingProgress{
|
||||
Slug: slug,
|
||||
Chapter: body.Chapter,
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
if err := s.store.SetProgress(r.Context(), sid, p); err != nil {
|
||||
s.log.Error("SetProgress failed", "slug", slug, "err", err)
|
||||
http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{})
|
||||
}
|
||||
|
||||
// handleDeleteProgress handles DELETE /api/progress/{slug}.
|
||||
func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil {
|
||||
s.log.Error("DeleteProgress failed", "slug", slug, "err", err)
|
||||
// Non-fatal — treat as success.
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{})
|
||||
}
|
||||
|
||||
// handleChapterText returns the plain text of a chapter (markdown stripped)
|
||||
// for server-side audio generation. Called by handleAudioGenerate internally.
|
||||
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
fmt.Fprint(w, stripMarkdown(raw))
|
||||
}
|
||||
86
scraper/internal/server/handlers_ranking.go
Normal file
86
scraper/internal/server/handlers_ranking.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// handleGetRanking returns all ranking items sorted by rank ascending.
|
||||
// Cover fields that hold a MinIO object key (e.g. "novelfire.net/assets/book-covers/slug.jpg")
|
||||
// are rewritten to a /api/cover/{key} proxy URL so the UI can fetch them
|
||||
// without knowing about the internal MinIO topology.
|
||||
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
|
||||
items, err := s.store.ReadRankingItems(r.Context())
|
||||
if err != nil {
|
||||
s.log.Error("ranking read failed", "err", err)
|
||||
http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if items == nil {
|
||||
items = []storage.RankingItem{}
|
||||
}
|
||||
// Rewrite cover keys to proxy URLs.
|
||||
// Keys stored by triggerDirectScrape look like:
|
||||
// "novelfire.net/assets/book-covers/shadow-slave.jpg"
|
||||
// We expose them as:
|
||||
// "/api/cover/novelfire.net/shadow-slave"
|
||||
// (the handler strips the domain and slug from the path, reconstructs the key)
|
||||
for i := range items {
|
||||
cover := items[i].Cover
|
||||
if cover != "" && !strings.HasPrefix(cover, "http") {
|
||||
// cover is a MinIO key; extract domain + slug for the proxy path.
|
||||
// Key format: {domain}/assets/book-covers/{slug}.jpg
|
||||
parts := strings.SplitN(cover, "/assets/book-covers/", 2)
|
||||
if len(parts) == 2 {
|
||||
domain := parts[0]
|
||||
slug := strings.TrimSuffix(parts[1], ".jpg")
|
||||
items[i].Cover = "/api/cover/" + domain + "/" + slug
|
||||
}
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(items)
|
||||
}
|
||||
|
||||
// handleGetCover proxies a cover image stored in the MinIO browse bucket.
|
||||
// Route: GET /api/cover/{domain}/{slug}
|
||||
// It reconstructs the MinIO key as {domain}/assets/book-covers/{slug}.jpg,
|
||||
// fetches the object, and streams it to the client.
|
||||
// Returns 404 if not yet downloaded, allowing the UI to fall back to the
|
||||
// original source URL.
|
||||
func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
|
||||
domain := r.PathValue("domain")
|
||||
slug := r.PathValue("slug")
|
||||
if domain == "" || slug == "" {
|
||||
http.Error(w, "missing domain or slug", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
key := s.store.BrowseCoverKey(domain, slug)
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
data, contentType, ok, err := s.store.GetBrowseAsset(ctx, key)
|
||||
if err != nil {
|
||||
s.log.Warn("handleGetCover: GetBrowseAsset error", "key", key, "err", err)
|
||||
http.Error(w, "storage error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
if contentType == "" {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Cache-Control", "public, max-age=86400")
|
||||
_, _ = w.Write(data)
|
||||
}
|
||||
270
scraper/internal/server/handlers_scrape.go
Normal file
270
scraper/internal/server/handlers_scrape.go
Normal file
@@ -0,0 +1,270 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
|
||||
cfg := s.oCfg
|
||||
cfg.SingleBookURL = "" // full catalogue
|
||||
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
||||
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := s.oCfg
|
||||
cfg.SingleBookURL = body.URL
|
||||
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
// handleScrapeBookRange handles POST /api/scrape/book/range.
|
||||
// Body: {"url": "...", "from": N, "to": M}
|
||||
// Scrapes only chapters in the range [from, to] (inclusive).
|
||||
// from=0 means "start from chapter 1"; to=0 means "no upper limit".
|
||||
func (s *Server) handleScrapeBookRange(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
URL string `json:"url"`
|
||||
From int `json:"from"`
|
||||
To int `json:"to"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
||||
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := s.oCfg
|
||||
cfg.SingleBookURL = body.URL
|
||||
cfg.FromChapter = body.From
|
||||
cfg.ToChapter = body.To
|
||||
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
// runAsync launches an orchestrator in the background and returns 202 Accepted.
|
||||
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
|
||||
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
||||
s.mu.Lock()
|
||||
if s.running {
|
||||
s.mu.Unlock()
|
||||
http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
|
||||
return
|
||||
}
|
||||
s.running = true
|
||||
s.mu.Unlock()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
s.running = false
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
|
||||
defer cancel()
|
||||
|
||||
// Determine task kind and target.
|
||||
kind := "catalogue"
|
||||
targetURL := ""
|
||||
if cfg.SingleBookURL != "" {
|
||||
kind = "book"
|
||||
targetURL = cfg.SingleBookURL
|
||||
}
|
||||
|
||||
// Create the task record in PocketBase.
|
||||
taskID, err := s.store.CreateScrapeTask(ctx, kind, targetURL)
|
||||
if err != nil {
|
||||
s.log.Warn("could not create scraping_tasks record", "err", err)
|
||||
// Non-fatal: continue without task tracking.
|
||||
}
|
||||
|
||||
// flush pushes the latest counters to PocketBase (best-effort).
|
||||
flush := func(p orchestrator.Progress, status, errMsg string, finished bool) {
|
||||
if taskID == "" {
|
||||
return
|
||||
}
|
||||
u := storage.ScrapeTaskUpdate{
|
||||
Status: status,
|
||||
BooksFound: p.BooksFound,
|
||||
ChaptersScraped: p.ChaptersScraped,
|
||||
ChaptersSkipped: p.ChaptersSkipped,
|
||||
Errors: p.Errors,
|
||||
ErrorMessage: errMsg,
|
||||
}
|
||||
if finished {
|
||||
u.Finished = time.Now().UTC()
|
||||
}
|
||||
if updateErr := s.store.UpdateScrapeTask(ctx, taskID, u); updateErr != nil {
|
||||
s.log.Warn("could not update scraping_tasks record", "task_id", taskID, "err", updateErr)
|
||||
}
|
||||
}
|
||||
|
||||
cfg.OnProgress = func(p orchestrator.Progress) {
|
||||
flush(p, "running", "", false)
|
||||
}
|
||||
|
||||
o := orchestrator.New(cfg, s.novel, s.log, s.store)
|
||||
runErr := o.Run(ctx)
|
||||
|
||||
// After a successful full-catalogue run, refresh the ranking list.
|
||||
if runErr == nil && cfg.SingleBookURL == "" {
|
||||
s.log.Info("runAsync: starting ScrapeRanking after catalogue run")
|
||||
rankCtx, rankCancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer rankCancel()
|
||||
rankEntries, rankErrs := s.novel.ScrapeRanking(rankCtx, 0)
|
||||
rank := 1
|
||||
for meta := range rankEntries {
|
||||
item := storage.RankingItem{
|
||||
Rank: rank,
|
||||
Slug: meta.Slug,
|
||||
Title: meta.Title,
|
||||
Author: meta.Author,
|
||||
Cover: meta.Cover,
|
||||
Status: meta.Status,
|
||||
Genres: meta.Genres,
|
||||
SourceURL: meta.SourceURL,
|
||||
}
|
||||
if werr := s.store.WriteRankingItem(rankCtx, item); werr != nil {
|
||||
s.log.Warn("runAsync: WriteRankingItem failed", "slug", meta.Slug, "err", werr)
|
||||
}
|
||||
rank++
|
||||
}
|
||||
if rerr := <-rankErrs; rerr != nil {
|
||||
s.log.Warn("runAsync: ScrapeRanking finished with error", "err", rerr)
|
||||
} else {
|
||||
s.log.Info("runAsync: ScrapeRanking complete", "count", rank-1)
|
||||
}
|
||||
}
|
||||
|
||||
// Determine final status.
|
||||
finalStatus := "done"
|
||||
errMsg := ""
|
||||
if runErr != nil {
|
||||
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", runErr))
|
||||
if ctx.Err() != nil {
|
||||
finalStatus = "cancelled"
|
||||
} else {
|
||||
finalStatus = "failed"
|
||||
}
|
||||
errMsg = runErr.Error()
|
||||
}
|
||||
|
||||
// Best-effort: read last known progress counters via a zero-value
|
||||
// OnProgress — we don't have a snapshot here, so re-use whatever the
|
||||
// last OnProgress call delivered (the orchestrator calls notify() at
|
||||
// the very end, so this is always accurate after Run returns).
|
||||
// We issue one final flush with the terminal status and finished time.
|
||||
if taskID != "" {
|
||||
// Re-fetch current counters by listing the task (cheapest path).
|
||||
tasks, listErr := s.store.ListScrapeTasks(ctx)
|
||||
var last storage.ScrapeTaskUpdate
|
||||
if listErr == nil {
|
||||
for _, t := range tasks {
|
||||
if t.ID == taskID {
|
||||
last = storage.ScrapeTaskUpdate{
|
||||
BooksFound: t.BooksFound,
|
||||
ChaptersScraped: t.ChaptersScraped,
|
||||
ChaptersSkipped: t.ChaptersSkipped,
|
||||
Errors: t.Errors,
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
last.Status = finalStatus
|
||||
last.ErrorMessage = errMsg
|
||||
last.Finished = time.Now().UTC()
|
||||
if updateErr := s.store.UpdateScrapeTask(ctx, taskID, last); updateErr != nil {
|
||||
s.log.Warn("could not finalize scraping_tasks record", "task_id", taskID, "err", updateErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// ─── Scrape status API ────────────────────────────────────────────────────────
|
||||
|
||||
// handleScrapeStatus handles GET /api/scrape/status.
|
||||
// Returns JSON: {"running": bool}
|
||||
func (s *Server) handleScrapeStatus(w http.ResponseWriter, _ *http.Request) {
|
||||
s.mu.Lock()
|
||||
running := s.running
|
||||
s.mu.Unlock()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]bool{"running": running})
|
||||
}
|
||||
|
||||
// handleScrapeTasks handles GET /api/scrape/tasks.
|
||||
// Returns JSON array of all scraping_tasks records, newest first.
|
||||
func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) {
|
||||
tasks, err := s.store.ListScrapeTasks(r.Context())
|
||||
if err != nil {
|
||||
s.log.Error("handleScrapeTasks: list failed", "err", err)
|
||||
http.Error(w, `{"error":"failed to list tasks"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if tasks == nil {
|
||||
tasks = []storage.ScrapeTask{}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(tasks)
|
||||
}
|
||||
|
||||
// handleReindex handles POST /api/reindex/{slug}.
|
||||
// It rebuilds the chapters_idx PocketBase collection for the given book by
|
||||
// walking its MinIO objects. Use this when chapters were scraped but the index
|
||||
// is out of sync (e.g. after a failed UpsertChapterIdx during scraping).
|
||||
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
type reindexer interface {
|
||||
ReindexChapters(ctx context.Context, slug string) (int, error)
|
||||
}
|
||||
ri, ok := s.store.(reindexer)
|
||||
if !ok {
|
||||
http.Error(w, `{"error":"store does not support reindex"}`, http.StatusNotImplemented)
|
||||
return
|
||||
}
|
||||
|
||||
count, err := ri.ReindexChapters(r.Context(), slug)
|
||||
if err != nil {
|
||||
s.log.Error("reindex failed", "slug", slug, "indexed", count, "err", err)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"indexed": count,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
s.log.Info("reindex complete", "slug", slug, "indexed", count)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"slug": slug,
|
||||
"indexed": count,
|
||||
})
|
||||
}
|
||||
62
scraper/internal/server/helpers.go
Normal file
62
scraper/internal/server/helpers.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// kokoroVoices is the built-in fallback list of voices shipped with Kokoro-FastAPI.
|
||||
// Used when the live GET /v1/audio/voices request to Kokoro fails.
|
||||
// Grouped by language prefix:
|
||||
//
|
||||
// af_ / am_ American English female / male
|
||||
// bf_ / bm_ British English female / male
|
||||
// ef_ / em_ Spanish female / male
|
||||
// ff_ French female
|
||||
// hf_ / hm_ Hindi female / male
|
||||
// if_ / im_ Italian female / male
|
||||
// jf_ / jm_ Japanese female / male
|
||||
// pf_ / pm_ Portuguese female / male
|
||||
// zf_ / zm_ Chinese female / male
|
||||
var kokoroVoices = []string{
|
||||
// American English
|
||||
"af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia",
|
||||
"af_jessica", "af_kore", "af_nicole", "af_nova", "af_river",
|
||||
"af_sarah", "af_sky",
|
||||
"am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam",
|
||||
"am_michael", "am_onyx", "am_puck",
|
||||
// British English
|
||||
"bf_alice", "bf_emma", "bf_lily",
|
||||
"bm_daniel", "bm_fable", "bm_george", "bm_lewis",
|
||||
// Spanish
|
||||
"ef_dora", "em_alex",
|
||||
// French
|
||||
"ff_siwis",
|
||||
// Hindi
|
||||
"hf_alpha", "hf_beta", "hm_omega", "hm_psi",
|
||||
// Italian
|
||||
"if_sara", "im_nicola",
|
||||
// Japanese
|
||||
"jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo",
|
||||
// Portuguese
|
||||
"pf_dora", "pm_alex",
|
||||
// Chinese
|
||||
"zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi",
|
||||
"zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang",
|
||||
}
|
||||
|
||||
// stripMarkdown removes common markdown syntax from src, returning plain text
|
||||
// suitable for TTS or display. Not a full markdown parser — handles the most
|
||||
// common constructs (headings, bold/italic, code blocks, links, blockquotes).
|
||||
func stripMarkdown(src string) string {
|
||||
src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1")
|
||||
src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n")
|
||||
return strings.TrimSpace(src)
|
||||
}
|
||||
412
scraper/internal/server/integration_test.go
Normal file
412
scraper/internal/server/integration_test.go
Normal file
@@ -0,0 +1,412 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests for the HTTP server against live MinIO + PocketBase backends.
|
||||
//
|
||||
// The server is started on a random port for each test; real HybridStore
|
||||
// backends are used. Browserless-dependent tests are skipped unless
|
||||
// BROWSERLESS_URL is set.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 120s \
|
||||
// github.com/libnovel/scraper/internal/server
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
neturl "net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// ─── fixture helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// newTestStore creates a HybridStore from env vars, skipping if not configured.
|
||||
func newTestStore(t *testing.T) *storage.HybridStore {
|
||||
t.Helper()
|
||||
if os.Getenv("MINIO_ENDPOINT") == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping server integration test")
|
||||
}
|
||||
if os.Getenv("POCKETBASE_URL") == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping server integration test")
|
||||
}
|
||||
|
||||
pbCfg := storage.PocketBaseConfig{
|
||||
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||
}
|
||||
minioCfg := storage.MinioConfig{
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: envOr("MINIO_USE_SSL", "false") == "true",
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
hs, err := storage.NewHybridStore(ctx, pbCfg, minioCfg, slog.Default())
|
||||
if err != nil {
|
||||
t.Fatalf("NewHybridStore: %v", err)
|
||||
}
|
||||
return hs
|
||||
}
|
||||
|
||||
// startTestServer starts a real Server on a random free port and returns the
|
||||
// base URL. The server is shut down when the test finishes.
|
||||
func startTestServer(t *testing.T, store storage.Store) string {
|
||||
t.Helper()
|
||||
|
||||
// Find a free port.
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen: %v", err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
ln.Close()
|
||||
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
|
||||
// nopScraper satisfies scraper.NovelScraper without hitting the network.
|
||||
srv := New(addr, orchestrator.Config{}, nopScraper{}, log, store, "", "af_bella")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
ready := make(chan struct{})
|
||||
go func() {
|
||||
// Signal readiness after a short delay to let the listener bind.
|
||||
go func() {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
close(ready)
|
||||
}()
|
||||
_ = srv.ListenAndServe(ctx)
|
||||
}()
|
||||
|
||||
<-ready
|
||||
|
||||
// Wait until the server actually accepts connections.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err := http.Get("http://" + addr + "/health")
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
|
||||
return "http://" + addr
|
||||
}
|
||||
|
||||
// nopScraper is a no-op NovelScraper implementation for tests that don't
|
||||
// exercise scraping functionality.
|
||||
type nopScraper struct{}
|
||||
|
||||
func (nopScraper) SourceName() string { return "nop" }
|
||||
func (nopScraper) ScrapeCatalogue(_ context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
|
||||
ch := make(chan scraper.CatalogueEntry)
|
||||
errs := make(chan error)
|
||||
close(ch)
|
||||
close(errs)
|
||||
return ch, errs
|
||||
}
|
||||
func (nopScraper) ScrapeMetadata(_ context.Context, _ string) (scraper.BookMeta, error) {
|
||||
return scraper.BookMeta{}, nil
|
||||
}
|
||||
func (nopScraper) ScrapeChapterList(_ context.Context, _ string) ([]scraper.ChapterRef, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (nopScraper) ScrapeChapterText(_ context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
|
||||
return scraper.Chapter{Ref: ref}, nil
|
||||
}
|
||||
func (nopScraper) ScrapeRanking(_ context.Context, _ int) (<-chan scraper.BookMeta, <-chan error) {
|
||||
ch := make(chan scraper.BookMeta)
|
||||
errs := make(chan error)
|
||||
close(ch)
|
||||
close(errs)
|
||||
return ch, errs
|
||||
}
|
||||
|
||||
// ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// TestServer_Health verifies GET /health returns 200 with status:ok.
|
||||
func TestServer_Health(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
resp, err := http.Get(base + "/health")
|
||||
if err != nil {
|
||||
t.Fatalf("GET /health: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode health body: %v", err)
|
||||
}
|
||||
if body["status"] != "ok" {
|
||||
t.Errorf("status field = %q, want %q", body["status"], "ok")
|
||||
}
|
||||
t.Logf("health response: %v", body)
|
||||
}
|
||||
|
||||
// TestServer_ScrapeStatus verifies GET /api/scrape/status returns running:false
|
||||
// when no scrape is running.
|
||||
func TestServer_ScrapeStatus(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
resp, err := http.Get(base + "/api/scrape/status")
|
||||
if err != nil {
|
||||
t.Fatalf("GET /api/scrape/status: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var body map[string]bool
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
if body["running"] {
|
||||
t.Error("scrape/status.running = true, want false")
|
||||
}
|
||||
t.Logf("scrape status: %v", body)
|
||||
}
|
||||
|
||||
// TestServer_PresignChapter writes a chapter to MinIO, then calls
|
||||
// GET /api/presign/chapter/{slug}/{n} and verifies a URL is returned.
|
||||
func TestServer_PresignChapter(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
// Write a chapter directly via the store so we have something to presign.
|
||||
slug := fmt.Sprintf("server-presign-test-%d", time.Now().UnixMilli()%100000)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ch := scraper.Chapter{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Server Presign Test", Volume: 0},
|
||||
Text: "Content for the server presign integration test.",
|
||||
}
|
||||
if err := store.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter: %v", err)
|
||||
}
|
||||
t.Logf("stored chapter for slug=%q", slug)
|
||||
|
||||
// Call the presign endpoint.
|
||||
url := fmt.Sprintf("%s/api/presign/chapter/%s/1", base, slug)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode presign response: %v", err)
|
||||
}
|
||||
presignedURL := body["url"]
|
||||
if presignedURL == "" {
|
||||
t.Fatal("presign response has empty url field")
|
||||
}
|
||||
if !strings.HasPrefix(presignedURL, "http") {
|
||||
t.Errorf("presigned URL does not start with http: %q", presignedURL)
|
||||
}
|
||||
t.Logf("presigned URL: %s", presignedURL)
|
||||
}
|
||||
|
||||
// TestServer_Progress exercises POST /api/progress/{slug} and GET /api/progress.
|
||||
func TestServer_Progress(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
slug := fmt.Sprintf("server-progress-test-%d", time.Now().UnixMilli()%100000)
|
||||
|
||||
// Use a persistent http.Client to carry the session cookie.
|
||||
jar := &cookieJar{cookies: make(map[string][]*http.Cookie)}
|
||||
client := &http.Client{Jar: jar}
|
||||
|
||||
// POST /api/progress/{slug}
|
||||
setURL := fmt.Sprintf("%s/api/progress/%s", base, slug)
|
||||
body, _ := json.Marshal(map[string]int{"chapter": 5})
|
||||
resp, err := client.Post(setURL, "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("POST %s: %v", setURL, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("POST progress status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
t.Logf("POST /api/progress/%s → %d", slug, resp.StatusCode)
|
||||
|
||||
// GET /api/progress
|
||||
getURL := fmt.Sprintf("%s/api/progress", base)
|
||||
resp2, err := client.Get(getURL)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", getURL, err)
|
||||
}
|
||||
defer resp2.Body.Close()
|
||||
|
||||
if resp2.StatusCode != http.StatusOK {
|
||||
t.Errorf("GET progress status = %d, want 200", resp2.StatusCode)
|
||||
}
|
||||
|
||||
var progress map[string]interface{}
|
||||
if err := json.NewDecoder(resp2.Body).Decode(&progress); err != nil {
|
||||
t.Fatalf("decode progress response: %v", err)
|
||||
}
|
||||
t.Logf("progress: %v", progress)
|
||||
|
||||
// The slug should appear with chapter value 5.
|
||||
if ch, ok := progress[slug]; !ok {
|
||||
t.Errorf("slug %q not found in progress map; keys: %v", slug, mapKeys(progress))
|
||||
} else {
|
||||
// JSON numbers decode as float64.
|
||||
chNum, _ := ch.(float64)
|
||||
if int(chNum) != 5 {
|
||||
t.Errorf("progress[%q] = %v, want 5", slug, ch)
|
||||
}
|
||||
}
|
||||
|
||||
// DELETE /api/progress/{slug}
|
||||
delURL := fmt.Sprintf("%s/api/progress/%s", base, slug)
|
||||
delReq, _ := http.NewRequest(http.MethodDelete, delURL, nil)
|
||||
delResp, err := client.Do(delReq)
|
||||
if err != nil {
|
||||
t.Fatalf("DELETE %s: %v", delURL, err)
|
||||
}
|
||||
delResp.Body.Close()
|
||||
if delResp.StatusCode != http.StatusOK {
|
||||
t.Errorf("DELETE progress status = %d, want 200", delResp.StatusCode)
|
||||
}
|
||||
t.Logf("DELETE /api/progress/%s → %d", slug, delResp.StatusCode)
|
||||
}
|
||||
|
||||
// TestServer_PresignChapter_NotFound verifies that presigning a non-existent
|
||||
// chapter returns 500 (presign fails on missing object).
|
||||
func TestServer_PresignChapter_NotFound(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
url := fmt.Sprintf("%s/api/presign/chapter/does-not-exist-slug/999", base)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// MinIO presign on a non-existent key returns an error; server returns 500.
|
||||
// (Some MinIO versions return a valid presigned URL anyway, which is also acceptable.)
|
||||
t.Logf("presign non-existent chapter status: %d", resp.StatusCode)
|
||||
if resp.StatusCode != http.StatusInternalServerError && resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 500 or 200", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestServer_ChapterText writes a chapter and verifies
|
||||
// GET /api/chapter-text/{slug}/{n} returns the stripped plain text.
|
||||
func TestServer_ChapterText(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
slug := fmt.Sprintf("server-chtext-test-%d", time.Now().UnixMilli()%100000)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
const chapterText = "The quick brown fox jumps over the lazy dog near the river."
|
||||
ch := scraper.Chapter{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Text Test", Volume: 0},
|
||||
Text: chapterText,
|
||||
}
|
||||
if err := store.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter: %v", err)
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/api/chapter-text/%s/1", base, slug)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var buf strings.Builder
|
||||
rawBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read body: %v", err)
|
||||
}
|
||||
buf.Write(rawBytes)
|
||||
text := buf.String()
|
||||
t.Logf("chapter text (%d bytes): %q", len(text), text[:min(len(text), 120)])
|
||||
|
||||
if text == "" {
|
||||
t.Error("chapter-text returned empty body")
|
||||
}
|
||||
// The stripped text should contain our chapter text (markdown heading stripped).
|
||||
if !strings.Contains(text, chapterText) {
|
||||
t.Errorf("chapter text does not contain expected content %q", chapterText)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func mapKeys(m map[string]interface{}) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// cookieJar is a minimal http.CookieJar that stores cookies by host.
|
||||
type cookieJar struct {
|
||||
cookies map[string][]*http.Cookie
|
||||
}
|
||||
|
||||
func (j *cookieJar) SetCookies(u *neturl.URL, cookies []*http.Cookie) {
|
||||
j.cookies[u.Host] = append(j.cookies[u.Host], cookies...)
|
||||
}
|
||||
|
||||
func (j *cookieJar) Cookies(u *neturl.URL) []*http.Cookie {
|
||||
return j.cookies[u.Host]
|
||||
}
|
||||
@@ -1,68 +1,91 @@
|
||||
// Package server exposes the scraper as an HTTP service.
|
||||
// Package server exposes the scraper as an HTTP API service.
|
||||
//
|
||||
// Endpoints:
|
||||
//
|
||||
// POST /scrape — enqueue a full catalogue scrape
|
||||
// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."})
|
||||
// GET /health — liveness probe
|
||||
// POST /scrape — enqueue a full catalogue scrape
|
||||
// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."})
|
||||
// GET /health — liveness probe
|
||||
// GET /api/progress — get reading progress map (session-scoped)
|
||||
// POST /api/progress/{slug} — set reading progress
|
||||
// DELETE /api/progress/{slug} — delete reading progress
|
||||
// GET /api/presign/chapter/{slug}/{n} — presigned MinIO URL for chapter markdown
|
||||
// GET /api/presign/audio/{slug}/{n} — presigned MinIO URL for chapter audio
|
||||
// GET /api/chapter-text/{slug}/{n} — plain text of chapter (markdown stripped)
|
||||
// POST /api/audio/{slug}/{n} — trigger Kokoro audio generation (async, returns 202)
|
||||
// GET /api/audio/status/{slug}/{n} — poll audio generation job status
|
||||
// GET /api/audio-proxy/{slug}/{n} — proxy generated audio from Kokoro
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// Server wraps an HTTP mux with the scraping endpoints.
|
||||
type Server struct {
|
||||
addr string
|
||||
oCfg orchestrator.Config
|
||||
novel scraper.NovelScraper
|
||||
log *slog.Logger
|
||||
writer *writer.Writer
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
rankingRunning bool
|
||||
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
|
||||
kokoroVoice string // default voice, e.g. af_bella
|
||||
addr string
|
||||
oCfg orchestrator.Config
|
||||
novel scraper.NovelScraper
|
||||
log *slog.Logger
|
||||
store storage.Store
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
|
||||
kokoroVoice string // default voice, e.g. af_bella
|
||||
|
||||
// voiceMu guards cachedVoices.
|
||||
voiceMu sync.RWMutex
|
||||
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
|
||||
|
||||
// audioMu guards audioCache and audioInFlight.
|
||||
// audioCache maps a cache key to the Kokoro download filename returned by
|
||||
// POST /v1/audio/speech with return_download_link=true.
|
||||
// audioInFlight deduplicates concurrent generation requests for the same key.
|
||||
audioMu sync.Mutex
|
||||
audioCache map[string]string // cacheKey → kokoro download filename
|
||||
audioInFlight map[string]chan struct{} // cacheKey → closed when done
|
||||
// audioMu guards audioJobIDs only.
|
||||
// Completed audio filenames are persisted to the Store (PocketBase).
|
||||
// audioJobIDs deduplicates concurrent generation requests for the same key.
|
||||
audioMu sync.Mutex
|
||||
audioJobIDs map[string]string // cacheKey → PocketBase job ID (empty string if record creation failed)
|
||||
|
||||
// browseMu guards browseInFlight — keys currently being refreshed
|
||||
// in the background.
|
||||
browseMu sync.Mutex
|
||||
browseInFlight map[string]struct{}
|
||||
|
||||
// browseMemCache is a short-lived in-process cache for browse results.
|
||||
// It is populated whenever a live upstream fetch succeeds and used as a
|
||||
// last-resort fallback when both MinIO and the upstream are unavailable.
|
||||
// Key: the MinIO cache key (same as used for BrowseHTMLKey).
|
||||
browseMemCacheMu sync.RWMutex
|
||||
browseMemCache map[string]browseCacheEntry
|
||||
}
|
||||
|
||||
type browseCacheEntry struct {
|
||||
novels []NovelListing
|
||||
hasNext bool
|
||||
cachedAt time.Time
|
||||
}
|
||||
|
||||
// New creates a new Server.
|
||||
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server {
|
||||
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
|
||||
return &Server{
|
||||
addr: addr,
|
||||
oCfg: oCfg,
|
||||
novel: novel,
|
||||
log: log,
|
||||
writer: writer.New(oCfg.StaticRoot),
|
||||
kokoroURL: kokoroURL,
|
||||
kokoroVoice: kokoroVoice,
|
||||
audioCache: make(map[string]string),
|
||||
audioInFlight: make(map[string]chan struct{}),
|
||||
addr: addr,
|
||||
oCfg: oCfg,
|
||||
novel: novel,
|
||||
log: log,
|
||||
store: store,
|
||||
kokoroURL: kokoroURL,
|
||||
kokoroVoice: kokoroVoice,
|
||||
audioJobIDs: make(map[string]string),
|
||||
browseInFlight: make(map[string]struct{}),
|
||||
browseMemCache: make(map[string]browseCacheEntry),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,31 +101,36 @@ func (s *Server) voices() []string {
|
||||
return cached
|
||||
}
|
||||
|
||||
if s.kokoroURL != "" {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
|
||||
if err == nil {
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err == nil {
|
||||
defer resp.Body.Close()
|
||||
var payload struct {
|
||||
Voices []string `json:"voices"`
|
||||
}
|
||||
if resp.StatusCode == http.StatusOK && json.NewDecoder(resp.Body).Decode(&payload) == nil && len(payload.Voices) > 0 {
|
||||
s.voiceMu.Lock()
|
||||
s.cachedVoices = payload.Voices
|
||||
s.voiceMu.Unlock()
|
||||
s.log.Info("fetched kokoro voices", "count", len(payload.Voices))
|
||||
return payload.Voices
|
||||
}
|
||||
}
|
||||
}
|
||||
s.log.Warn("could not fetch kokoro voices, using built-in list")
|
||||
if s.kokoroURL == "" {
|
||||
return kokoroVoices
|
||||
}
|
||||
|
||||
return kokoroVoices
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
|
||||
if err != nil {
|
||||
s.log.Warn("could not fetch kokoro voices, using built-in list", "err", err)
|
||||
return kokoroVoices
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
s.log.Warn("could not fetch kokoro voices, using built-in list", "err", err)
|
||||
return kokoroVoices
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var payload struct {
|
||||
Voices []string `json:"voices"`
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK || json.NewDecoder(resp.Body).Decode(&payload) != nil || len(payload.Voices) == 0 {
|
||||
s.log.Warn("could not fetch kokoro voices, using built-in list")
|
||||
return kokoroVoices
|
||||
}
|
||||
s.voiceMu.Lock()
|
||||
s.cachedVoices = payload.Voices
|
||||
s.voiceMu.Unlock()
|
||||
s.log.Info("fetched kokoro voices", "count", len(payload.Voices))
|
||||
return payload.Voices
|
||||
}
|
||||
|
||||
// ListenAndServe starts the HTTP server and blocks until the provided context
|
||||
@@ -112,30 +140,53 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("GET /health", s.handleHealth)
|
||||
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
|
||||
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
||||
// UI routes
|
||||
mux.HandleFunc("GET /", s.handleHome)
|
||||
mux.HandleFunc("GET /scrape", s.handleScrape)
|
||||
mux.HandleFunc("GET /ranking", s.handleRanking)
|
||||
mux.HandleFunc("POST /ranking/refresh", s.handleRankingRefresh)
|
||||
mux.HandleFunc("GET /ranking/view", s.handleRankingView)
|
||||
mux.HandleFunc("GET /books/{slug}", s.handleBook)
|
||||
mux.HandleFunc("GET /books/{slug}/chapters/{n}", s.handleChapter)
|
||||
mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage)
|
||||
mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
|
||||
mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
|
||||
mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus)
|
||||
// Plain-text chapter content for browser-side TTS
|
||||
mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText)
|
||||
// Server-side audio generation via Kokoro /v1/audio/speech.
|
||||
// Generation can take several minutes, so wrap in its own timeout handler.
|
||||
audioGenHandler := http.TimeoutHandler(
|
||||
http.HandlerFunc(s.handleAudioGenerate),
|
||||
10*time.Minute,
|
||||
`{"error":"audio generation timed out"}`,
|
||||
mux.HandleFunc("POST /scrape/book/range", s.handleScrapeBookRange)
|
||||
// Browse API — fetches and parses novelfire catalogue page
|
||||
mux.HandleFunc("GET /api/browse", s.handleBrowse)
|
||||
// Ranking API
|
||||
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
|
||||
// Cover image proxy (serves images stored in browse MinIO bucket)
|
||||
mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
|
||||
// Scrape status
|
||||
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
|
||||
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
|
||||
// Re-index chapters for a book from MinIO into PocketBase chapters_idx
|
||||
mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
|
||||
// On-demand preview (no store writes) — for books not yet in the library
|
||||
mux.HandleFunc("GET /api/book-preview/{slug}", s.handleBookPreview)
|
||||
mux.HandleFunc("GET /api/chapter-text-preview/{slug}/{n}", s.handleChapterTextPreview)
|
||||
// Search: local PocketBase + remote novelfire.net
|
||||
mux.HandleFunc("GET /api/search", s.handleSearch)
|
||||
// Progress API
|
||||
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
||||
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
||||
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
|
||||
// Presigned URL API (for SvelteKit UI)
|
||||
mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter)
|
||||
mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio)
|
||||
mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample)
|
||||
mux.HandleFunc("GET /api/presign/avatar-upload/{userId}", s.handlePresignAvatarUpload)
|
||||
mux.HandleFunc("GET /api/presign/avatar/{userId}", s.handlePresignAvatar)
|
||||
// Plain-text chapter content (used server-side for audio generation)
|
||||
mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText)
|
||||
// Voices list (proxied from Kokoro)
|
||||
mux.HandleFunc("GET /api/voices", s.handleVoices)
|
||||
// Voice sample generation — generates a short audio clip for each voice
|
||||
// and stores it in MinIO for UI preview playback.
|
||||
voiceSampleHandler := http.TimeoutHandler(
|
||||
http.HandlerFunc(s.handleGenerateVoiceSamples),
|
||||
15*time.Minute,
|
||||
`{"error":"voice sample generation timed out"}`,
|
||||
)
|
||||
mux.Handle("POST /ui/audio/{slug}/{n}", audioGenHandler)
|
||||
mux.Handle("POST /api/audio/voice-samples", voiceSampleHandler)
|
||||
// Server-side audio generation via Kokoro /v1/audio/speech.
|
||||
// POST returns 202 immediately and starts a background goroutine;
|
||||
// poll GET /api/audio/status/{slug}/{n} to track progress.
|
||||
mux.HandleFunc("POST /api/audio/{slug}/{n}", s.handleAudioGenerate)
|
||||
// Audio job status polling endpoint.
|
||||
mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus)
|
||||
// Proxy route: fetches the generated file from Kokoro /v1/download/{filename}.
|
||||
mux.HandleFunc("GET /ui/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
||||
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: s.addr,
|
||||
@@ -150,6 +201,15 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
|
||||
s.log.Info("HTTP server listening", "addr", s.addr)
|
||||
|
||||
// Pre-populate voice samples in the background so the UI voice selector
|
||||
// has playable previews without requiring a manual trigger.
|
||||
go s.warmVoiceSamples(ctx)
|
||||
|
||||
// Warm the browse cache on startup: if page 1 is not cached in MinIO yet,
|
||||
// trigger a background SingleFile snapshot immediately so the first user
|
||||
// request is served from cache rather than hitting novelfire.net live.
|
||||
go s.warmBrowseCache()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
@@ -165,306 +225,46 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// handleChapterText returns the plain text of a chapter (markdown stripped)
|
||||
// for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI.
|
||||
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
raw, err := s.writer.ReadChapter(slug, n)
|
||||
// ─── Session cookie helpers ───────────────────────────────────────────────────
|
||||
|
||||
const sessionCookieName = "libnovel_session"
|
||||
|
||||
// sessionID returns the session ID from the request cookie, or "" if absent.
|
||||
func sessionID(r *http.Request) string {
|
||||
c, err := r.Cookie(sessionCookieName)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
return ""
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
fmt.Fprint(w, stripMarkdown(raw))
|
||||
return c.Value
|
||||
}
|
||||
|
||||
// ─── Audio generation via Kokoro /v1/audio/speech ────────────────────────────
|
||||
//
|
||||
// handleAudioGenerate handles POST /ui/audio/{slug}/{n}.
|
||||
//
|
||||
// It calls Kokoro's POST /v1/audio/speech with return_download_link=true.
|
||||
// Kokoro generates the audio, saves it to its own temp storage, and returns
|
||||
// the download filename in the X-Download-Path response header.
|
||||
// We cache that filename (in memory, keyed by slug/chapter/voice/speed) and
|
||||
// return a proxy URL that the browser sets as audio.src.
|
||||
//
|
||||
// On a cache hit the proxy URL is returned immediately without re-generating.
|
||||
// Concurrent requests for the same key are deduplicated.
|
||||
func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 {
|
||||
http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest)
|
||||
return
|
||||
// newSessionID generates a random 16-byte hex session ID.
|
||||
func newSessionID() (string, error) {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Parse optional voice/speed from JSON body.
|
||||
voice := s.kokoroVoice
|
||||
speed := 1.0
|
||||
var body struct {
|
||||
Voice string `json:"voice"`
|
||||
Speed float64 `json:"speed"`
|
||||
}
|
||||
if r.Body != nil {
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
}
|
||||
if body.Voice != "" {
|
||||
voice = body.Voice
|
||||
}
|
||||
if body.Speed > 0 {
|
||||
speed = body.Speed
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
|
||||
|
||||
// Fast path: already generated this session.
|
||||
s.audioMu.Lock()
|
||||
if filename, ok := s.audioCache[cacheKey]; ok {
|
||||
s.audioMu.Unlock()
|
||||
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
||||
return
|
||||
}
|
||||
|
||||
// Deduplicate concurrent generation for the same key.
|
||||
if ch, ok := s.audioInFlight[cacheKey]; ok {
|
||||
s.audioMu.Unlock()
|
||||
select {
|
||||
case <-ch:
|
||||
case <-r.Context().Done():
|
||||
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
s.audioMu.Lock()
|
||||
filename, ok := s.audioCache[cacheKey]
|
||||
s.audioMu.Unlock()
|
||||
if ok {
|
||||
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
||||
} else {
|
||||
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
|
||||
}
|
||||
return
|
||||
}
|
||||
ch := make(chan struct{})
|
||||
s.audioInFlight[cacheKey] = ch
|
||||
s.audioMu.Unlock()
|
||||
|
||||
defer func() {
|
||||
s.audioMu.Lock()
|
||||
delete(s.audioInFlight, cacheKey)
|
||||
s.audioMu.Unlock()
|
||||
close(ch)
|
||||
}()
|
||||
|
||||
// Load and validate chapter text.
|
||||
raw, err := s.writer.ReadChapter(slug, n)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
text := stripMarkdown(raw)
|
||||
if text == "" {
|
||||
http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity)
|
||||
return
|
||||
}
|
||||
if s.kokoroURL == "" {
|
||||
http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Call Kokoro POST /v1/audio/speech with return_download_link=true.
|
||||
// Kokoro saves the generated audio to its own temp storage and returns the
|
||||
// download path in the X-Download-Path response header.
|
||||
filename, err := s.generateSpeech(r.Context(), text, voice, speed)
|
||||
if err != nil {
|
||||
s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
|
||||
http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
s.audioMu.Lock()
|
||||
s.audioCache[cacheKey] = filename
|
||||
s.audioMu.Unlock()
|
||||
|
||||
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
|
||||
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
// generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true
|
||||
// and returns the filename from the X-Download-Path response header.
|
||||
func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) {
|
||||
reqBody, _ := json.Marshal(map[string]interface{}{
|
||||
"model": "kokoro",
|
||||
"input": text,
|
||||
"voice": voice,
|
||||
"response_format": "mp3",
|
||||
"speed": speed,
|
||||
"stream": false,
|
||||
"return_download_link": true,
|
||||
// ensureSession issues a new session cookie if the request does not already
|
||||
// carry one, and returns the session ID (either existing or newly issued).
|
||||
func ensureSession(w http.ResponseWriter, r *http.Request) string {
|
||||
if id := sessionID(r); id != "" {
|
||||
return id
|
||||
}
|
||||
id, err := newSessionID()
|
||||
if err != nil {
|
||||
// Very unlikely, but fall back to a timestamp-based ID.
|
||||
id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
|
||||
}
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: sessionCookieName,
|
||||
Value: id,
|
||||
Path: "/",
|
||||
HttpOnly: true,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
MaxAge: 365 * 24 * 60 * 60, // 1 year
|
||||
})
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("kokoro request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
// Drain body so the connection can be reused.
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("kokoro status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// X-Download-Path is e.g. "/download/speech_abc123.mp3"
|
||||
dlPath := resp.Header.Get("X-Download-Path")
|
||||
if dlPath == "" {
|
||||
return "", fmt.Errorf("kokoro did not return X-Download-Path header")
|
||||
}
|
||||
|
||||
// Extract just the filename from the path.
|
||||
filename := dlPath
|
||||
if idx := strings.LastIndex(dlPath, "/"); idx >= 0 {
|
||||
filename = dlPath[idx+1:]
|
||||
}
|
||||
if filename == "" {
|
||||
return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath)
|
||||
}
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
// writeAudioResponse writes the JSON response for a generated audio chapter.
|
||||
// The URL points to our proxy handler which fetches from Kokoro on demand.
|
||||
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) {
|
||||
proxyURL := fmt.Sprintf("/ui/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"url": proxyURL,
|
||||
"filename": filename,
|
||||
})
|
||||
}
|
||||
|
||||
// handleAudioProxy handles GET /ui/audio-proxy/{slug}/{n}.
|
||||
// It looks up the Kokoro download filename for this chapter (voice/speed) and
|
||||
// proxies GET /v1/download/{filename} from the Kokoro server back to the browser.
|
||||
func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
voice := r.URL.Query().Get("voice")
|
||||
if voice == "" {
|
||||
voice = s.kokoroVoice
|
||||
}
|
||||
speedStr := r.URL.Query().Get("speed")
|
||||
speed := 1.0
|
||||
if speedStr != "" {
|
||||
if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 {
|
||||
speed = v
|
||||
}
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
|
||||
s.audioMu.Lock()
|
||||
filename, ok := s.audioCache[cacheKey]
|
||||
s.audioMu.Unlock()
|
||||
|
||||
if !ok {
|
||||
http.Error(w, "audio not generated yet", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
kokoroURL := s.kokoroURL + "/v1/download/" + filename
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to build proxy request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
http.Error(w, "kokoro download failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "audio/mpeg")
|
||||
w.Header().Set("Cache-Control", "public, max-age=3600")
|
||||
if cl := resp.Header.Get("Content-Length"); cl != "" {
|
||||
w.Header().Set("Content-Length", cl)
|
||||
}
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
|
||||
cfg := s.oCfg
|
||||
cfg.SingleBookURL = "" // full catalogue
|
||||
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
||||
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := s.oCfg
|
||||
cfg.SingleBookURL = body.URL
|
||||
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
// runAsync launches an orchestrator in the background and returns 202 Accepted.
|
||||
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
|
||||
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
||||
s.mu.Lock()
|
||||
if s.running {
|
||||
s.mu.Unlock()
|
||||
http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
|
||||
return
|
||||
}
|
||||
s.running = true
|
||||
s.mu.Unlock()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
s.running = false
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
|
||||
defer cancel()
|
||||
|
||||
o := orchestrator.New(cfg, s.novel, s.log)
|
||||
if err := o.Run(ctx); err != nil {
|
||||
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err))
|
||||
}
|
||||
}()
|
||||
return id
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
59
scraper/internal/storage/coverutil.go
Normal file
59
scraper/internal/storage/coverutil.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DownloadAndStoreCover fetches the image at imageURL and stores it in the
|
||||
// store under key. Errors are logged but not returned — this is best-effort.
|
||||
// If the asset is already present the download is skipped.
|
||||
func DownloadAndStoreCover(store Store, log *slog.Logger, key, imageURL string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Skip if already stored.
|
||||
if _, _, ok, _ := store.GetBrowseAsset(ctx, key); ok {
|
||||
return
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
|
||||
if err != nil {
|
||||
log.Warn("cover: build request failed", "key", key, "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
log.Warn("cover: fetch failed", "key", key, "url", imageURL, "err", fmt.Errorf("%w", err))
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Warn("cover: non-200 response", "key", key, "url", imageURL, "status", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Warn("cover: read body failed", "key", key, "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
|
||||
if err := store.SaveBrowseAsset(ctx, key, data, contentType); err != nil {
|
||||
log.Warn("cover: SaveBrowseAsset failed", "key", key, "err", err)
|
||||
return
|
||||
}
|
||||
log.Debug("cover: stored", "key", key, "bytes", len(data))
|
||||
}
|
||||
560
scraper/internal/storage/hybrid.go
Normal file
560
scraper/internal/storage/hybrid.go
Normal file
@@ -0,0 +1,560 @@
|
||||
// hybrid.go implements the Store interface using PocketBase for structured data
|
||||
// and MinIO for binary chapter/audio blobs.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// HybridStore satisfies Store by routing structured data to PocketBase and
|
||||
// binary objects (chapters, audio) to MinIO.
|
||||
type HybridStore struct {
|
||||
pb *PocketBaseStore
|
||||
minio *MinioClient
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewHybridStore constructs a HybridStore. It connects to both backends and
|
||||
// calls EnsureCollections to bootstrap any missing PocketBase collections.
|
||||
func NewHybridStore(ctx context.Context, pbCfg PocketBaseConfig, minioCfg MinioConfig, log *slog.Logger) (*HybridStore, error) {
|
||||
mc, err := NewMinioClient(ctx, minioCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: minio: %w", err)
|
||||
}
|
||||
pb := NewPocketBaseStore(pbCfg, log)
|
||||
// Verify PocketBase credentials before proceeding.
|
||||
if err := pb.Ping(ctx); err != nil {
|
||||
return nil, fmt.Errorf("storage: pocketbase auth: %w", err)
|
||||
}
|
||||
if err := pb.EnsureCollections(ctx); err != nil {
|
||||
// Non-fatal: 400/422 means collections already exist.
|
||||
log.Warn("EnsureCollections returned an error (may be safe to ignore)", "err", err)
|
||||
}
|
||||
if err := pb.EnsureMigrations(ctx); err != nil {
|
||||
log.Warn("EnsureMigrations returned an error", "err", err)
|
||||
}
|
||||
return &HybridStore{pb: pb, minio: mc, log: log}, nil
|
||||
}
|
||||
|
||||
// ─── Book metadata ────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteMetadata(ctx context.Context, meta scraper.BookMeta) error {
|
||||
return h.pb.UpsertBook(ctx,
|
||||
meta.Slug, meta.Title, meta.Author, meta.Cover,
|
||||
meta.Status, meta.Summary, meta.SourceURL,
|
||||
meta.Genres, meta.TotalChapters, meta.Ranking,
|
||||
)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error) {
|
||||
rec, found, err := h.pb.GetBook(ctx, slug)
|
||||
if err != nil || !found {
|
||||
return scraper.BookMeta{}, found, err
|
||||
}
|
||||
return recToBookMeta(rec), true, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListBooks(ctx context.Context) ([]scraper.BookMeta, error) {
|
||||
rows, err := h.pb.ListBooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
books := make([]scraper.BookMeta, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
books = append(books, recToBookMeta(r))
|
||||
}
|
||||
return books, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) LocalSlugs(ctx context.Context) (map[string]bool, error) {
|
||||
books, err := h.ListBooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slugs := make(map[string]bool, len(books))
|
||||
for _, b := range books {
|
||||
slugs[b.Slug] = true
|
||||
}
|
||||
return slugs, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) MetadataMtime(ctx context.Context, slug string) int64 {
|
||||
t, err := h.pb.BookMetaUpdated(ctx, slug)
|
||||
if err != nil {
|
||||
h.log.Warn("MetadataMtime: BookMetaUpdated failed", "slug", slug, "err", err)
|
||||
return 0
|
||||
}
|
||||
if t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
// ─── Chapters ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool {
|
||||
return h.minio.ChapterExists(ctx, slug, ref.Volume, ref.Number)
|
||||
}
|
||||
|
||||
func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error {
|
||||
content := "# " + chapter.Ref.Title + "\n\n" + chapter.Text + "\n"
|
||||
if err := h.minio.PutChapter(ctx, slug, chapter.Ref.Volume, chapter.Ref.Number, content); err != nil {
|
||||
return err
|
||||
}
|
||||
// Update chapter index in PocketBase.
|
||||
title, dateLabel := splitChapterTitle(chapter.Ref.Title)
|
||||
if err := h.pb.UpsertChapterIdx(ctx, slug, chapter.Ref.Number, title, dateLabel); err != nil {
|
||||
h.log.Warn("WriteChapter: failed to upsert chapter index in PocketBase",
|
||||
"slug", slug, "chapter", chapter.Ref.Number, "err", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteChapterRefs upserts chapter index rows (number + title) for all refs
|
||||
// without writing any chapter text to MinIO. This pre-populates the chapter
|
||||
// list when a book is first seen via a live preview.
|
||||
func (h *HybridStore) WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error {
|
||||
return h.pb.WriteChapterRefs(ctx, slug, refs)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
|
||||
return h.minio.GetChapter(ctx, slug, 0, n)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) {
|
||||
rows, err := h.pb.ListChapterIdx(ctx, slug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
infos := make([]ChapterInfo, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
n := int(floatVal(r, "number"))
|
||||
title, _ := r["title"].(string)
|
||||
date, _ := r["date_label"].(string)
|
||||
infos = append(infos, ChapterInfo{Number: n, Title: title, Date: date})
|
||||
}
|
||||
sort.Slice(infos, func(i, j int) bool { return infos[i].Number < infos[j].Number })
|
||||
return infos, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) CountChapters(ctx context.Context, slug string) int {
|
||||
return h.pb.CountChapterIdx(ctx, slug)
|
||||
}
|
||||
|
||||
// ReindexChapters walks all MinIO objects for slug, reads the title from the
|
||||
// first line of each chapter markdown, and upserts them into chapters_idx.
|
||||
// This repairs the PocketBase index when it falls out of sync with MinIO.
|
||||
// Returns the number of chapters indexed and any non-fatal errors encountered.
|
||||
func (h *HybridStore) ReindexChapters(ctx context.Context, slug string) (int, error) {
|
||||
keys, err := h.minio.ListChapterKeys(ctx, slug)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("reindex: list chapter keys: %w", err)
|
||||
}
|
||||
|
||||
count := 0
|
||||
var errs []string
|
||||
for _, key := range keys {
|
||||
// Parse chapter number from key: {slug}/vol-N/lo-hi/chapter-N.md
|
||||
n := chapterNumberFromKey(key)
|
||||
if n <= 0 {
|
||||
h.log.Warn("ReindexChapters: could not parse chapter number from key", "key", key)
|
||||
continue
|
||||
}
|
||||
|
||||
raw, readErr := h.minio.GetChapter(ctx, slug, 0, n)
|
||||
if readErr != nil {
|
||||
errs = append(errs, fmt.Sprintf("ch%d: %v", n, readErr))
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract title from first line ("# Title text") or fall back to empty.
|
||||
rawTitle := ""
|
||||
if line, _, found := strings.Cut(raw, "\n"); found || raw != "" {
|
||||
rawTitle = strings.TrimPrefix(strings.TrimSpace(line), "# ")
|
||||
}
|
||||
title, dateLabel := splitChapterTitle(rawTitle)
|
||||
|
||||
if upsertErr := h.pb.UpsertChapterIdx(ctx, slug, n, title, dateLabel); upsertErr != nil {
|
||||
errs = append(errs, fmt.Sprintf("ch%d upsert: %v", n, upsertErr))
|
||||
continue
|
||||
}
|
||||
count++
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
return count, fmt.Errorf("reindex: %d error(s): %s", len(errs), strings.Join(errs, "; "))
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// chapterNumberFromKey parses the chapter number from a MinIO object key of the
|
||||
// form "{slug}/vol-N/lo-hi/chapter-N.md".
|
||||
func chapterNumberFromKey(key string) int {
|
||||
// Grab the filename portion after the last '/'.
|
||||
parts := strings.Split(key, "/")
|
||||
if len(parts) == 0 {
|
||||
return 0
|
||||
}
|
||||
filename := parts[len(parts)-1]
|
||||
// filename is "chapter-N.md"
|
||||
filename = strings.TrimSuffix(filename, ".md")
|
||||
filename = strings.TrimPrefix(filename, "chapter-")
|
||||
n, err := strconv.Atoi(filename)
|
||||
if err != nil || n <= 0 {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ─── Ranking ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteRankingItem(ctx context.Context, item RankingItem) error {
|
||||
return h.pb.UpsertRankingItem(ctx, item)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadRankingItems(ctx context.Context) ([]RankingItem, error) {
|
||||
return h.pb.ListRankingItems(ctx)
|
||||
}
|
||||
|
||||
func (h *HybridStore) RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) {
|
||||
last, err := h.pb.RankingLastUpdated(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if last.IsZero() {
|
||||
return false, nil
|
||||
}
|
||||
return time.Since(last) < maxAge, nil
|
||||
}
|
||||
|
||||
// ─── Audio cache ──────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool) {
|
||||
filename, ok, err := h.pb.GetAudioCache(ctx, cacheKey)
|
||||
if err != nil {
|
||||
h.log.Warn("GetAudioCache: PocketBase lookup failed", "cache_key", cacheKey, "err", err)
|
||||
}
|
||||
return filename, ok
|
||||
}
|
||||
|
||||
func (h *HybridStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error {
|
||||
return h.pb.SetAudioCache(ctx, cacheKey, filename)
|
||||
}
|
||||
|
||||
// ─── Reading progress ─────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool) {
|
||||
ch, updated, ok, err := h.pb.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil {
|
||||
h.log.Warn("GetProgress: PocketBase lookup failed", "slug", slug, "err", err)
|
||||
return ReadingProgress{}, false
|
||||
}
|
||||
if !ok {
|
||||
return ReadingProgress{}, false
|
||||
}
|
||||
return ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}, true
|
||||
}
|
||||
|
||||
func (h *HybridStore) SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error {
|
||||
return h.pb.SetProgress(ctx, sessionID, p.Slug, p.Chapter)
|
||||
}
|
||||
|
||||
func (h *HybridStore) AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error) {
|
||||
rows, err := h.pb.AllProgress(ctx, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]ReadingProgress, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
slug, _ := r["slug"].(string)
|
||||
ch := int(floatVal(r, "chapter"))
|
||||
var updated time.Time
|
||||
if ts, ok := r["updated"].(string); ok {
|
||||
updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
out = append(out, ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) DeleteProgress(ctx context.Context, sessionID, slug string) error {
|
||||
return h.pb.DeleteProgress(ctx, sessionID, slug)
|
||||
}
|
||||
|
||||
// ─── AudioObjectKey ───────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) AudioObjectKey(slug string, n int, voice string) string {
|
||||
return AudioObjectKey(slug, n, voice)
|
||||
}
|
||||
|
||||
func (h *HybridStore) AudioExists(ctx context.Context, key string) bool {
|
||||
return h.minio.AudioExists(ctx, key)
|
||||
}
|
||||
|
||||
// ─── PutAudio ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) PutAudio(ctx context.Context, key string, data []byte) error {
|
||||
return h.minio.PutAudio(ctx, key, data)
|
||||
}
|
||||
|
||||
// ─── Presigned URLs ───────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) {
|
||||
return h.minio.PresignChapter(ctx, slug, 0, n, expires)
|
||||
}
|
||||
|
||||
func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
|
||||
return h.minio.PresignAudio(ctx, key, expires)
|
||||
}
|
||||
|
||||
func (h *HybridStore) PresignAvatarUpload(ctx context.Context, userID, ext string) (string, string, error) {
|
||||
return h.minio.PresignAvatarUploadURL(ctx, userID, ext)
|
||||
}
|
||||
|
||||
func (h *HybridStore) PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) {
|
||||
return h.minio.PresignAvatarURL(ctx, userID)
|
||||
}
|
||||
|
||||
func (h *HybridStore) DeleteAvatar(ctx context.Context, userID string) error {
|
||||
return h.minio.DeleteAvatar(ctx, userID)
|
||||
}
|
||||
|
||||
// ─── Browse page snapshots ────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) SaveBrowsePage(ctx context.Context, key, html string) error {
|
||||
return h.minio.PutBrowsePage(ctx, key, html)
|
||||
}
|
||||
|
||||
func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bool, error) {
|
||||
return h.minio.GetBrowsePage(ctx, key)
|
||||
}
|
||||
|
||||
func (h *HybridStore) BrowseHTMLKey(domain string, page int) string {
|
||||
return BrowseHTMLKey(domain, page)
|
||||
}
|
||||
|
||||
func (h *HybridStore) BrowseFilteredHTMLKey(domain string, page int, sort, genre, status string) string {
|
||||
return BrowseFilteredHTMLKey(domain, page, sort, genre, status)
|
||||
}
|
||||
|
||||
func (h *HybridStore) BrowseCoverKey(domain, slug string) string {
|
||||
return BrowseCoverKey(domain, slug)
|
||||
}
|
||||
|
||||
func (h *HybridStore) SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
|
||||
return h.minio.PutBrowseAsset(ctx, key, data, contentType)
|
||||
}
|
||||
|
||||
func (h *HybridStore) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
|
||||
return h.minio.GetBrowseAsset(ctx, key)
|
||||
}
|
||||
|
||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) {
|
||||
return h.pb.CreateScrapingTask(ctx, kind, targetURL)
|
||||
}
|
||||
|
||||
func (h *HybridStore) UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error {
|
||||
data := map[string]interface{}{
|
||||
"status": u.Status,
|
||||
"books_found": u.BooksFound,
|
||||
"chapters_scraped": u.ChaptersScraped,
|
||||
"chapters_skipped": u.ChaptersSkipped,
|
||||
"errors": u.Errors,
|
||||
"error_message": u.ErrorMessage,
|
||||
}
|
||||
if !u.Finished.IsZero() {
|
||||
data["finished"] = u.Finished.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return h.pb.UpdateScrapingTask(ctx, id, data)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error) {
|
||||
rows, err := h.pb.ListScrapingTasks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tasks := make([]ScrapeTask, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
t := ScrapeTask{
|
||||
ID: strVal(r, "id"),
|
||||
Kind: strVal(r, "kind"),
|
||||
TargetURL: strVal(r, "target_url"),
|
||||
Status: strVal(r, "status"),
|
||||
BooksFound: int(floatVal(r, "books_found")),
|
||||
ChaptersScraped: int(floatVal(r, "chapters_scraped")),
|
||||
ChaptersSkipped: int(floatVal(r, "chapters_skipped")),
|
||||
Errors: int(floatVal(r, "errors")),
|
||||
ErrorMessage: strVal(r, "error_message"),
|
||||
}
|
||||
if ts, ok := r["started"].(string); ok {
|
||||
t.Started, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
if ts, ok := r["finished"].(string); ok && ts != "" {
|
||||
t.Finished, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
// ─── Audio jobs ───────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) CreateAudioJob(ctx context.Context, slug string, chapter int, voice string) (string, error) {
|
||||
return h.pb.CreateAudioJob(ctx, slug, chapter, voice)
|
||||
}
|
||||
|
||||
func (h *HybridStore) UpdateAudioJob(ctx context.Context, id, status, errMsg string, finished time.Time) error {
|
||||
return h.pb.UpdateAudioJob(ctx, id, status, errMsg, finished)
|
||||
}
|
||||
|
||||
func (h *HybridStore) GetAudioJob(ctx context.Context, cacheKey string) (AudioJob, bool, error) {
|
||||
rec, ok, err := h.pb.GetAudioJob(ctx, cacheKey)
|
||||
if err != nil || !ok {
|
||||
return AudioJob{}, ok, err
|
||||
}
|
||||
job := AudioJob{
|
||||
ID: strVal(rec, "id"),
|
||||
CacheKey: strVal(rec, "cache_key"),
|
||||
Slug: strVal(rec, "slug"),
|
||||
Chapter: int(floatVal(rec, "chapter")),
|
||||
Voice: strVal(rec, "voice"),
|
||||
Status: strVal(rec, "status"),
|
||||
ErrorMessage: strVal(rec, "error_message"),
|
||||
}
|
||||
if ts, ok := rec["started"].(string); ok {
|
||||
job.Started, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
if ts, ok := rec["finished"].(string); ok && ts != "" {
|
||||
job.Finished, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
return job, true, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListAudioJobs(ctx context.Context) ([]AudioJob, error) {
|
||||
rows, err := h.pb.ListAudioJobs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jobs := make([]AudioJob, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
job := AudioJob{
|
||||
ID: strVal(r, "id"),
|
||||
CacheKey: strVal(r, "cache_key"),
|
||||
Slug: strVal(r, "slug"),
|
||||
Chapter: int(floatVal(r, "chapter")),
|
||||
Voice: strVal(r, "voice"),
|
||||
Status: strVal(r, "status"),
|
||||
ErrorMessage: strVal(r, "error_message"),
|
||||
}
|
||||
if ts, ok := r["started"].(string); ok {
|
||||
job.Started, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
if ts, ok := r["finished"].(string); ok && ts != "" {
|
||||
job.Finished, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func recToBookMeta(rec map[string]interface{}) scraper.BookMeta {
|
||||
m := scraper.BookMeta{
|
||||
Slug: strVal(rec, "slug"),
|
||||
Title: strVal(rec, "title"),
|
||||
Author: strVal(rec, "author"),
|
||||
Cover: strVal(rec, "cover"),
|
||||
Status: strVal(rec, "status"),
|
||||
Summary: strVal(rec, "summary"),
|
||||
SourceURL: strVal(rec, "source_url"),
|
||||
}
|
||||
if tc := floatVal(rec, "total_chapters"); tc > 0 {
|
||||
m.TotalChapters = int(tc)
|
||||
}
|
||||
if rk := floatVal(rec, "ranking"); rk > 0 {
|
||||
m.Ranking = int(rk)
|
||||
}
|
||||
// Genres stored as JSON string or array.
|
||||
switch v := rec["genres"].(type) {
|
||||
case string:
|
||||
_ = json.Unmarshal([]byte(v), &m.Genres)
|
||||
case []interface{}:
|
||||
for _, g := range v {
|
||||
if s, ok := g.(string); ok {
|
||||
m.Genres = append(m.Genres, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func strVal(m map[string]interface{}, key string) string {
|
||||
if v, ok := m[key].(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// splitChapterTitle mirrors writer.SplitChapterTitle logic (simplified).
|
||||
func splitChapterTitle(raw string) (title, date string) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
// Strip leading numeric index.
|
||||
if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
|
||||
prefix := raw[:idx]
|
||||
allDigit := true
|
||||
for _, c := range prefix {
|
||||
if c < '0' || c > '9' {
|
||||
allDigit = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allDigit {
|
||||
raw = strings.TrimSpace(raw[idx:])
|
||||
}
|
||||
}
|
||||
// Detect trailing relative date. Build a flat list of all suffixes once
|
||||
// to avoid a double-nested loop.
|
||||
units := []string{"second", "minute", "hour", "day", "week", "month", "year"}
|
||||
suffixes := make([]string, 0, len(units)*2)
|
||||
for _, u := range units {
|
||||
suffixes = append(suffixes, u+"s ago", u+" ago")
|
||||
}
|
||||
lower := strings.ToLower(raw)
|
||||
for _, suffix := range suffixes {
|
||||
idx := strings.LastIndex(lower, suffix)
|
||||
if idx <= 0 {
|
||||
continue
|
||||
}
|
||||
// Find start of the numeric token that precedes the unit.
|
||||
// Strip any whitespace that separates the number from the unit so
|
||||
// that LastIndex finds the space before the digit, not the one
|
||||
// between the digit and the unit word.
|
||||
before := strings.TrimRight(raw[:idx], " \t")
|
||||
start := strings.LastIndex(before, " ")
|
||||
if start < 0 {
|
||||
start = 0
|
||||
} else {
|
||||
start++ // advance past the space to point at the digit
|
||||
}
|
||||
numPart := strings.TrimSpace(raw[start:idx])
|
||||
fields := strings.Fields(numPart)
|
||||
if len(fields) > 0 {
|
||||
if _, err := strconv.Atoi(fields[0]); err == nil {
|
||||
return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)])
|
||||
}
|
||||
}
|
||||
}
|
||||
return raw, ""
|
||||
}
|
||||
473
scraper/internal/storage/hybrid_integration_test.go
Normal file
473
scraper/internal/storage/hybrid_integration_test.go
Normal file
@@ -0,0 +1,473 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests for HybridStore (PocketBase + MinIO) end-to-end.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 120s \
|
||||
// github.com/libnovel/scraper/internal/storage
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// newTestHybridStore constructs a HybridStore from environment variables.
|
||||
// Skips the test if either MINIO_ENDPOINT or POCKETBASE_URL is unset.
|
||||
func newTestHybridStore(t *testing.T) *HybridStore {
|
||||
t.Helper()
|
||||
if ep := envOr("MINIO_ENDPOINT", ""); ep == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping HybridStore integration test")
|
||||
}
|
||||
if u := envOr("POCKETBASE_URL", ""); u == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping HybridStore integration test")
|
||||
}
|
||||
|
||||
pbCfg := PocketBaseConfig{
|
||||
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||
}
|
||||
minioCfg := MinioConfig{
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: envOr("MINIO_USE_SSL", "false") == "true",
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
hs, err := NewHybridStore(ctx, pbCfg, minioCfg, slog.Default())
|
||||
if err != nil {
|
||||
t.Fatalf("NewHybridStore: %v", err)
|
||||
}
|
||||
return hs
|
||||
}
|
||||
|
||||
// ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// TestHybridStore_WriteReadMetadata exercises WriteMetadata → ReadMetadata round-trip.
|
||||
func TestHybridStore_WriteReadMetadata(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
meta := scraper.BookMeta{
|
||||
Slug: slug,
|
||||
Title: "Hybrid Store Test Novel",
|
||||
Author: "Test Author",
|
||||
Cover: "https://example.com/cover.jpg",
|
||||
Status: "Ongoing",
|
||||
Genres: []string{"Fantasy", "Action"},
|
||||
Summary: "A novel for integration testing.",
|
||||
TotalChapters: 99,
|
||||
SourceURL: fmt.Sprintf("https://example.com/book/%s", slug),
|
||||
Ranking: 5,
|
||||
}
|
||||
|
||||
t.Run("WriteMetadata", func(t *testing.T) {
|
||||
if err := hs.WriteMetadata(ctx, meta); err != nil {
|
||||
t.Fatalf("WriteMetadata: %v", err)
|
||||
}
|
||||
t.Logf("wrote metadata for slug=%q", slug)
|
||||
})
|
||||
|
||||
t.Run("ReadMetadata", func(t *testing.T) {
|
||||
got, found, err := hs.ReadMetadata(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMetadata: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("ReadMetadata: not found after WriteMetadata")
|
||||
}
|
||||
t.Logf("read: %+v", got)
|
||||
if got.Title != meta.Title {
|
||||
t.Errorf("Title = %q, want %q", got.Title, meta.Title)
|
||||
}
|
||||
if got.Author != meta.Author {
|
||||
t.Errorf("Author = %q, want %q", got.Author, meta.Author)
|
||||
}
|
||||
if got.TotalChapters != meta.TotalChapters {
|
||||
t.Errorf("TotalChapters = %d, want %d", got.TotalChapters, meta.TotalChapters)
|
||||
}
|
||||
if got.Ranking != meta.Ranking {
|
||||
t.Errorf("Ranking = %d, want %d", got.Ranking, meta.Ranking)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("MetadataMtime", func(t *testing.T) {
|
||||
mtime := hs.MetadataMtime(ctx, slug)
|
||||
if mtime == 0 {
|
||||
t.Error("MetadataMtime returned 0")
|
||||
}
|
||||
t.Logf("mtime: %d (%s)", mtime, time.Unix(mtime, 0))
|
||||
})
|
||||
|
||||
t.Run("ReadMetadata_NotFound", func(t *testing.T) {
|
||||
_, found, err := hs.ReadMetadata(ctx, "this-slug-does-not-exist-xyz")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMetadata (miss): %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Error("ReadMetadata returned found=true for a non-existent slug")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_WriteReadChapter exercises WriteChapter (MinIO blob + PocketBase
|
||||
// index), ReadChapter, CountChapters, and ListChapters.
|
||||
func TestHybridStore_WriteReadChapter(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
|
||||
// MinIO objects are not cleaned up — they use the test slug as prefix
|
||||
// and are effectively isolated.
|
||||
})
|
||||
|
||||
chapters := []scraper.Chapter{
|
||||
{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: The Beginning", Volume: 0},
|
||||
Text: "The first chapter text with enough content to be meaningful for a real novel chapter.",
|
||||
},
|
||||
{
|
||||
Ref: scraper.ChapterRef{Number: 2, Title: "Chapter 2: Rising Action", Volume: 0},
|
||||
Text: "The second chapter text continues the story from where the first left off.",
|
||||
},
|
||||
{
|
||||
Ref: scraper.ChapterRef{Number: 3, Title: "Chapter 3: Climax", Volume: 0},
|
||||
Text: "The third chapter text reaches the peak of tension and conflict.",
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("WriteChapter", func(t *testing.T) {
|
||||
for _, ch := range chapters {
|
||||
if err := hs.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter(%d): %v", ch.Ref.Number, err)
|
||||
}
|
||||
t.Logf("wrote chapter %d", ch.Ref.Number)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ChapterExists", func(t *testing.T) {
|
||||
for _, ch := range chapters {
|
||||
if !hs.ChapterExists(ctx, slug, ch.Ref) {
|
||||
t.Errorf("ChapterExists(chapter %d) = false after WriteChapter", ch.Ref.Number)
|
||||
}
|
||||
}
|
||||
missing := scraper.ChapterRef{Number: 999, Volume: 0}
|
||||
if hs.ChapterExists(ctx, slug, missing) {
|
||||
t.Error("ChapterExists(999) = true for a chapter that was never written")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ReadChapter", func(t *testing.T) {
|
||||
for _, ch := range chapters {
|
||||
got, err := hs.ReadChapter(ctx, slug, ch.Ref.Number)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadChapter(%d): %v", ch.Ref.Number, err)
|
||||
}
|
||||
// WriteChapter prepends "# <title>\n\n" and appends "\n".
|
||||
expectedPrefix := "# " + ch.Ref.Title
|
||||
if !strings.HasPrefix(got, expectedPrefix) {
|
||||
t.Errorf("chapter %d: content doesn't start with expected header\ngot: %q\nwant prefix: %q",
|
||||
ch.Ref.Number, got[:min(len(got), 80)], expectedPrefix)
|
||||
}
|
||||
if !strings.Contains(got, ch.Text) {
|
||||
t.Errorf("chapter %d: content doesn't contain original text", ch.Ref.Number)
|
||||
}
|
||||
t.Logf("chapter %d: %d bytes", ch.Ref.Number, len(got))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("CountChapters", func(t *testing.T) {
|
||||
count := hs.CountChapters(ctx, slug)
|
||||
if count != len(chapters) {
|
||||
t.Errorf("CountChapters = %d, want %d", count, len(chapters))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ListChapters", func(t *testing.T) {
|
||||
infos, err := hs.ListChapters(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapters: %v", err)
|
||||
}
|
||||
if len(infos) != len(chapters) {
|
||||
t.Errorf("ListChapters returned %d entries, want %d", len(infos), len(chapters))
|
||||
}
|
||||
for i, info := range infos {
|
||||
t.Logf("infos[%d]: number=%d title=%q date=%q", i, info.Number, info.Title, info.Date)
|
||||
}
|
||||
// Verify sorted order.
|
||||
for i := 1; i < len(infos); i++ {
|
||||
if infos[i].Number <= infos[i-1].Number {
|
||||
t.Errorf("ListChapters not sorted: infos[%d].Number=%d <= infos[%d].Number=%d",
|
||||
i, infos[i].Number, i-1, infos[i-1].Number)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_WriteReadRanking exercises WriteRankingItem → ReadRankingItems
|
||||
// round-trip and RankingFreshEnough.
|
||||
func TestHybridStore_WriteReadRanking(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug1 := "integ-rank-1-" + fmt.Sprintf("%d", time.Now().UnixMilli())
|
||||
slug2 := "integ-rank-2-" + fmt.Sprintf("%d", time.Now().UnixMilli())
|
||||
slug3 := "integ-rank-3-" + fmt.Sprintf("%d", time.Now().UnixMilli())
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
for _, sl := range []string{slug1, slug2, slug3} {
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "ranking", fmt.Sprintf(`slug="%s"`, sl))
|
||||
}
|
||||
})
|
||||
|
||||
items := []RankingItem{
|
||||
{Rank: 1, Slug: slug1, Title: "Top Novel", Author: "Author A", Status: "Ongoing", SourceURL: "https://example.com/book/top"},
|
||||
{Rank: 2, Slug: slug2, Title: "Second Novel", Author: "Author B", Genres: []string{"Action"}, Status: "Completed"},
|
||||
{Rank: 3, Slug: slug3, Title: "Third Novel"},
|
||||
}
|
||||
|
||||
t.Run("WriteRankingItem", func(t *testing.T) {
|
||||
for _, item := range items {
|
||||
if err := hs.WriteRankingItem(ctx, item); err != nil {
|
||||
t.Fatalf("WriteRankingItem(%s): %v", item.Slug, err)
|
||||
}
|
||||
}
|
||||
t.Logf("wrote %d ranking items", len(items))
|
||||
})
|
||||
|
||||
t.Run("ReadRankingItems", func(t *testing.T) {
|
||||
got, err := hs.ReadRankingItems(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadRankingItems: %v", err)
|
||||
}
|
||||
// Filter to just our test slugs (other tests may leave rows).
|
||||
var ours []RankingItem
|
||||
slugSet := map[string]bool{slug1: true, slug2: true, slug3: true}
|
||||
for _, g := range got {
|
||||
if slugSet[g.Slug] {
|
||||
ours = append(ours, g)
|
||||
}
|
||||
}
|
||||
if len(ours) != 3 {
|
||||
t.Fatalf("ReadRankingItems returned %d test items, want 3", len(ours))
|
||||
}
|
||||
// Verify order by rank.
|
||||
for i := 1; i < len(ours); i++ {
|
||||
if ours[i].Rank <= ours[i-1].Rank {
|
||||
t.Errorf("items not sorted by rank: ours[%d].Rank=%d, ours[%d].Rank=%d",
|
||||
i, ours[i].Rank, i-1, ours[i-1].Rank)
|
||||
}
|
||||
}
|
||||
// Verify fields.
|
||||
if ours[0].Title != "Top Novel" {
|
||||
t.Errorf("ours[0].Title = %q, want %q", ours[0].Title, "Top Novel")
|
||||
}
|
||||
if ours[0].Author != "Author A" {
|
||||
t.Errorf("ours[0].Author = %q, want %q", ours[0].Author, "Author A")
|
||||
}
|
||||
t.Logf("ranking items: %+v", ours)
|
||||
})
|
||||
|
||||
t.Run("RankingFreshEnough", func(t *testing.T) {
|
||||
fresh, err := hs.RankingFreshEnough(ctx, 24*time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("RankingFreshEnough: %v", err)
|
||||
}
|
||||
if !fresh {
|
||||
t.Error("RankingFreshEnough(24h) returned false immediately after writing items")
|
||||
}
|
||||
t.Logf("ranking fresh=true")
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_Progress exercises SetProgress → GetProgress → AllProgress →
|
||||
// DeleteProgress via the HybridStore.
|
||||
func TestHybridStore_Progress(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
const sessionID = "hybrid-test-session-abc"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"`, sessionID))
|
||||
})
|
||||
|
||||
p := ReadingProgress{Slug: slug, Chapter: 7, UpdatedAt: time.Now()}
|
||||
|
||||
t.Run("SetProgress", func(t *testing.T) {
|
||||
if err := hs.SetProgress(ctx, sessionID, p); err != nil {
|
||||
t.Fatalf("SetProgress: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("GetProgress", func(t *testing.T) {
|
||||
got, ok := hs.GetProgress(ctx, sessionID, slug)
|
||||
if !ok {
|
||||
t.Fatal("GetProgress: not found after SetProgress")
|
||||
}
|
||||
if got.Chapter != 7 {
|
||||
t.Errorf("Chapter = %d, want 7", got.Chapter)
|
||||
}
|
||||
if got.Slug != slug {
|
||||
t.Errorf("Slug = %q, want %q", got.Slug, slug)
|
||||
}
|
||||
t.Logf("progress: chapter=%d slug=%q updated=%s", got.Chapter, got.Slug, got.UpdatedAt)
|
||||
})
|
||||
|
||||
t.Run("AllProgress", func(t *testing.T) {
|
||||
all, err := hs.AllProgress(ctx, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("AllProgress: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, item := range all {
|
||||
if item.Slug == slug {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("AllProgress did not contain slug %q (total=%d)", slug, len(all))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("DeleteProgress", func(t *testing.T) {
|
||||
if err := hs.DeleteProgress(ctx, sessionID, slug); err != nil {
|
||||
t.Fatalf("DeleteProgress: %v", err)
|
||||
}
|
||||
_, ok := hs.GetProgress(ctx, sessionID, slug)
|
||||
if ok {
|
||||
t.Error("GetProgress returned ok=true after DeleteProgress")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_PresignChapter writes a chapter to MinIO via HybridStore,
|
||||
// then calls PresignChapter and verifies a non-empty URL is returned.
|
||||
func TestHybridStore_PresignChapter(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ch := scraper.Chapter{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Presign Test", Volume: 0},
|
||||
Text: "Text for the presign chapter test.",
|
||||
}
|
||||
|
||||
if err := hs.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter: %v", err)
|
||||
}
|
||||
|
||||
url, err := hs.PresignChapter(ctx, slug, 1, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignChapter: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignChapter returned empty URL")
|
||||
}
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("PresignChapter URL does not start with http: %q", url)
|
||||
}
|
||||
t.Logf("presigned chapter URL: %s", url)
|
||||
}
|
||||
|
||||
// TestHybridStore_PresignAudio puts a fake audio blob into MinIO via the
|
||||
// underlying MinioClient and verifies PresignAudio returns a valid URL.
|
||||
func TestHybridStore_PresignAudio(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
key := hs.AudioObjectKey(slug, 1, "af_bella")
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00hybrid-presign-audio-test")
|
||||
|
||||
if err := hs.minio.PutAudio(ctx, key, fakeAudio); err != nil {
|
||||
t.Fatalf("PutAudio: %v", err)
|
||||
}
|
||||
|
||||
url, err := hs.PresignAudio(ctx, key, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignAudio: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignAudio returned empty URL")
|
||||
}
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("PresignAudio URL does not start with http: %q", url)
|
||||
}
|
||||
t.Logf("presigned audio URL: %s", url)
|
||||
}
|
||||
|
||||
// TestHybridStore_AudioCache exercises SetAudioCache → GetAudioCache via HybridStore.
|
||||
func TestHybridStore_AudioCache(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cacheKey := fmt.Sprintf("hybrid-audio-test-%d", time.Now().UnixMilli())
|
||||
const filename = "speech_hybrid123.mp3"
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, cacheKey))
|
||||
})
|
||||
|
||||
if err := hs.SetAudioCache(ctx, cacheKey, filename); err != nil {
|
||||
t.Fatalf("SetAudioCache: %v", err)
|
||||
}
|
||||
|
||||
got, ok := hs.GetAudioCache(ctx, cacheKey)
|
||||
if !ok {
|
||||
t.Fatal("GetAudioCache returned ok=false after SetAudioCache")
|
||||
}
|
||||
if got != filename {
|
||||
t.Errorf("filename = %q, want %q", got, filename)
|
||||
}
|
||||
t.Logf("audio cache: cacheKey=%q filename=%q", cacheKey, got)
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
77
scraper/internal/storage/hybrid_unit_test.go
Normal file
77
scraper/internal/storage/hybrid_unit_test.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ── chapterNumberFromKey ──────────────────────────────────────────────────────
|
||||
|
||||
func TestChapterNumberFromKey(t *testing.T) {
|
||||
cases := []struct {
|
||||
key string
|
||||
want int
|
||||
}{
|
||||
// Standard four-segment key.
|
||||
{"my-novel/vol-0/1-50/chapter-1.md", 1},
|
||||
{"my-novel/vol-0/1-50/chapter-42.md", 42},
|
||||
{"my-novel/vol-0/51-100/chapter-99.md", 99},
|
||||
// Large chapter numbers.
|
||||
{"some-novel/vol-1/1001-1050/chapter-1024.md", 1024},
|
||||
// Nested deeper paths should still work (last segment used).
|
||||
{"a/b/c/d/chapter-7.md", 7},
|
||||
// Malformed / unexpected inputs — should return 0 without panicking.
|
||||
{"chapter-notanumber.md", 0},
|
||||
{"", 0},
|
||||
// No .md extension — TrimSuffix is a no-op; TrimPrefix still strips
|
||||
// "chapter-", so the number is parsed successfully.
|
||||
{"no-md-extension/chapter-5", 5},
|
||||
{"my-novel/vol-0/1-50/chapter-0.md", 0}, // 0 is invalid (chapters are 1-based)
|
||||
{"my-novel/vol-0/1-50/chapter--1.md", 0},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
got := chapterNumberFromKey(tc.key)
|
||||
if got != tc.want {
|
||||
t.Errorf("chapterNumberFromKey(%q) = %d, want %d", tc.key, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── splitChapterTitle ─────────────────────────────────────────────────────────
|
||||
|
||||
func TestSplitChapterTitle(t *testing.T) {
|
||||
cases := []struct {
|
||||
raw string
|
||||
wantTitle string
|
||||
wantDate string
|
||||
}{
|
||||
// No date — title is returned as-is.
|
||||
{"The Great Battle", "The Great Battle", ""},
|
||||
// Leading numeric index is stripped.
|
||||
{"42 The Great Battle", "The Great Battle", ""},
|
||||
// Relative date with plural unit.
|
||||
{"The Storm Arrives 3 days ago", "The Storm Arrives", "3 days ago"},
|
||||
// Singular unit.
|
||||
{"A New Hope 1 week ago", "A New Hope", "1 week ago"},
|
||||
// Minutes and seconds.
|
||||
{"Flash Fight 5 minutes ago", "Flash Fight", "5 minutes ago"},
|
||||
{"Quick Strike 30 seconds ago", "Quick Strike", "30 seconds ago"},
|
||||
// Months and years.
|
||||
{"Old Chapter 2 months ago", "Old Chapter", "2 months ago"},
|
||||
{"Ancient Story 1 year ago", "Ancient Story", "1 year ago"},
|
||||
// Leading index AND trailing date.
|
||||
{"5 The Final Chapter 2 hours ago", "The Final Chapter", "2 hours ago"},
|
||||
// Extra whitespace.
|
||||
{" The Calm ", "The Calm", ""},
|
||||
// Empty string.
|
||||
{"", "", ""},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
title, date := splitChapterTitle(tc.raw)
|
||||
if title != tc.wantTitle || date != tc.wantDate {
|
||||
t.Errorf("splitChapterTitle(%q) = (%q, %q), want (%q, %q)",
|
||||
tc.raw, title, date, tc.wantTitle, tc.wantDate)
|
||||
}
|
||||
}
|
||||
}
|
||||
655
scraper/internal/storage/integration_test.go
Normal file
655
scraper/internal/storage/integration_test.go
Normal file
@@ -0,0 +1,655 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests for MinioClient and PocketBaseStore against live instances.
|
||||
//
|
||||
// These tests require running MinIO and PocketBase services. They are gated
|
||||
// behind the "integration" build tag and are never run in a normal `go test ./...`.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 120s \
|
||||
// github.com/libnovel/scraper/internal/storage
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func newTestMinioClient(t *testing.T) *MinioClient {
|
||||
t.Helper()
|
||||
endpoint := os.Getenv("MINIO_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping MinIO integration test")
|
||||
}
|
||||
useSSL := os.Getenv("MINIO_USE_SSL") == "true"
|
||||
cfg := MinioConfig{
|
||||
Endpoint: endpoint,
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: useSSL,
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
mc, err := NewMinioClient(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMinioClient: %v", err)
|
||||
}
|
||||
return mc
|
||||
}
|
||||
|
||||
func newTestPocketBaseStore(t *testing.T) *PocketBaseStore {
|
||||
t.Helper()
|
||||
pbURL := os.Getenv("POCKETBASE_URL")
|
||||
if pbURL == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping PocketBase integration test")
|
||||
}
|
||||
cfg := PocketBaseConfig{
|
||||
BaseURL: pbURL,
|
||||
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||
}
|
||||
store := NewPocketBaseStore(cfg, slog.Default())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
if err := store.EnsureCollections(ctx); err != nil {
|
||||
t.Logf("EnsureCollections (may be harmless): %v", err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
// testSlug generates a unique test slug to avoid collisions between parallel runs.
|
||||
func testSlug(t *testing.T) string {
|
||||
t.Helper()
|
||||
safe := strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '-'
|
||||
}, strings.ToLower(t.Name()))
|
||||
// Truncate and append a timestamp to keep it unique.
|
||||
if len(safe) > 30 {
|
||||
safe = safe[:30]
|
||||
}
|
||||
return fmt.Sprintf("test-%s-%d", safe, time.Now().UnixMilli()%100000)
|
||||
}
|
||||
|
||||
// ─── MinioClient tests ────────────────────────────────────────────────────────
|
||||
|
||||
// TestMinioClient_ChapterRoundTrip verifies PutChapter → GetChapter →
|
||||
// ChapterExists → ListChapterKeys for a single chapter.
|
||||
func TestMinioClient_ChapterRoundTrip(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
const vol = 0
|
||||
const n = 1
|
||||
content := "# Chapter 1\n\nHello integration world.\n"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Run("PutChapter", func(t *testing.T) {
|
||||
if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
|
||||
t.Fatalf("PutChapter: %v", err)
|
||||
}
|
||||
t.Logf("stored chapter at key: %s", chapterKey(slug, vol, n))
|
||||
})
|
||||
|
||||
t.Run("GetChapter", func(t *testing.T) {
|
||||
got, err := mc.GetChapter(ctx, slug, vol, n)
|
||||
if err != nil {
|
||||
t.Fatalf("GetChapter: %v", err)
|
||||
}
|
||||
if got != content {
|
||||
t.Errorf("GetChapter round-trip mismatch:\ngot: %q\nwant: %q", got, content)
|
||||
}
|
||||
t.Logf("retrieved %d bytes", len(got))
|
||||
})
|
||||
|
||||
t.Run("ChapterExists", func(t *testing.T) {
|
||||
if !mc.ChapterExists(ctx, slug, vol, n) {
|
||||
t.Error("ChapterExists returned false for a just-stored chapter")
|
||||
}
|
||||
if mc.ChapterExists(ctx, slug, vol, 999) {
|
||||
t.Error("ChapterExists returned true for a chapter that was never stored")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ListChapterKeys", func(t *testing.T) {
|
||||
keys, err := mc.ListChapterKeys(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterKeys: %v", err)
|
||||
}
|
||||
if len(keys) != 1 {
|
||||
t.Fatalf("ListChapterKeys returned %d keys, want 1: %v", len(keys), keys)
|
||||
}
|
||||
expectedKey := chapterKey(slug, vol, n)
|
||||
if keys[0] != expectedKey {
|
||||
t.Errorf("key = %q, want %q", keys[0], expectedKey)
|
||||
}
|
||||
t.Logf("keys: %v", keys)
|
||||
})
|
||||
}
|
||||
|
||||
// TestMinioClient_MultiChapterList stores several chapters and verifies
|
||||
// ListChapterKeys returns them all.
|
||||
func TestMinioClient_MultiChapterList(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
const vol = 0
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Store chapters 1, 2, 51 (crosses the 1-50 folder boundary).
|
||||
chapters := []int{1, 2, 51}
|
||||
for _, n := range chapters {
|
||||
content := fmt.Sprintf("# Chapter %d\n\nContent for chapter %d.\n", n, n)
|
||||
if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
|
||||
t.Fatalf("PutChapter(%d): %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
keys, err := mc.ListChapterKeys(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterKeys: %v", err)
|
||||
}
|
||||
t.Logf("keys: %v", keys)
|
||||
if len(keys) != len(chapters) {
|
||||
t.Errorf("ListChapterKeys returned %d keys, want %d", len(keys), len(chapters))
|
||||
}
|
||||
|
||||
count := mc.CountChapters(ctx, slug)
|
||||
if count != len(chapters) {
|
||||
t.Errorf("CountChapters = %d, want %d", count, len(chapters))
|
||||
}
|
||||
}
|
||||
|
||||
// TestMinioClient_PresignChapter verifies PresignChapter returns a non-empty URL.
|
||||
func TestMinioClient_PresignChapter(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
const vol = 0
|
||||
const n = 1
|
||||
content := "# Presign test\n\nSome content.\n"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
|
||||
t.Fatalf("PutChapter: %v", err)
|
||||
}
|
||||
|
||||
url, err := mc.PresignChapter(ctx, slug, vol, n, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignChapter: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignChapter returned empty URL")
|
||||
}
|
||||
t.Logf("presigned URL: %s", url)
|
||||
|
||||
// URL must be an http(s) URL and contain the slug somewhere.
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("URL does not start with http: %q", url)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMinioClient_AudioRoundTrip verifies PutAudio → GetAudio → AudioExists.
|
||||
func TestMinioClient_AudioRoundTrip(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
key := AudioObjectKey(slug, 1, "af_bella")
|
||||
|
||||
// Use minimal fake MP3 bytes (just a recognisable prefix).
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00integration-test-audio")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Run("PutAudio", func(t *testing.T) {
|
||||
if err := mc.PutAudio(ctx, key, fakeAudio); err != nil {
|
||||
t.Fatalf("PutAudio: %v", err)
|
||||
}
|
||||
t.Logf("stored audio at key: %s", key)
|
||||
})
|
||||
|
||||
t.Run("GetAudio", func(t *testing.T) {
|
||||
got, err := mc.GetAudio(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudio: %v", err)
|
||||
}
|
||||
if string(got) != string(fakeAudio) {
|
||||
t.Errorf("GetAudio round-trip mismatch: got %d bytes, want %d", len(got), len(fakeAudio))
|
||||
}
|
||||
t.Logf("retrieved %d bytes", len(got))
|
||||
})
|
||||
|
||||
t.Run("AudioExists", func(t *testing.T) {
|
||||
if !mc.AudioExists(ctx, key) {
|
||||
t.Error("AudioExists returned false for a just-stored audio object")
|
||||
}
|
||||
if mc.AudioExists(ctx, "nonexistent/key.mp3") {
|
||||
t.Error("AudioExists returned true for a key that was never stored")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestMinioClient_PresignAudio verifies PresignAudio returns a non-empty URL.
|
||||
func TestMinioClient_PresignAudio(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
key := AudioObjectKey(slug, 1, "af_bella")
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00presign-audio-test")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := mc.PutAudio(ctx, key, fakeAudio); err != nil {
|
||||
t.Fatalf("PutAudio: %v", err)
|
||||
}
|
||||
|
||||
url, err := mc.PresignAudio(ctx, key, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignAudio: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignAudio returned empty URL")
|
||||
}
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("URL does not start with http: %q", url)
|
||||
}
|
||||
t.Logf("presigned audio URL: %s", url)
|
||||
}
|
||||
|
||||
// ─── PocketBaseStore tests ────────────────────────────────────────────────────
|
||||
|
||||
// TestPocketBaseStore_Ping verifies that admin auth works.
|
||||
func TestPocketBaseStore_Ping(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := store.Ping(ctx); err != nil {
|
||||
t.Fatalf("Ping: %v", err)
|
||||
}
|
||||
t.Log("Ping succeeded")
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_BookRoundTrip tests UpsertBook → GetBook → ListBooks →
|
||||
// BookMetaUpdated.
|
||||
func TestPocketBaseStore_BookRoundTrip(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Clean up after test.
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
t.Run("UpsertBook_Create", func(t *testing.T) {
|
||||
err := store.UpsertBook(ctx, slug,
|
||||
"Integration Test Novel", "Test Author",
|
||||
"https://example.com/cover.jpg", "Ongoing",
|
||||
"A test summary.", "https://example.com/book/test",
|
||||
[]string{"Action", "Fantasy"}, 42, 7,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertBook (create): %v", err)
|
||||
}
|
||||
t.Logf("created book %q", slug)
|
||||
})
|
||||
|
||||
t.Run("GetBook", func(t *testing.T) {
|
||||
rec, found, err := store.GetBook(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("GetBook: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("GetBook: book not found after UpsertBook")
|
||||
}
|
||||
t.Logf("GetBook record: %v", rec)
|
||||
if rec["title"] != "Integration Test Novel" {
|
||||
t.Errorf("title = %v, want %q", rec["title"], "Integration Test Novel")
|
||||
}
|
||||
if rec["author"] != "Test Author" {
|
||||
t.Errorf("author = %v, want %q", rec["author"], "Test Author")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ListBooks", func(t *testing.T) {
|
||||
books, err := store.ListBooks(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListBooks: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, b := range books {
|
||||
if s, _ := b["slug"].(string); s == slug {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("ListBooks did not return book with slug %q (total=%d)", slug, len(books))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("UpsertBook_Update", func(t *testing.T) {
|
||||
err := store.UpsertBook(ctx, slug,
|
||||
"Integration Test Novel", "Test Author Updated",
|
||||
"", "Completed", "", "https://example.com/book/test",
|
||||
nil, 100, 3,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertBook (update): %v", err)
|
||||
}
|
||||
rec, found, err := store.GetBook(ctx, slug)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("GetBook after update: found=%v err=%v", found, err)
|
||||
}
|
||||
if rec["author"] != "Test Author Updated" {
|
||||
t.Errorf("author after update = %v, want %q", rec["author"], "Test Author Updated")
|
||||
}
|
||||
if rec["status"] != "Completed" {
|
||||
t.Errorf("status after update = %v, want %q", rec["status"], "Completed")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("BookMetaUpdated", func(t *testing.T) {
|
||||
ts, err := store.BookMetaUpdated(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("BookMetaUpdated: %v", err)
|
||||
}
|
||||
if ts.IsZero() {
|
||||
t.Error("BookMetaUpdated returned zero time")
|
||||
}
|
||||
t.Logf("meta_updated: %s", ts)
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_ChapterIdx tests UpsertChapterIdx → ListChapterIdx →
|
||||
// CountChapterIdx.
|
||||
func TestPocketBaseStore_ChapterIdx(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
chapters := []struct {
|
||||
n int
|
||||
title string
|
||||
date string
|
||||
}{
|
||||
{1, "Chapter 1: The Beginning", "2 days ago"},
|
||||
{2, "Chapter 2: Rising Action", "1 day ago"},
|
||||
{3, "Chapter 3: Climax", "3 hours ago"},
|
||||
}
|
||||
|
||||
for _, ch := range chapters {
|
||||
if err := store.UpsertChapterIdx(ctx, slug, ch.n, ch.title, ch.date); err != nil {
|
||||
t.Fatalf("UpsertChapterIdx(%d): %v", ch.n, err)
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("ListChapterIdx", func(t *testing.T) {
|
||||
rows, err := store.ListChapterIdx(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterIdx: %v", err)
|
||||
}
|
||||
if len(rows) != len(chapters) {
|
||||
t.Errorf("ListChapterIdx returned %d rows, want %d", len(rows), len(chapters))
|
||||
}
|
||||
for i, row := range rows {
|
||||
t.Logf("row[%d]: number=%v title=%v date_label=%v", i, row["number"], row["title"], row["date_label"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("CountChapterIdx", func(t *testing.T) {
|
||||
count := store.CountChapterIdx(ctx, slug)
|
||||
if count != len(chapters) {
|
||||
t.Errorf("CountChapterIdx = %d, want %d", count, len(chapters))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("UpsertChapterIdx_Update", func(t *testing.T) {
|
||||
// Re-upsert chapter 2 with an updated title.
|
||||
if err := store.UpsertChapterIdx(ctx, slug, 2, "Chapter 2: Revised Title", "1 day ago"); err != nil {
|
||||
t.Fatalf("UpsertChapterIdx (update): %v", err)
|
||||
}
|
||||
rows, err := store.ListChapterIdx(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterIdx after update: %v", err)
|
||||
}
|
||||
if store.CountChapterIdx(ctx, slug) != len(chapters) {
|
||||
t.Errorf("count changed after update: got %d, want %d", len(rows), len(chapters))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_Ranking tests SetRanking → GetRanking → RankingModTime.
|
||||
func TestPocketBaseStore_Ranking(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
slug1 := testSlug(t) + "-rank1"
|
||||
slug2 := testSlug(t) + "-rank2"
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
for _, sl := range []string{slug1, slug2} {
|
||||
_ = store.pb.deleteWhere(cleanCtx, "ranking", fmt.Sprintf(`slug="%s"`, sl))
|
||||
}
|
||||
})
|
||||
|
||||
items := []RankingItem{
|
||||
{Rank: 1, Slug: slug1, Title: "Test Book One", SourceURL: "https://example.com/1"},
|
||||
{Rank: 2, Slug: slug2, Title: "Test Book Two", SourceURL: "https://example.com/2"},
|
||||
}
|
||||
|
||||
t.Run("WriteRankingItem", func(t *testing.T) {
|
||||
for _, item := range items {
|
||||
if err := store.UpsertRankingItem(ctx, item); err != nil {
|
||||
t.Fatalf("UpsertRankingItem(%q): %v", item.Slug, err)
|
||||
}
|
||||
}
|
||||
t.Log("UpsertRankingItem succeeded")
|
||||
})
|
||||
|
||||
t.Run("ReadRankingItems", func(t *testing.T) {
|
||||
got, err := store.ListRankingItems(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListRankingItems: %v", err)
|
||||
}
|
||||
found := 0
|
||||
for _, g := range got {
|
||||
if g.Slug == slug1 || g.Slug == slug2 {
|
||||
found++
|
||||
}
|
||||
}
|
||||
if found != 2 {
|
||||
t.Errorf("ListRankingItems: found %d of 2 test items in %d total", found, len(got))
|
||||
}
|
||||
t.Logf("ListRankingItems returned %d total items, %d test items", len(got), found)
|
||||
})
|
||||
|
||||
t.Run("RankingFreshEnough", func(t *testing.T) {
|
||||
updated, err := store.RankingLastUpdated(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("RankingLastUpdated: %v", err)
|
||||
}
|
||||
if updated.IsZero() {
|
||||
t.Error("RankingLastUpdated returned zero time immediately after write")
|
||||
}
|
||||
fresh := time.Since(updated) < 24*time.Hour
|
||||
if !fresh {
|
||||
t.Errorf("RankingLastUpdated = %s; want within 24h", updated)
|
||||
}
|
||||
t.Logf("RankingLastUpdated = %s (fresh=%v)", updated, fresh)
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_Progress tests SetProgress → GetProgress → AllProgress →
|
||||
// DeleteProgress.
|
||||
func TestPocketBaseStore_Progress(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
slug := testSlug(t)
|
||||
const sessionID = "integration-test-session-xyz"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"`, sessionID))
|
||||
})
|
||||
|
||||
t.Run("SetProgress", func(t *testing.T) {
|
||||
if err := store.SetProgress(ctx, sessionID, slug, 5); err != nil {
|
||||
t.Fatalf("SetProgress: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("GetProgress", func(t *testing.T) {
|
||||
ch, updated, found, err := store.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("GetProgress: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("GetProgress: not found after SetProgress")
|
||||
}
|
||||
if ch != 5 {
|
||||
t.Errorf("chapter = %d, want 5", ch)
|
||||
}
|
||||
if updated.IsZero() {
|
||||
t.Error("updated time is zero")
|
||||
}
|
||||
t.Logf("chapter=%d updated=%s", ch, updated)
|
||||
})
|
||||
|
||||
t.Run("AllProgress", func(t *testing.T) {
|
||||
rows, err := store.AllProgress(ctx, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("AllProgress: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range rows {
|
||||
if s, _ := r["slug"].(string); s == slug {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("AllProgress did not include slug %q (total=%d)", slug, len(rows))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("SetProgress_Update", func(t *testing.T) {
|
||||
if err := store.SetProgress(ctx, sessionID, slug, 12); err != nil {
|
||||
t.Fatalf("SetProgress (update): %v", err)
|
||||
}
|
||||
ch, _, found, err := store.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("GetProgress after update: found=%v err=%v", found, err)
|
||||
}
|
||||
if ch != 12 {
|
||||
t.Errorf("chapter after update = %d, want 12", ch)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("DeleteProgress", func(t *testing.T) {
|
||||
if err := store.DeleteProgress(ctx, sessionID, slug); err != nil {
|
||||
t.Fatalf("DeleteProgress: %v", err)
|
||||
}
|
||||
_, _, found, err := store.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("GetProgress after delete: %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Error("GetProgress returned found=true after DeleteProgress")
|
||||
}
|
||||
t.Log("DeleteProgress confirmed")
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_AudioCache tests SetAudioCache → GetAudioCache.
|
||||
func TestPocketBaseStore_AudioCache(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cacheKey := fmt.Sprintf("integration-audio-cache-test-%d", time.Now().UnixMilli())
|
||||
const filename = "speech_abc123.mp3"
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, cacheKey))
|
||||
})
|
||||
|
||||
t.Run("SetAudioCache", func(t *testing.T) {
|
||||
if err := store.SetAudioCache(ctx, cacheKey, filename); err != nil {
|
||||
t.Fatalf("SetAudioCache: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("GetAudioCache", func(t *testing.T) {
|
||||
got, found, err := store.GetAudioCache(ctx, cacheKey)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudioCache: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("GetAudioCache: not found after SetAudioCache")
|
||||
}
|
||||
if got != filename {
|
||||
t.Errorf("filename = %q, want %q", got, filename)
|
||||
}
|
||||
t.Logf("filename: %s", got)
|
||||
})
|
||||
|
||||
t.Run("GetAudioCache_Miss", func(t *testing.T) {
|
||||
got, found, err := store.GetAudioCache(ctx, "does-not-exist-ever")
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudioCache (miss): %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Errorf("GetAudioCache returned found=true for missing key, filename=%q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
413
scraper/internal/storage/minio.go
Normal file
413
scraper/internal/storage/minio.go
Normal file
@@ -0,0 +1,413 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// MinioConfig holds connection parameters for MinIO.
|
||||
type MinioConfig struct {
|
||||
Endpoint string // e.g. "minio:9000" — internal address used for all operations
|
||||
PublicEndpoint string // e.g. "minio.kalekber.cc" — used to sign presigned URLs so browsers can reach them; leave empty to use Endpoint
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
UseSSL bool
|
||||
PublicUseSSL bool // TLS for the public endpoint (usually true in prod)
|
||||
BucketChapters string // e.g. "libnovel-chapters"
|
||||
BucketAudio string // e.g. "libnovel-audio"
|
||||
BucketBrowse string // e.g. "libnovel-browse"
|
||||
BucketAvatars string // e.g. "libnovel-avatars"
|
||||
}
|
||||
|
||||
// MinioClient wraps a minio.Client and exposes object operations for
|
||||
// chapters and audio files.
|
||||
type MinioClient struct {
|
||||
c *minio.Client // internal client — used for all read/write operations
|
||||
pub *minio.Client // public client — used only for generating presigned URLs
|
||||
cfg MinioConfig
|
||||
}
|
||||
|
||||
// NewMinioClient creates a connected MinIO client and ensures the required
|
||||
// buckets exist.
|
||||
func NewMinioClient(ctx context.Context, cfg MinioConfig) (*MinioClient, error) {
|
||||
c, err := minio.New(cfg.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio: new client: %w", err)
|
||||
}
|
||||
|
||||
// Public client: signs presigned URLs with the public hostname so browsers
|
||||
// can fetch them directly. Falls back to the internal client if no public
|
||||
// endpoint is configured.
|
||||
pub := c
|
||||
if cfg.PublicEndpoint != "" && cfg.PublicEndpoint != cfg.Endpoint {
|
||||
pub, err = minio.New(cfg.PublicEndpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: cfg.PublicUseSSL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio: new public client: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
mc := &MinioClient{c: c, pub: pub, cfg: cfg}
|
||||
for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio, cfg.BucketBrowse, cfg.BucketAvatars} {
|
||||
if bucket == "" {
|
||||
continue
|
||||
}
|
||||
if err := mc.ensureBucket(ctx, bucket); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return mc, nil
|
||||
}
|
||||
|
||||
// ensureBucket creates a bucket if it does not exist.
|
||||
func (m *MinioClient) ensureBucket(ctx context.Context, bucket string) error {
|
||||
exists, err := m.c.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: bucket exists %q: %w", bucket, err)
|
||||
}
|
||||
if !exists {
|
||||
if err := m.c.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
return fmt.Errorf("minio: make bucket %q: %w", bucket, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── Chapter objects ──────────────────────────────────────────────────────────
|
||||
|
||||
// chapterKey returns the MinIO object key for a chapter.
|
||||
// Layout: {slug}/vol-{vol}/{lo}-{hi}/chapter-{n}.md
|
||||
func chapterKey(slug string, vol, n int) string {
|
||||
const chaptersPerFolder = 50
|
||||
lo := ((n-1)/chaptersPerFolder)*chaptersPerFolder + 1
|
||||
hi := lo + chaptersPerFolder - 1
|
||||
return fmt.Sprintf("%s/vol-%d/%d-%d/chapter-%d.md", slug, vol, lo, hi, n)
|
||||
}
|
||||
|
||||
// PutChapter stores chapter markdown in MinIO.
|
||||
func (m *MinioClient) PutChapter(ctx context.Context, slug string, vol, n int, content string) error {
|
||||
key := chapterKey(slug, vol, n)
|
||||
data := []byte(content)
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketChapters, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: "text/markdown; charset=utf-8"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put chapter %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetChapter retrieves chapter markdown from MinIO.
|
||||
func (m *MinioClient) GetChapter(ctx context.Context, slug string, vol, n int) (string, error) {
|
||||
key := chapterKey(slug, vol, n)
|
||||
obj, err := m.c.GetObject(ctx, m.cfg.BucketChapters, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio: get chapter %s: %w", key, err)
|
||||
}
|
||||
defer obj.Close()
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio: read chapter %s: %w", key, err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
// ChapterExists returns true if the object for this chapter is present.
|
||||
func (m *MinioClient) ChapterExists(ctx context.Context, slug string, vol, n int) bool {
|
||||
key := chapterKey(slug, vol, n)
|
||||
_, err := m.c.StatObject(ctx, m.cfg.BucketChapters, key, minio.StatObjectOptions{})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// ListChapterKeys returns all object keys under slug/ in the chapters bucket,
|
||||
// sorted lexicographically (MinIO returns them in order).
|
||||
func (m *MinioClient) ListChapterKeys(ctx context.Context, slug string) ([]string, error) {
|
||||
prefix := slug + "/"
|
||||
var keys []string
|
||||
for obj := range m.c.ListObjects(ctx, m.cfg.BucketChapters,
|
||||
minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
|
||||
if obj.Err != nil {
|
||||
return nil, fmt.Errorf("minio: list chapters %s: %w", slug, obj.Err)
|
||||
}
|
||||
keys = append(keys, obj.Key)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// CountChapters returns the number of chapter objects for a slug.
|
||||
func (m *MinioClient) CountChapters(ctx context.Context, slug string) int {
|
||||
keys, _ := m.ListChapterKeys(ctx, slug)
|
||||
return len(keys)
|
||||
}
|
||||
|
||||
// ─── Audio objects ────────────────────────────────────────────────────────────
|
||||
|
||||
// AudioObjectKey returns the MinIO key for a cached audio file.
|
||||
// Key: {slug}/ch{n}-{voice}.mp3
|
||||
func AudioObjectKey(slug string, n int, voice string) string {
|
||||
safe := sanitiseVoice(voice)
|
||||
return fmt.Sprintf("%s/ch%d-%s.mp3", slug, n, safe)
|
||||
}
|
||||
|
||||
// PutAudio stores an audio file in the audio bucket.
|
||||
func (m *MinioClient) PutAudio(ctx context.Context, key string, data []byte) error {
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketAudio, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: "audio/mpeg"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put audio %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAudio retrieves audio bytes from the audio bucket.
|
||||
func (m *MinioClient) GetAudio(ctx context.Context, key string) ([]byte, error) {
|
||||
obj, err := m.c.GetObject(ctx, m.cfg.BucketAudio, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio: get audio %s: %w", key, err)
|
||||
}
|
||||
defer obj.Close()
|
||||
return io.ReadAll(obj)
|
||||
}
|
||||
|
||||
// AudioExists returns true if the audio object is present in the bucket.
|
||||
func (m *MinioClient) AudioExists(ctx context.Context, key string) bool {
|
||||
_, err := m.c.StatObject(ctx, m.cfg.BucketAudio, key, minio.StatObjectOptions{})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// ─── Presigned URLs ───────────────────────────────────────────────────────────
|
||||
|
||||
// PresignChapter returns a presigned GET URL for a chapter object signed with
|
||||
// the internal endpoint — intended for server-side fetches only.
|
||||
func (m *MinioClient) PresignChapter(ctx context.Context, slug string, vol, n int, expires time.Duration) (string, error) {
|
||||
key := chapterKey(slug, vol, n)
|
||||
u, err := m.c.PresignedGetObject(ctx, m.cfg.BucketChapters, key, expires, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio: presign chapter %s: %w", key, err)
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
// PresignAudio returns a presigned GET URL for an audio object signed with
|
||||
// the public endpoint so the browser can fetch it directly.
|
||||
func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
|
||||
u, err := m.pub.PresignedGetObject(ctx, m.cfg.BucketAudio, key, expires, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio: presign audio %s: %w", key, err)
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
// ─── Browse page snapshots ────────────────────────────────────────────────────
|
||||
//
|
||||
// New bucket layout (libnovel-browse):
|
||||
//
|
||||
// {domain}/html/page-{n}.html — SingleFile HTML snapshot
|
||||
// {domain}/assets/book-covers/{slug}.jpg — downloaded cover image
|
||||
//
|
||||
// The domain segment is derived from the source URL hostname
|
||||
// (e.g. "novelfire.net"). This makes the bucket self-describing and
|
||||
// extensible to multiple sources.
|
||||
|
||||
// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
|
||||
// Layout: {domain}/html/page-{n}.html
|
||||
// This uses the default (popular/all/all) filter combination.
|
||||
func BrowseHTMLKey(domain string, page int) string {
|
||||
return fmt.Sprintf("%s/html/page-%d.html", domain, page)
|
||||
}
|
||||
|
||||
// BrowseFilteredHTMLKey returns the MinIO object key for a browse page snapshot
|
||||
// that includes filter parameters (sort, genre, status) in the key so that
|
||||
// different filter combinations are cached independently.
|
||||
// Layout: {domain}/html/{sort}-{genre}-{status}/page-{n}.html
|
||||
// Falls back to BrowseHTMLKey when all filters are at their default values
|
||||
// (sort=popular, genre=all, status=all) for cache compatibility.
|
||||
func BrowseFilteredHTMLKey(domain string, page int, sort, genre, status string) string {
|
||||
if (sort == "" || sort == "popular") && (genre == "" || genre == "all") && (status == "" || status == "all") {
|
||||
return BrowseHTMLKey(domain, page)
|
||||
}
|
||||
if sort == "" {
|
||||
sort = "popular"
|
||||
}
|
||||
if genre == "" {
|
||||
genre = "all"
|
||||
}
|
||||
if status == "" {
|
||||
status = "all"
|
||||
}
|
||||
return fmt.Sprintf("%s/html/%s-%s-%s/page-%d.html", domain, sort, genre, status, page)
|
||||
}
|
||||
|
||||
// BrowseCoverKey returns the MinIO object key for a cached book cover image.
|
||||
// Layout: {domain}/assets/book-covers/{slug}.jpg
|
||||
func BrowseCoverKey(domain, slug string) string {
|
||||
return fmt.Sprintf("%s/assets/book-covers/%s.jpg", domain, slug)
|
||||
}
|
||||
|
||||
// PutBrowsePage stores a SingleFile HTML snapshot in the browse bucket.
|
||||
func (m *MinioClient) PutBrowsePage(ctx context.Context, key, html string) error {
|
||||
data := []byte(html)
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: "text/html; charset=utf-8"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put browse page %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBrowsePage retrieves a SingleFile HTML snapshot from the browse bucket.
|
||||
// Returns ("", false, nil) when the object does not exist.
|
||||
func (m *MinioClient) GetBrowsePage(ctx context.Context, key string) (string, bool, error) {
|
||||
obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("minio: get browse page %s: %w", key, err)
|
||||
}
|
||||
defer obj.Close()
|
||||
// Check whether the object actually exists by inspecting the Stat.
|
||||
if _, statErr := obj.Stat(); statErr != nil {
|
||||
return "", false, nil // not found
|
||||
}
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("minio: read browse page %s: %w", key, err)
|
||||
}
|
||||
return string(data), true, nil
|
||||
}
|
||||
|
||||
// BrowsePageExists returns true if a snapshot object is present in the browse bucket.
|
||||
func (m *MinioClient) BrowsePageExists(ctx context.Context, key string) bool {
|
||||
_, err := m.c.StatObject(ctx, m.cfg.BucketBrowse, key, minio.StatObjectOptions{})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// PutBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
|
||||
// contentType should be the MIME type, e.g. "image/jpeg".
|
||||
func (m *MinioClient) PutBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: contentType})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put browse asset %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBrowseAsset retrieves a binary asset from the browse bucket.
|
||||
// Returns (nil, false, nil) when the object does not exist.
|
||||
func (m *MinioClient) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
|
||||
obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, "", false, fmt.Errorf("minio: get browse asset %s: %w", key, err)
|
||||
}
|
||||
defer obj.Close()
|
||||
info, statErr := obj.Stat()
|
||||
if statErr != nil {
|
||||
return nil, "", false, nil // not found
|
||||
}
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return nil, "", false, fmt.Errorf("minio: read browse asset %s: %w", key, err)
|
||||
}
|
||||
return data, info.ContentType, true, nil
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// sanitiseVoice converts a voice name to a filename-safe string.
|
||||
func sanitiseVoice(voice string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
|
||||
(r >= '0' && r <= '9') || r == '_' || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '_'
|
||||
}, voice)
|
||||
}
|
||||
|
||||
// ─── Avatar objects ───────────────────────────────────────────────────────────
|
||||
|
||||
// avatarKey returns the MinIO object key for a user avatar.
|
||||
// Layout: avatars/{userId}.{ext}
|
||||
func avatarKey(userID, ext string) string {
|
||||
return fmt.Sprintf("avatars/%s.%s", userID, ext)
|
||||
}
|
||||
|
||||
// PutAvatar stores an avatar image in the avatars bucket.
|
||||
// ext should be "jpg", "png", or "webp".
|
||||
func (m *MinioClient) PutAvatar(ctx context.Context, userID, ext string, data []byte, contentType string) error {
|
||||
if m.cfg.BucketAvatars == "" {
|
||||
return fmt.Errorf("minio: avatars bucket not configured")
|
||||
}
|
||||
key := avatarKey(userID, ext)
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketAvatars, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: contentType})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put avatar %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PresignAvatarUploadURL returns a presigned PUT URL for uploading an avatar image
|
||||
// directly to MinIO from the client. Signed with the public endpoint so iOS/browser
|
||||
// can PUT bytes straight to MinIO without routing through the server.
|
||||
// ext should be "jpg", "png", or "webp". Expires in 15 minutes.
|
||||
func (m *MinioClient) PresignAvatarUploadURL(ctx context.Context, userID, ext string) (string, string, error) {
|
||||
if m.cfg.BucketAvatars == "" {
|
||||
return "", "", fmt.Errorf("minio: avatars bucket not configured")
|
||||
}
|
||||
key := avatarKey(userID, ext)
|
||||
u, err := m.pub.PresignedPutObject(ctx, m.cfg.BucketAvatars, key, 15*time.Minute)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("minio: presign avatar upload %s: %w", key, err)
|
||||
}
|
||||
return u.String(), key, nil
|
||||
}
|
||||
|
||||
// PresignAvatarURL returns a presigned GET URL for a user avatar.
|
||||
// Returns ("", false, nil) when no avatar exists for the given userID.
|
||||
func (m *MinioClient) PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) {
|
||||
if m.cfg.BucketAvatars == "" {
|
||||
return "", false, nil
|
||||
}
|
||||
// Try common extensions in order of preference.
|
||||
for _, ext := range []string{"jpg", "png", "webp", "gif"} {
|
||||
key := avatarKey(userID, ext)
|
||||
_, statErr := m.c.StatObject(ctx, m.cfg.BucketAvatars, key, minio.StatObjectOptions{})
|
||||
if statErr != nil {
|
||||
continue
|
||||
}
|
||||
u, err := m.pub.PresignedGetObject(ctx, m.cfg.BucketAvatars, key, 24*time.Hour, nil)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("minio: presign avatar %s: %w", key, err)
|
||||
}
|
||||
return u.String(), true, nil
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// DeleteAvatar removes any existing avatar for the given userID (all extensions).
|
||||
func (m *MinioClient) DeleteAvatar(ctx context.Context, userID string) error {
|
||||
if m.cfg.BucketAvatars == "" {
|
||||
return nil
|
||||
}
|
||||
for _, ext := range []string{"jpg", "png", "webp", "gif"} {
|
||||
key := avatarKey(userID, ext)
|
||||
_ = m.c.RemoveObject(ctx, m.cfg.BucketAvatars, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
926
scraper/internal/storage/pocketbase.go
Normal file
926
scraper/internal/storage/pocketbase.go
Normal file
@@ -0,0 +1,926 @@
|
||||
// Package storage — PocketBase REST client.
|
||||
//
|
||||
// Collections expected in PocketBase:
|
||||
//
|
||||
// books — slug(text,unique), title, author, cover, status, genres(json),
|
||||
// summary, total_chapters(number), source_url, ranking(number), updated(date)
|
||||
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
|
||||
// ranking — rank(number), slug(text,unique), title, author, cover, status,
|
||||
// genres(json), source_url, updated(date)
|
||||
// progress — session_id(text), slug(text), chapter(number), updated(date)
|
||||
// audio_cache — cache_key(text,unique), filename(text), updated(date)
|
||||
// app_users — username(text,unique), password_hash(text), role(text), created(date)
|
||||
// scraping_tasks — id(auto), kind(text), target_url(text), status(text),
|
||||
// books_found(number), chapters_scraped(number),
|
||||
// chapters_skipped(number), errors(number),
|
||||
// started(date), finished(date), error_message(text)
|
||||
// user_sessions — user_id(text), session_id(text,unique), user_agent(text),
|
||||
// ip(text), created_at(date), last_seen(date)
|
||||
// book_comments — slug(text), user_id(text), username(text), body(text),
|
||||
// upvotes(number), downvotes(number), created(date)
|
||||
// comment_votes — comment_id(text), user_id(text), session_id(text), vote(text: up|down)
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// PocketBaseConfig holds PocketBase connection settings.
|
||||
type PocketBaseConfig struct {
|
||||
BaseURL string // e.g. "http://pocketbase:8090"
|
||||
AdminEmail string
|
||||
AdminPassword string
|
||||
}
|
||||
|
||||
// pbClient is a minimal PocketBase admin REST client.
|
||||
type pbClient struct {
|
||||
cfg PocketBaseConfig
|
||||
httpClient *http.Client
|
||||
log *slog.Logger
|
||||
|
||||
tokenMu sync.RWMutex
|
||||
token string
|
||||
tokenExp time.Time
|
||||
}
|
||||
|
||||
// newPBClient creates a new PocketBase client. It does not authenticate yet;
|
||||
// authentication happens lazily on the first API call.
|
||||
func newPBClient(cfg PocketBaseConfig, log *slog.Logger) *pbClient {
|
||||
return &pbClient{
|
||||
cfg: cfg,
|
||||
httpClient: &http.Client{Timeout: 15 * time.Second},
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Auth ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
func (p *pbClient) authenticate(ctx context.Context) error {
|
||||
body, _ := json.Marshal(map[string]string{
|
||||
"identity": p.cfg.AdminEmail,
|
||||
"password": p.cfg.AdminPassword,
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
p.cfg.BaseURL+"/api/collections/_superusers/auth-with-password", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pocketbase: auth: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pocketbase: auth status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
var result struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return fmt.Errorf("pocketbase: decode auth: %w", err)
|
||||
}
|
||||
p.tokenMu.Lock()
|
||||
p.token = result.Token
|
||||
p.tokenExp = time.Now().Add(12 * time.Hour)
|
||||
p.tokenMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *pbClient) authToken(ctx context.Context) (string, error) {
|
||||
p.tokenMu.RLock()
|
||||
tok, exp := p.token, p.tokenExp
|
||||
p.tokenMu.RUnlock()
|
||||
if tok != "" && time.Now().Before(exp) {
|
||||
return tok, nil
|
||||
}
|
||||
if err := p.authenticate(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
p.tokenMu.RLock()
|
||||
defer p.tokenMu.RUnlock()
|
||||
return p.token, nil
|
||||
}
|
||||
|
||||
// ─── Generic CRUD helpers ──────────────────────────────────────────────────────
|
||||
|
||||
func (p *pbClient) do(ctx context.Context, method, path string, body interface{}) (*http.Response, error) {
|
||||
tok, err := p.authToken(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var bodyReader io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
bodyReader = bytes.NewReader(b)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, p.cfg.BaseURL+path, bodyReader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+tok)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
return p.httpClient.Do(req)
|
||||
}
|
||||
|
||||
// listOne fetches the first matching record from a collection.
|
||||
func (p *pbClient) listOne(ctx context.Context, collection, filter string) (map[string]interface{}, error) {
|
||||
q := url.Values{}
|
||||
q.Set("filter", filter)
|
||||
q.Set("perPage", "1")
|
||||
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||
resp, err := p.do(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("pocketbase: listOne %s: status %d: %s", collection, resp.StatusCode, b)
|
||||
}
|
||||
var result struct {
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("pocketbase: listOne %s: decode: %w", collection, err)
|
||||
}
|
||||
if len(result.Items) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return result.Items[0], nil
|
||||
}
|
||||
|
||||
// listAll returns all records from a collection matching filter by paginating
|
||||
// through all pages (PocketBase default page size is capped at 500).
|
||||
func (p *pbClient) listAll(ctx context.Context, collection, filter, sort string) ([]map[string]interface{}, error) {
|
||||
const perPage = 500
|
||||
var all []map[string]interface{}
|
||||
|
||||
for page := 1; ; page++ {
|
||||
q := url.Values{}
|
||||
if filter != "" {
|
||||
q.Set("filter", filter)
|
||||
}
|
||||
if sort != "" {
|
||||
q.Set("sort", sort)
|
||||
}
|
||||
q.Set("perPage", fmt.Sprintf("%d", perPage))
|
||||
q.Set("page", fmt.Sprintf("%d", page))
|
||||
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||
resp, err := p.do(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("pocketbase: listAll %s: status %d: %s", collection, resp.StatusCode, b)
|
||||
}
|
||||
var result struct {
|
||||
Page int `json:"page"`
|
||||
TotalPages int `json:"totalPages"`
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("pocketbase: listAll %s: decode: %w", collection, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
all = append(all, result.Items...)
|
||||
if page >= result.TotalPages || len(result.Items) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// upsert creates a record; if one matching filter already exists it updates it.
|
||||
func (p *pbClient) upsert(ctx context.Context, collection, filter string, data map[string]interface{}) error {
|
||||
existing, err := p.listOne(ctx, collection, filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if existing != nil {
|
||||
id := existing["id"].(string)
|
||||
resp, err := p.do(ctx, http.MethodPatch,
|
||||
fmt.Sprintf("/api/collections/%s/records/%s", collection, id), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pocketbase: upsert (patch) %s id=%s: status %d: %s", collection, id, resp.StatusCode, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
resp, err := p.do(ctx, http.MethodPost,
|
||||
fmt.Sprintf("/api/collections/%s/records", collection), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pocketbase: upsert (create) %s: status %d: %s", collection, resp.StatusCode, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteWhere deletes all records matching filter in collection.
|
||||
func (p *pbClient) deleteWhere(ctx context.Context, collection, filter string) error {
|
||||
items, err := p.listAll(ctx, collection, filter, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range items {
|
||||
id, _ := item["id"].(string)
|
||||
resp, err := p.do(ctx, http.MethodDelete,
|
||||
fmt.Sprintf("/api/collections/%s/records/%s", collection, id), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return fmt.Errorf("pocketbase: deleteWhere %s id=%s: status %d: %s", collection, id, resp.StatusCode, b)
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── PocketBaseStore ──────────────────────────────────────────────────────────
|
||||
|
||||
// PocketBaseStore implements the structured-data portion of the Store interface
|
||||
// backed by PocketBase REST API.
|
||||
type PocketBaseStore struct {
|
||||
pb *pbClient
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewPocketBaseStore returns a connected PocketBaseStore.
|
||||
func NewPocketBaseStore(cfg PocketBaseConfig, log *slog.Logger) *PocketBaseStore {
|
||||
return &PocketBaseStore{pb: newPBClient(cfg, log), log: log}
|
||||
}
|
||||
|
||||
// Ping verifies connectivity by authenticating.
|
||||
func (s *PocketBaseStore) Ping(ctx context.Context) error {
|
||||
_, err := s.pb.authToken(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// ─── Collections schema bootstrap ────────────────────────────────────────────
|
||||
// CollectionDef maps a collection name to its fields for auto-creation.
|
||||
|
||||
// EnsureCollections creates missing collections via the PocketBase API.
|
||||
// Safe to call on every startup — existing collections are skipped.
|
||||
func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
|
||||
// We just attempt to create each collection; 400/422 errors for "already
|
||||
// exists" are silently ignored.
|
||||
// PocketBase v0.22+ uses "fields"; older versions used "schema".
|
||||
// We use "fields" which is the current API.
|
||||
collections := []map[string]interface{}{
|
||||
{
|
||||
"name": "books",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "title", "type": "text", "required": true},
|
||||
{"name": "author", "type": "text"},
|
||||
{"name": "cover", "type": "text"},
|
||||
{"name": "status", "type": "text"},
|
||||
{"name": "genres", "type": "json"},
|
||||
{"name": "summary", "type": "text"},
|
||||
{"name": "total_chapters", "type": "number"},
|
||||
{"name": "source_url", "type": "text"},
|
||||
{"name": "ranking", "type": "number"},
|
||||
{"name": "meta_updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "chapters_idx",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "number", "type": "number", "required": true},
|
||||
{"name": "title", "type": "text"},
|
||||
{"name": "date_label", "type": "text"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "ranking",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "rank", "type": "number", "required": true},
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "title", "type": "text"},
|
||||
{"name": "author", "type": "text"},
|
||||
{"name": "cover", "type": "text"},
|
||||
{"name": "status", "type": "text"},
|
||||
{"name": "genres", "type": "json"},
|
||||
{"name": "source_url", "type": "text"},
|
||||
{"name": "updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "progress",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "session_id", "type": "text", "required": true},
|
||||
{"name": "user_id", "type": "text"},
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "chapter", "type": "number"},
|
||||
{"name": "updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "audio_cache",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "cache_key", "type": "text", "required": true},
|
||||
{"name": "filename", "type": "text"},
|
||||
{"name": "updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "app_users",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "username", "type": "text", "required": true},
|
||||
{"name": "password_hash", "type": "text", "required": true},
|
||||
{"name": "role", "type": "text"},
|
||||
{"name": "created", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "user_library",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "session_id", "type": "text", "required": true},
|
||||
{"name": "user_id", "type": "text"},
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "saved_at", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "scraping_tasks",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "kind", "type": "text", "required": true}, // "catalogue" | "book"
|
||||
{"name": "target_url", "type": "text"}, // set for single-book scrapes
|
||||
{"name": "status", "type": "text", "required": true}, // "running" | "done" | "failed" | "cancelled"
|
||||
{"name": "books_found", "type": "number"},
|
||||
{"name": "chapters_scraped", "type": "number"},
|
||||
{"name": "chapters_skipped", "type": "number"},
|
||||
{"name": "errors", "type": "number"},
|
||||
{"name": "started", "type": "date"},
|
||||
{"name": "finished", "type": "date"},
|
||||
{"name": "error_message", "type": "text"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "audio_jobs",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "cache_key", "type": "text", "required": true}, // "slug/chapter/voice"
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "chapter", "type": "number"},
|
||||
{"name": "voice", "type": "text"},
|
||||
{"name": "status", "type": "text", "required": true}, // "pending" | "generating" | "done" | "failed"
|
||||
{"name": "error_message", "type": "text"},
|
||||
{"name": "started", "type": "date"},
|
||||
{"name": "finished", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "user_sessions",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "user_id", "type": "text", "required": true},
|
||||
{"name": "session_id", "type": "text", "required": true}, // random ID embedded in auth token
|
||||
{"name": "user_agent", "type": "text"},
|
||||
{"name": "ip", "type": "text"},
|
||||
{"name": "created_at", "type": "date"},
|
||||
{"name": "last_seen", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "book_comments",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "user_id", "type": "text"},
|
||||
{"name": "username", "type": "text"},
|
||||
{"name": "body", "type": "text", "required": true},
|
||||
{"name": "upvotes", "type": "number"},
|
||||
{"name": "downvotes", "type": "number"},
|
||||
{"name": "created", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "comment_votes",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "comment_id", "type": "text", "required": true},
|
||||
{"name": "user_id", "type": "text"},
|
||||
{"name": "session_id", "type": "text", "required": true},
|
||||
{"name": "vote", "type": "text", "required": true}, // "up" | "down"
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, col := range collections {
|
||||
name, _ := col["name"].(string)
|
||||
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections", col)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pocketbase: ensure collection %q: %w", name, err)
|
||||
}
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK, http.StatusCreated:
|
||||
s.log.Info("pocketbase: collection created", "collection", name)
|
||||
case http.StatusBadRequest, http.StatusUnprocessableEntity:
|
||||
// Already exists or schema mismatch — expected on subsequent startups.
|
||||
s.log.Debug("pocketbase: collection already exists (skipped)", "collection", name)
|
||||
default:
|
||||
s.log.Warn("pocketbase: unexpected status ensuring collection",
|
||||
"collection", name, "status", resp.StatusCode, "body", string(b))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── Schema migrations ────────────────────────────────────────────────────────
|
||||
|
||||
// migration describes a single field to guarantee exists in a collection.
|
||||
type migration struct {
|
||||
collection string
|
||||
fieldName string
|
||||
fieldType string
|
||||
}
|
||||
|
||||
// migrations is the ordered list of schema changes applied on every startup.
|
||||
var migrations = []migration{
|
||||
// user_id was added to progress after initial deploy.
|
||||
{"progress", "user_id", "text"},
|
||||
// avatar_url stores the MinIO presign path for the user's profile picture.
|
||||
{"app_users", "avatar_url", "text"},
|
||||
}
|
||||
|
||||
// EnsureMigrations idempotently adds any fields that are missing from existing
|
||||
// collections. It fetches the current schema, checks for each field by name,
|
||||
// and PATCHes the collection only when something is absent.
|
||||
// Safe to call on every startup — no-ops when schema is already up to date.
|
||||
func (s *PocketBaseStore) EnsureMigrations(ctx context.Context) error {
|
||||
for _, m := range migrations {
|
||||
if err := s.ensureField(ctx, m); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) ensureField(ctx context.Context, m migration) error {
|
||||
// Fetch current collection schema.
|
||||
resp, err := s.pb.do(ctx, http.MethodGet, "/api/collections/"+m.collection, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pocketbase: ensureField %s.%s: fetch schema: %w", m.collection, m.fieldName, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("pocketbase: ensureField %s.%s: fetch schema status %d: %s", m.collection, m.fieldName, resp.StatusCode, body)
|
||||
}
|
||||
|
||||
var schema struct {
|
||||
ID string `json:"id"`
|
||||
Fields []map[string]interface{} `json:"fields"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &schema); err != nil {
|
||||
return fmt.Errorf("pocketbase: ensureField %s.%s: decode schema: %w", m.collection, m.fieldName, err)
|
||||
}
|
||||
|
||||
// Check if field already exists.
|
||||
for _, f := range schema.Fields {
|
||||
if name, _ := f["name"].(string); name == m.fieldName {
|
||||
s.log.Debug("pocketbase: field already exists, skipping migration",
|
||||
"collection", m.collection, "field", m.fieldName)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Append the new field and PATCH the collection.
|
||||
newFields := append(schema.Fields, map[string]interface{}{
|
||||
"name": m.fieldName,
|
||||
"type": m.fieldType,
|
||||
})
|
||||
patch := map[string]interface{}{"fields": newFields}
|
||||
patchResp, err := s.pb.do(ctx, http.MethodPatch, "/api/collections/"+schema.ID, patch)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pocketbase: ensureField %s.%s: patch: %w", m.collection, m.fieldName, err)
|
||||
}
|
||||
defer patchResp.Body.Close()
|
||||
patchBody, _ := io.ReadAll(patchResp.Body)
|
||||
if patchResp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("pocketbase: ensureField %s.%s: patch status %d: %s", m.collection, m.fieldName, patchResp.StatusCode, patchBody)
|
||||
}
|
||||
s.log.Info("pocketbase: schema migration applied", "collection", m.collection, "field", m.fieldName, "type", m.fieldType)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── Book metadata ────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) UpsertBook(ctx context.Context, slug, title, author, cover, status, summary, sourceURL string, genres []string, totalChapters, ranking int) error {
|
||||
genresJSON, _ := json.Marshal(genres)
|
||||
return s.pb.upsert(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)), map[string]interface{}{
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
"author": author,
|
||||
"cover": cover,
|
||||
"status": status,
|
||||
"genres": string(genresJSON),
|
||||
"summary": summary,
|
||||
"total_chapters": totalChapters,
|
||||
"source_url": sourceURL,
|
||||
"ranking": ranking,
|
||||
"meta_updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetBook(ctx context.Context, slug string) (map[string]interface{}, bool, error) {
|
||||
rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if rec == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
return rec, true, nil
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) ListBooks(ctx context.Context) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "books", "", "+title")
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) BookMetaUpdated(ctx context.Context, slug string) (time.Time, error) {
|
||||
rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)))
|
||||
if err != nil || rec == nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
if ts, ok := rec["meta_updated"].(string); ok {
|
||||
t, err := time.Parse(time.RFC3339, ts)
|
||||
if err == nil {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, nil
|
||||
}
|
||||
|
||||
// ─── Chapter index ────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) UpsertChapterIdx(ctx context.Context, slug string, number int, title, dateLabel string) error {
|
||||
return s.pb.upsert(ctx, "chapters_idx",
|
||||
fmt.Sprintf(`slug="%s"&&number=%d`, pbEsc(slug), number),
|
||||
map[string]interface{}{
|
||||
"slug": slug,
|
||||
"number": number,
|
||||
"title": title,
|
||||
"date_label": dateLabel,
|
||||
})
|
||||
}
|
||||
|
||||
// WriteChapterRefs upserts chapter index rows (number + title) for all refs
|
||||
// without writing any chapter text. Errors are logged and skipped; the
|
||||
// operation is best-effort.
|
||||
func (s *PocketBaseStore) WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error {
|
||||
var firstErr error
|
||||
for _, ref := range refs {
|
||||
if err := s.UpsertChapterIdx(ctx, slug, ref.Number, ref.Title, ""); err != nil {
|
||||
s.log.Warn("pocketbase: WriteChapterRefs: upsert failed",
|
||||
"slug", slug, "chapter", ref.Number, "err", err)
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) ListChapterIdx(ctx context.Context, slug string) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "chapters_idx",
|
||||
fmt.Sprintf(`slug="%s"`, pbEsc(slug)), "+number")
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) CountChapterIdx(ctx context.Context, slug string) int {
|
||||
rows, err := s.ListChapterIdx(ctx, slug)
|
||||
if err != nil {
|
||||
s.log.Warn("pocketbase: CountChapterIdx failed", "slug", slug, "err", err)
|
||||
return 0
|
||||
}
|
||||
return len(rows)
|
||||
}
|
||||
|
||||
// ─── Ranking (per-item) ───────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) UpsertRankingItem(ctx context.Context, item RankingItem) error {
|
||||
genresJSON, _ := json.Marshal(item.Genres)
|
||||
return s.pb.upsert(ctx, "ranking", fmt.Sprintf(`slug="%s"`, pbEsc(item.Slug)), map[string]interface{}{
|
||||
"rank": item.Rank,
|
||||
"slug": item.Slug,
|
||||
"title": item.Title,
|
||||
"author": item.Author,
|
||||
"cover": item.Cover,
|
||||
"status": item.Status,
|
||||
"genres": string(genresJSON),
|
||||
"source_url": item.SourceURL,
|
||||
"updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) ListRankingItems(ctx context.Context) ([]RankingItem, error) {
|
||||
rows, err := s.pb.listAll(ctx, "ranking", "", "+rank")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := make([]RankingItem, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
item := RankingItem{
|
||||
Rank: int(floatVal(r, "rank")),
|
||||
Slug: strVal(r, "slug"),
|
||||
Title: strVal(r, "title"),
|
||||
Author: strVal(r, "author"),
|
||||
Cover: strVal(r, "cover"),
|
||||
Status: strVal(r, "status"),
|
||||
SourceURL: strVal(r, "source_url"),
|
||||
}
|
||||
if ts, ok := r["updated"].(string); ok {
|
||||
item.Updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
switch v := r["genres"].(type) {
|
||||
case string:
|
||||
_ = json.Unmarshal([]byte(v), &item.Genres)
|
||||
case []interface{}:
|
||||
for _, g := range v {
|
||||
if s, ok := g.(string); ok {
|
||||
item.Genres = append(item.Genres, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// RankingLastUpdated returns the most recent Updated time across all ranking rows,
|
||||
// or the zero time if no rows exist.
|
||||
func (s *PocketBaseStore) RankingLastUpdated(ctx context.Context) (time.Time, error) {
|
||||
// listAll with sort "-updated" and perPage=1 is the cheapest approach.
|
||||
q := url.Values{}
|
||||
q.Set("sort", "-updated")
|
||||
q.Set("perPage", "1")
|
||||
path := fmt.Sprintf("/api/collections/ranking/records?%s", q.Encode())
|
||||
resp, err := s.pb.do(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return time.Time{}, fmt.Errorf("pocketbase: RankingLastUpdated: status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
var result struct {
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return time.Time{}, fmt.Errorf("pocketbase: RankingLastUpdated: decode: %w", err)
|
||||
}
|
||||
if len(result.Items) == 0 {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
ts, _ := result.Items[0]["updated"].(string)
|
||||
t, _ := time.Parse(time.RFC3339, ts)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// ─── Reading progress ─────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) SetProgress(ctx context.Context, sessionID, slug string, chapter int) error {
|
||||
return s.pb.upsert(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)),
|
||||
map[string]interface{}{
|
||||
"session_id": sessionID,
|
||||
"slug": slug,
|
||||
"chapter": chapter,
|
||||
"updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetProgress(ctx context.Context, sessionID, slug string) (int, time.Time, bool, error) {
|
||||
rec, err := s.pb.listOne(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)))
|
||||
if err != nil {
|
||||
return 0, time.Time{}, false, err
|
||||
}
|
||||
if rec == nil {
|
||||
return 0, time.Time{}, false, nil
|
||||
}
|
||||
ch := int(floatVal(rec, "chapter"))
|
||||
var updated time.Time
|
||||
if ts, ok := rec["updated"].(string); ok {
|
||||
updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
return ch, updated, true, nil
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) AllProgress(ctx context.Context, sessionID string) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"`, pbEsc(sessionID)), "-updated")
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) DeleteProgress(ctx context.Context, sessionID, slug string) error {
|
||||
return s.pb.deleteWhere(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)))
|
||||
}
|
||||
|
||||
// ─── Audio cache ──────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error {
|
||||
return s.pb.upsert(ctx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)),
|
||||
map[string]interface{}{
|
||||
"cache_key": cacheKey,
|
||||
"filename": filename,
|
||||
"updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool, error) {
|
||||
rec, err := s.pb.listOne(ctx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)))
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if rec == nil {
|
||||
return "", false, nil
|
||||
}
|
||||
filename, _ := rec["filename"].(string)
|
||||
return filename, filename != "", nil
|
||||
}
|
||||
|
||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
// CreateScrapingTask inserts a new scraping_tasks record with status="running"
|
||||
// and returns the newly created record's ID.
|
||||
func (s *PocketBaseStore) CreateScrapingTask(ctx context.Context, kind, targetURL string) (string, error) {
|
||||
data := map[string]interface{}{
|
||||
"kind": kind,
|
||||
"target_url": targetURL,
|
||||
"status": "running",
|
||||
"books_found": 0,
|
||||
"chapters_scraped": 0,
|
||||
"chapters_skipped": 0,
|
||||
"errors": 0,
|
||||
"started": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections/scraping_tasks/records", data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
||||
return "", fmt.Errorf("pocketbase: CreateScrapingTask: status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
var rec map[string]interface{}
|
||||
if err := json.Unmarshal(b, &rec); err != nil {
|
||||
return "", fmt.Errorf("pocketbase: CreateScrapingTask: decode: %w", err)
|
||||
}
|
||||
id, _ := rec["id"].(string)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// UpdateScrapingTask patches counters on an existing scraping_tasks record.
|
||||
func (s *PocketBaseStore) UpdateScrapingTask(ctx context.Context, id string, data map[string]interface{}) error {
|
||||
resp, err := s.pb.do(ctx, http.MethodPatch,
|
||||
fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pocketbase: UpdateScrapingTask id=%s: status %d: %s", id, resp.StatusCode, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListScrapingTasks returns all scraping_tasks sorted by started descending.
|
||||
func (s *PocketBaseStore) ListScrapingTasks(ctx context.Context) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "scraping_tasks", "", "-started")
|
||||
}
|
||||
|
||||
// ─── Audio jobs ───────────────────────────────────────────────────────────────
|
||||
|
||||
// CreateAudioJob inserts a new audio_jobs record with status="pending".
|
||||
func (s *PocketBaseStore) CreateAudioJob(ctx context.Context, slug string, chapter int, voice string) (string, error) {
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, voice)
|
||||
data := map[string]interface{}{
|
||||
"cache_key": cacheKey,
|
||||
"slug": slug,
|
||||
"chapter": chapter,
|
||||
"voice": voice,
|
||||
"status": "pending",
|
||||
"started": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections/audio_jobs/records", data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
||||
return "", fmt.Errorf("pocketbase: CreateAudioJob: status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
var rec map[string]interface{}
|
||||
if err := json.Unmarshal(b, &rec); err != nil {
|
||||
return "", fmt.Errorf("pocketbase: CreateAudioJob: decode: %w", err)
|
||||
}
|
||||
id, _ := rec["id"].(string)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// UpdateAudioJob patches status, error_message, and optionally finished on an audio_jobs record.
|
||||
func (s *PocketBaseStore) UpdateAudioJob(ctx context.Context, id, status, errMsg string, finished time.Time) error {
|
||||
data := map[string]interface{}{
|
||||
"status": status,
|
||||
"error_message": errMsg,
|
||||
}
|
||||
if !finished.IsZero() {
|
||||
data["finished"] = finished.UTC().Format(time.RFC3339)
|
||||
}
|
||||
resp, err := s.pb.do(ctx, http.MethodPatch,
|
||||
fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pocketbase: UpdateAudioJob id=%s: status %d: %s", id, resp.StatusCode, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAudioJob returns the most recent audio_jobs record for the given cache key.
|
||||
func (s *PocketBaseStore) GetAudioJob(ctx context.Context, cacheKey string) (map[string]interface{}, bool, error) {
|
||||
rec, err := s.pb.listOne(ctx, "audio_jobs",
|
||||
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if rec == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
return rec, true, nil
|
||||
}
|
||||
|
||||
// ListAudioJobs returns all audio_jobs sorted by started descending.
|
||||
func (s *PocketBaseStore) ListAudioJobs(ctx context.Context) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "audio_jobs", "", "-started")
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// pbEsc escapes a string for use in a PocketBase filter expression.
|
||||
// Only escapes double-quotes to prevent injection.
|
||||
func pbEsc(s string) string {
|
||||
return strings.ReplaceAll(s, `"`, `\"`)
|
||||
}
|
||||
|
||||
func floatVal(m map[string]interface{}, key string) float64 {
|
||||
if v, ok := m[key].(float64); ok {
|
||||
return v
|
||||
}
|
||||
return 0
|
||||
}
|
||||
203
scraper/internal/storage/scrape_integration_test.go
Normal file
203
scraper/internal/storage/scrape_integration_test.go
Normal file
@@ -0,0 +1,203 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests that combine live scraping (Browserless) with real storage
|
||||
// (MinIO + PocketBase) via HybridStore.
|
||||
//
|
||||
// These tests require ALL THREE services to be running. They are gated behind
|
||||
// the "integration" build tag and skipped when any service URL is missing.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// BROWSERLESS_URL=http://localhost:3030 \
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 600s \
|
||||
// github.com/libnovel/scraper/internal/storage
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/browser"
|
||||
"github.com/libnovel/scraper/internal/novelfire"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
const (
|
||||
scrapeTestBookURL = "https://novelfire.net/book/a-dragon-against-the-whole-world"
|
||||
scrapeTestBookSlug = "a-dragon-against-the-whole-world"
|
||||
)
|
||||
|
||||
// newScrapeAndStoreFixture builds a novelfire Scraper and a HybridStore,
|
||||
// skipping the test if any required env var is absent.
|
||||
func newScrapeAndStoreFixture(t *testing.T) (*novelfire.Scraper, *HybridStore) {
|
||||
t.Helper()
|
||||
|
||||
browserlessURL := os.Getenv("BROWSERLESS_URL")
|
||||
if browserlessURL == "" {
|
||||
t.Skip("BROWSERLESS_URL not set — skipping scrape+store integration test")
|
||||
}
|
||||
if os.Getenv("MINIO_ENDPOINT") == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping scrape+store integration test")
|
||||
}
|
||||
if os.Getenv("POCKETBASE_URL") == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping scrape+store integration test")
|
||||
}
|
||||
|
||||
client := browser.NewContentClient(browser.Config{
|
||||
BaseURL: browserlessURL,
|
||||
Token: os.Getenv("BROWSERLESS_TOKEN"),
|
||||
Timeout: 120 * time.Second,
|
||||
MaxConcurrent: 1,
|
||||
})
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
sc := novelfire.New(client, log, client, nil, nil)
|
||||
hs := newTestHybridStore(t)
|
||||
return sc, hs
|
||||
}
|
||||
|
||||
// TestScrapeAndStore_BookMetadata scrapes the test book's metadata and stores
|
||||
// it via HybridStore.WriteMetadata, then verifies a ReadMetadata round-trip.
|
||||
func TestScrapeAndStore_BookMetadata(t *testing.T) {
|
||||
sc, hs := newScrapeAndStoreFixture(t)
|
||||
|
||||
slug := scrapeTestBookSlug + "-scrapetest"
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
// 1. Scrape metadata from the live site.
|
||||
scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer scrapeCancel()
|
||||
|
||||
meta, err := sc.ScrapeMetadata(scrapeCtx, scrapeTestBookURL)
|
||||
if err != nil {
|
||||
t.Fatalf("ScrapeMetadata: %v", err)
|
||||
}
|
||||
t.Logf("scraped: slug=%q title=%q author=%q totalChapters=%d",
|
||||
meta.Slug, meta.Title, meta.Author, meta.TotalChapters)
|
||||
|
||||
// Override slug with our test-specific value to avoid polluting real data.
|
||||
meta.Slug = slug
|
||||
|
||||
// 2. Write to HybridStore.
|
||||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer storeCancel()
|
||||
|
||||
if err := hs.WriteMetadata(storeCtx, meta); err != nil {
|
||||
t.Fatalf("WriteMetadata: %v", err)
|
||||
}
|
||||
|
||||
// 3. Read back and verify.
|
||||
got, found, err := hs.ReadMetadata(storeCtx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMetadata: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("ReadMetadata: not found after WriteMetadata")
|
||||
}
|
||||
|
||||
t.Logf("read back: title=%q author=%q totalChapters=%d", got.Title, got.Author, got.TotalChapters)
|
||||
|
||||
if got.Title == "" {
|
||||
t.Error("Title is empty after round-trip")
|
||||
}
|
||||
if got.Author == "" {
|
||||
t.Error("Author is empty after round-trip")
|
||||
}
|
||||
if got.TotalChapters < 1 {
|
||||
t.Errorf("TotalChapters = %d, want >= 1", got.TotalChapters)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScrapeAndStore_First3Chapters scrapes chapters 1, 2, and 3 from the
|
||||
// live site and stores each via HybridStore.WriteChapter, then verifies
|
||||
// ReadChapter returns non-empty markdown with the expected header.
|
||||
func TestScrapeAndStore_First3Chapters(t *testing.T) {
|
||||
sc, hs := newScrapeAndStoreFixture(t)
|
||||
|
||||
// Use a unique test slug so we don't pollute the real book.
|
||||
slug := fmt.Sprintf("%s-chtest-%d", scrapeTestBookSlug, time.Now().UnixMilli()%100000)
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
// Pre-build chapter refs (known URLs for this test book).
|
||||
refs := []scraper.ChapterRef{
|
||||
{Number: 1, Title: "Chapter 1", Volume: 0, URL: scrapeTestBookURL + "/chapter-1"},
|
||||
{Number: 2, Title: "Chapter 2", Volume: 0, URL: scrapeTestBookURL + "/chapter-2"},
|
||||
{Number: 3, Title: "Chapter 3", Volume: 0, URL: scrapeTestBookURL + "/chapter-3"},
|
||||
}
|
||||
|
||||
for _, ref := range refs {
|
||||
ref := ref // capture loop variable
|
||||
t.Run(fmt.Sprintf("chapter-%d", ref.Number), func(t *testing.T) {
|
||||
// 1. Scrape chapter text.
|
||||
scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer scrapeCancel()
|
||||
|
||||
ch, err := sc.ScrapeChapterText(scrapeCtx, ref)
|
||||
if err != nil {
|
||||
t.Fatalf("ScrapeChapterText(%d): %v", ref.Number, err)
|
||||
}
|
||||
t.Logf("scraped chapter %d: %d bytes of markdown", ref.Number, len(ch.Text))
|
||||
|
||||
if len(ch.Text) < 100 {
|
||||
t.Errorf("scraped text too short (%d bytes)", len(ch.Text))
|
||||
}
|
||||
|
||||
// 2. Write to HybridStore.
|
||||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer storeCancel()
|
||||
|
||||
if err := hs.WriteChapter(storeCtx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter(%d): %v", ref.Number, err)
|
||||
}
|
||||
|
||||
// 3. Read back and verify.
|
||||
got, err := hs.ReadChapter(storeCtx, slug, ref.Number)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadChapter(%d): %v", ref.Number, err)
|
||||
}
|
||||
if got == "" {
|
||||
t.Fatalf("ReadChapter(%d): returned empty string", ref.Number)
|
||||
}
|
||||
if len(got) < 100 {
|
||||
t.Errorf("ReadChapter(%d): content too short (%d bytes)", ref.Number, len(got))
|
||||
}
|
||||
|
||||
// WriteChapter prepends "# <title>\n\n".
|
||||
if !strings.HasPrefix(got, "# ") {
|
||||
t.Errorf("chapter %d: stored content does not start with markdown header: %q",
|
||||
ref.Number, got[:min(len(got), 60)])
|
||||
}
|
||||
|
||||
// Verify the original scraped text body is present.
|
||||
if !strings.Contains(got, ch.Text[:min(len(ch.Text), 50)]) {
|
||||
t.Errorf("chapter %d: stored content does not contain scraped text excerpt", ref.Number)
|
||||
}
|
||||
|
||||
t.Logf("chapter %d stored and verified: %d bytes", ref.Number, len(got))
|
||||
})
|
||||
}
|
||||
|
||||
// After all chapters written, verify count.
|
||||
countCtx, countCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer countCancel()
|
||||
|
||||
count := hs.CountChapters(countCtx, slug)
|
||||
if count != len(refs) {
|
||||
t.Errorf("CountChapters = %d, want %d", count, len(refs))
|
||||
}
|
||||
}
|
||||
218
scraper/internal/storage/store.go
Normal file
218
scraper/internal/storage/store.go
Normal file
@@ -0,0 +1,218 @@
|
||||
// Package storage defines the unified Store interface and helper types used by
|
||||
// the server and orchestrator. Concrete implementations back the interface
|
||||
// with PocketBase (structured data) and MinIO (binary objects).
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// ─── Shared types ─────────────────────────────────────────────────────────────
|
||||
|
||||
// ChapterInfo is a lightweight chapter descriptor (mirrors writer.ChapterInfo).
|
||||
type ChapterInfo struct {
|
||||
Number int
|
||||
Title string
|
||||
Date string
|
||||
}
|
||||
|
||||
// RankingItem represents a single entry in the novel ranking list.
|
||||
// Aliased from scraper.RankingItem for convenience within this package.
|
||||
type RankingItem = scraper.RankingItem
|
||||
|
||||
// ReadingProgress holds a single user's reading position for one book.
|
||||
type ReadingProgress struct {
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// AudioJob represents a single audio-generation job record from the
|
||||
// audio_jobs collection.
|
||||
type AudioJob struct {
|
||||
ID string `json:"id"`
|
||||
CacheKey string `json:"cache_key"` // "slug/chapter/voice"
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
Voice string `json:"voice"`
|
||||
Status string `json:"status"` // "pending" | "generating" | "done" | "failed"
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished time.Time `json:"finished,omitempty"`
|
||||
}
|
||||
|
||||
// ScrapeTask represents a single scraping job record from the scraping_tasks
|
||||
// collection.
|
||||
type ScrapeTask struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"` // "catalogue" | "book"
|
||||
TargetURL string `json:"target_url"` // non-empty for single-book scrapes
|
||||
Status string `json:"status"` // "running" | "done" | "failed" | "cancelled"
|
||||
BooksFound int `json:"books_found"`
|
||||
ChaptersScraped int `json:"chapters_scraped"`
|
||||
ChaptersSkipped int `json:"chapters_skipped"`
|
||||
Errors int `json:"errors"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished time.Time `json:"finished,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
}
|
||||
|
||||
// ScrapeTaskUpdate carries the fields that can be patched on a ScrapeTask.
|
||||
// Zero-value fields are still sent; callers should only include keys they want
|
||||
// to change via the map form used inside the store implementation.
|
||||
type ScrapeTaskUpdate struct {
|
||||
Status string
|
||||
BooksFound int
|
||||
ChaptersScraped int
|
||||
ChaptersSkipped int
|
||||
Errors int
|
||||
Finished time.Time // zero = not finished yet
|
||||
ErrorMessage string
|
||||
}
|
||||
|
||||
// ─── Store interface ──────────────────────────────────────────────────────────
|
||||
|
||||
// Store is the single persistence abstraction consumed by the server and the
|
||||
// orchestrator. Implementations may route calls to different backends
|
||||
// (PocketBase for structured records, MinIO for binary blobs).
|
||||
type Store interface {
|
||||
// ── Book metadata ──────────────────────────────────────────────────────
|
||||
|
||||
// WriteMetadata upserts book metadata.
|
||||
WriteMetadata(ctx context.Context, meta scraper.BookMeta) error
|
||||
// ReadMetadata returns the metadata for slug. Returns (zero, false, nil)
|
||||
// when the book is not found.
|
||||
ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error)
|
||||
// ListBooks returns all books, sorted alphabetically by title.
|
||||
ListBooks(ctx context.Context) ([]scraper.BookMeta, error)
|
||||
// LocalSlugs returns the set of slugs that have metadata stored.
|
||||
LocalSlugs(ctx context.Context) (map[string]bool, error)
|
||||
// MetadataMtime returns the Unix-second mtime of the metadata record, or 0.
|
||||
MetadataMtime(ctx context.Context, slug string) int64
|
||||
|
||||
// ── Chapters (binary blobs in MinIO) ───────────────────────────────────
|
||||
|
||||
// ChapterExists returns true if the markdown file for the given ref exists.
|
||||
ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool
|
||||
// WriteChapter stores the chapter markdown.
|
||||
WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error
|
||||
// WriteChapterRefs persists chapter metadata (number + title) into the
|
||||
// chapters_idx table without fetching or storing any chapter text.
|
||||
// It is used to pre-populate the chapter list when a book is first seen
|
||||
// via a live preview, before its chapter text has been scraped.
|
||||
WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error
|
||||
// ReadChapter returns the raw markdown for chapter number n.
|
||||
ReadChapter(ctx context.Context, slug string, n int) (string, error)
|
||||
// ListChapters returns all stored chapters for slug, sorted by number.
|
||||
ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error)
|
||||
// CountChapters returns the number of stored chapters for slug.
|
||||
CountChapters(ctx context.Context, slug string) int
|
||||
// ReindexChapters rebuilds chapters_idx from MinIO objects for slug.
|
||||
// Returns the number of chapters indexed.
|
||||
ReindexChapters(ctx context.Context, slug string) (int, error)
|
||||
|
||||
// ── Ranking ────────────────────────────────────────────────────────────
|
||||
|
||||
// WriteRankingItem upserts a single ranking entry (keyed on Slug).
|
||||
WriteRankingItem(ctx context.Context, item RankingItem) error
|
||||
// ReadRankingItems returns all ranking items sorted by rank ascending.
|
||||
ReadRankingItems(ctx context.Context) ([]RankingItem, error)
|
||||
// RankingFreshEnough returns true when ranking rows exist and the most
|
||||
// recent Updated timestamp is within maxAge of now.
|
||||
RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error)
|
||||
|
||||
// ── Audio cache ────────────────────────────────────────────────────────
|
||||
|
||||
// GetAudioCache returns the Kokoro filename for cacheKey, or ("", false).
|
||||
GetAudioCache(ctx context.Context, cacheKey string) (string, bool)
|
||||
// SetAudioCache persists a Kokoro filename for cacheKey.
|
||||
SetAudioCache(ctx context.Context, cacheKey, filename string) error
|
||||
// PutAudio stores raw audio bytes under the given MinIO object key.
|
||||
PutAudio(ctx context.Context, key string, data []byte) error
|
||||
|
||||
// ── Reading progress ───────────────────────────────────────────────────
|
||||
|
||||
// GetProgress returns the reading progress for the given session ID and slug.
|
||||
// Returns (zero, false) if no progress is recorded.
|
||||
GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool)
|
||||
// SetProgress saves or updates reading progress.
|
||||
SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error
|
||||
// AllProgress returns all progress entries for a session.
|
||||
AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error)
|
||||
// DeleteProgress removes progress for a specific slug.
|
||||
DeleteProgress(ctx context.Context, sessionID, slug string) error
|
||||
|
||||
// ── Audio object paths (MinIO) ─────────────────────────────────────────
|
||||
|
||||
// AudioObjectKey returns the MinIO object key for a cached audio file.
|
||||
AudioObjectKey(slug string, n int, voice string) string
|
||||
// AudioExists returns true when the audio object is present in the bucket.
|
||||
AudioExists(ctx context.Context, key string) bool
|
||||
|
||||
// ── Presigned URLs ─────────────────────────────────────────────────────
|
||||
|
||||
// PresignChapter returns a presigned GET URL for a chapter markdown object.
|
||||
PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error)
|
||||
|
||||
// PresignAudio returns a presigned GET URL for an audio object.
|
||||
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
|
||||
|
||||
// PresignAvatarUpload returns a short-lived presigned PUT URL for uploading
|
||||
// an avatar image directly to MinIO, and the object key that will be stored.
|
||||
// ext should be "jpg", "png", or "webp".
|
||||
PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error)
|
||||
|
||||
// PresignAvatarURL returns a presigned GET URL for a user's avatar, or ("", false, nil) if none.
|
||||
PresignAvatarURL(ctx context.Context, userID string) (string, bool, error)
|
||||
|
||||
// DeleteAvatar removes all avatar objects for a user (all extensions).
|
||||
DeleteAvatar(ctx context.Context, userID string) error
|
||||
|
||||
// ── Browse page snapshots (MinIO) ──────────────────────────────────────
|
||||
|
||||
// SaveBrowsePage stores a SingleFile HTML snapshot for the given cache key.
|
||||
SaveBrowsePage(ctx context.Context, key, html string) error
|
||||
// GetBrowsePage retrieves a cached HTML snapshot. Returns ("", false, nil)
|
||||
// when no snapshot exists for the key.
|
||||
GetBrowsePage(ctx context.Context, key string) (string, bool, error)
|
||||
// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
|
||||
// Layout: {domain}/html/page-{n}.html
|
||||
BrowseHTMLKey(domain string, page int) string
|
||||
// BrowseFilteredHTMLKey returns the MinIO object key for a browse page snapshot
|
||||
// that incorporates sort/genre/status so different filter combos are cached separately.
|
||||
BrowseFilteredHTMLKey(domain string, page int, sort, genre, status string) string
|
||||
// BrowseCoverKey returns the MinIO object key for a cached book cover image.
|
||||
// Layout: {domain}/assets/book-covers/{slug}.jpg
|
||||
BrowseCoverKey(domain, slug string) string
|
||||
// SaveBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
|
||||
SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error
|
||||
// GetBrowseAsset retrieves a binary asset from the browse bucket.
|
||||
// Returns (nil, "", false, nil) when the object does not exist.
|
||||
GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error)
|
||||
|
||||
// ── Scraping tasks ─────────────────────────────────────────────────────
|
||||
|
||||
// CreateScrapeTask inserts a new scraping_tasks record with status="running"
|
||||
// and returns the assigned ID.
|
||||
CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error)
|
||||
// UpdateScrapeTask patches an existing task record.
|
||||
UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error
|
||||
// ListScrapeTasks returns all tasks sorted by started descending.
|
||||
ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error)
|
||||
|
||||
// ── Audio jobs ─────────────────────────────────────────────────────────
|
||||
|
||||
// CreateAudioJob inserts a new audio_jobs record with status="pending"
|
||||
// and returns the assigned ID.
|
||||
CreateAudioJob(ctx context.Context, slug string, chapter int, voice string) (string, error)
|
||||
// UpdateAudioJob patches an existing audio job record (status, error, finished).
|
||||
UpdateAudioJob(ctx context.Context, id, status, errMsg string, finished time.Time) error
|
||||
// GetAudioJob returns the most recent audio job for the given cache key,
|
||||
// or (zero, false, nil) if none exists.
|
||||
GetAudioJob(ctx context.Context, cacheKey string) (AudioJob, bool, error)
|
||||
// ListAudioJobs returns all audio jobs sorted by started descending.
|
||||
ListAudioJobs(ctx context.Context) ([]AudioJob, error)
|
||||
}
|
||||
@@ -1,476 +0,0 @@
|
||||
// Package writer handles persistence of scraped chapters and metadata.
|
||||
//
|
||||
// Directory layout:
|
||||
//
|
||||
// static/books/
|
||||
// ├── {book-slug}/
|
||||
// │ ├── metadata.yaml
|
||||
// │ ├── vol-0/ (no volume grouping)
|
||||
// │ │ ├── 1-50/
|
||||
// │ │ │ ├── chapter-1.md
|
||||
// │ │ │ └── …
|
||||
// │ │ └── 51-100/
|
||||
// │ │ └── …
|
||||
// │ └── vol-1/
|
||||
// │ └── …
|
||||
package writer
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const chaptersPerFolder = 50
|
||||
|
||||
// Writer persists scraped content under a configurable root directory.
|
||||
type Writer struct {
|
||||
root string // e.g. "./static/books"
|
||||
}
|
||||
|
||||
// New creates a Writer that stores files under root.
|
||||
func New(root string) *Writer {
|
||||
return &Writer{root: root}
|
||||
}
|
||||
|
||||
// ─── Metadata ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// WriteMetadata serialises meta to static/books/{slug}/metadata.yaml.
|
||||
// It creates the directory if it does not exist and overwrites any existing file.
|
||||
func (w *Writer) WriteMetadata(meta scraper.BookMeta) error {
|
||||
dir := w.bookDir(meta.Slug)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("writer: mkdir %s: %w", dir, err)
|
||||
}
|
||||
|
||||
path := filepath.Join(dir, "metadata.yaml")
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writer: create metadata %s: %w", path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
enc := yaml.NewEncoder(f)
|
||||
enc.SetIndent(2)
|
||||
if err := enc.Encode(meta); err != nil {
|
||||
return fmt.Errorf("writer: encode metadata: %w", err)
|
||||
}
|
||||
return enc.Close()
|
||||
}
|
||||
|
||||
// ReadMetadata reads the metadata.yaml for slug if it exists.
|
||||
// Returns (zero-value, false, nil) when the file does not exist.
|
||||
func (w *Writer) ReadMetadata(slug string) (scraper.BookMeta, bool, error) {
|
||||
path := filepath.Join(w.bookDir(slug), "metadata.yaml")
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return scraper.BookMeta{}, false, nil
|
||||
}
|
||||
return scraper.BookMeta{}, false, fmt.Errorf("writer: read metadata %s: %w", path, err)
|
||||
}
|
||||
|
||||
var meta scraper.BookMeta
|
||||
if err := yaml.Unmarshal(data, &meta); err != nil {
|
||||
return scraper.BookMeta{}, true, fmt.Errorf("writer: unmarshal metadata %s: %w", path, err)
|
||||
}
|
||||
return meta, true, nil
|
||||
}
|
||||
|
||||
// MetadataMtime returns the modification time (Unix seconds) of the
|
||||
// metadata.yaml file for slug, or 0 if the file cannot be stat'd.
|
||||
func (w *Writer) MetadataMtime(slug string) int64 {
|
||||
path := filepath.Join(w.bookDir(slug), "metadata.yaml")
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return fi.ModTime().Unix()
|
||||
}
|
||||
|
||||
// ─── Chapters ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// ChapterExists returns true if the markdown file for ref already exists on disk.
|
||||
func (w *Writer) ChapterExists(slug string, ref scraper.ChapterRef) bool {
|
||||
_, err := os.Stat(w.chapterPath(slug, ref))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// WriteChapter writes chapter.Text to the appropriate markdown file.
|
||||
// The parent directories are created on demand.
|
||||
func (w *Writer) WriteChapter(slug string, chapter scraper.Chapter) error {
|
||||
path := w.chapterPath(slug, chapter.Ref)
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("writer: mkdir %s: %w", dir, err)
|
||||
}
|
||||
|
||||
// Build the markdown document.
|
||||
var sb strings.Builder
|
||||
sb.WriteString("# ")
|
||||
sb.WriteString(chapter.Ref.Title)
|
||||
sb.WriteString("\n\n")
|
||||
sb.WriteString(chapter.Text)
|
||||
sb.WriteString("\n")
|
||||
|
||||
if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil {
|
||||
return fmt.Errorf("writer: write chapter %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── Catalogue helpers ────────────────────────────────────────────────────────
|
||||
|
||||
// ListBooks returns metadata for every book that has a metadata.yaml under root.
|
||||
// Books with unreadable metadata files are silently skipped.
|
||||
func (w *Writer) ListBooks() ([]scraper.BookMeta, error) {
|
||||
entries, err := os.ReadDir(w.root)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("writer: list books: %w", err)
|
||||
}
|
||||
var books []scraper.BookMeta
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
meta, ok, _ := w.ReadMetadata(e.Name())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
books = append(books, meta)
|
||||
}
|
||||
sort.Slice(books, func(i, j int) bool {
|
||||
return books[i].Title < books[j].Title
|
||||
})
|
||||
return books, nil
|
||||
}
|
||||
|
||||
// LocalSlugs returns the set of book slugs that have a metadata.yaml on disk.
|
||||
// It is cheaper than ListBooks because it only checks for file existence rather
|
||||
// than fully parsing every YAML file.
|
||||
func (w *Writer) LocalSlugs() map[string]bool {
|
||||
entries, err := os.ReadDir(w.root)
|
||||
if err != nil {
|
||||
return map[string]bool{}
|
||||
}
|
||||
slugs := make(map[string]bool, len(entries))
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
metaPath := filepath.Join(w.root, e.Name(), "metadata.yaml")
|
||||
if _, err := os.Stat(metaPath); err == nil {
|
||||
slugs[e.Name()] = true
|
||||
}
|
||||
}
|
||||
return slugs
|
||||
}
|
||||
|
||||
// ChapterInfo is a lightweight chapter descriptor derived from on-disk files.
|
||||
type ChapterInfo struct {
|
||||
Number int
|
||||
Title string // chapter name, cleaned of number prefix and trailing date
|
||||
Date string // relative date scraped alongside the title, e.g. "1 year ago"
|
||||
}
|
||||
|
||||
// ListChapters returns all chapters on disk for slug, sorted by number.
|
||||
func (w *Writer) ListChapters(slug string) ([]ChapterInfo, error) {
|
||||
bookDir := w.bookDir(slug)
|
||||
var chapters []ChapterInfo
|
||||
|
||||
// Walk vol-*/range-*/ directories.
|
||||
volDirs, err := filepath.Glob(filepath.Join(bookDir, "vol-*"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("writer: list chapters glob: %w", err)
|
||||
}
|
||||
for _, vd := range volDirs {
|
||||
rangeDirs, _ := filepath.Glob(filepath.Join(vd, "*-*"))
|
||||
for _, rd := range rangeDirs {
|
||||
files, _ := filepath.Glob(filepath.Join(rd, "chapter-*.md"))
|
||||
for _, f := range files {
|
||||
base := filepath.Base(f) // chapter-N.md
|
||||
numStr := strings.TrimSuffix(strings.TrimPrefix(base, "chapter-"), ".md")
|
||||
n, err := strconv.Atoi(numStr)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
title, date := chapterTitle(f, n)
|
||||
chapters = append(chapters, ChapterInfo{Number: n, Title: title, Date: date})
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(chapters, func(i, j int) bool {
|
||||
return chapters[i].Number < chapters[j].Number
|
||||
})
|
||||
return chapters, nil
|
||||
}
|
||||
|
||||
// CountChapters returns the number of chapter markdown files on disk for slug.
|
||||
// It is cheaper than ListChapters because it does not read file contents.
|
||||
func (w *Writer) CountChapters(slug string) int {
|
||||
bookDir := w.bookDir(slug)
|
||||
volDirs, err := filepath.Glob(filepath.Join(bookDir, "vol-*"))
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
count := 0
|
||||
for _, vd := range volDirs {
|
||||
rangeDirs, _ := filepath.Glob(filepath.Join(vd, "*-*"))
|
||||
for _, rd := range rangeDirs {
|
||||
files, _ := filepath.Glob(filepath.Join(rd, "chapter-*.md"))
|
||||
count += len(files)
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// chapterTitle reads the first non-empty line of a markdown file and strips
|
||||
// the leading "# " heading marker. Falls back to "Chapter N".
|
||||
func chapterTitle(path string, n int) (title, date string) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("Chapter %d", n), ""
|
||||
}
|
||||
for _, line := range strings.SplitN(string(data), "\n", 10) {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
line = strings.TrimPrefix(line, "# ")
|
||||
return SplitChapterTitle(line)
|
||||
}
|
||||
return fmt.Sprintf("Chapter %d", n), ""
|
||||
}
|
||||
|
||||
// SplitChapterTitle separates the human-readable chapter name from the
|
||||
// trailing relative-date string that novelfire.net appends to the heading.
|
||||
// Examples of raw heading text (after stripping "# "):
|
||||
//
|
||||
// "1 Chapter 1 - 1: The Academy's Weakest1 year ago"
|
||||
// "2 Chapter 2 - Enter the Storm3 months ago"
|
||||
//
|
||||
// The pattern is: optional leading number+whitespace, then the real title,
|
||||
// then a date that matches /\d+\s+(second|minute|hour|day|week|month|year)s?\s+ago$/
|
||||
func SplitChapterTitle(raw string) (title, date string) {
|
||||
// Strip a leading chapter-number index that novelfire sometimes prepends.
|
||||
// It looks like "1 " or "12 " at the very start.
|
||||
raw = strings.TrimSpace(raw)
|
||||
if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
|
||||
prefix := raw[:idx]
|
||||
allDigit := true
|
||||
for _, c := range prefix {
|
||||
if c < '0' || c > '9' {
|
||||
allDigit = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allDigit {
|
||||
raw = strings.TrimSpace(raw[idx:])
|
||||
}
|
||||
}
|
||||
|
||||
// Strip "Chapter N - N: " prefix (novelfire double-number format).
|
||||
// Also handles "Chapter N: " (single number) and "Chapter N - Title" without colon.
|
||||
chNumRe := regexp.MustCompile(`(?i)^chapter\s+\d+(?:\s*-\s*\d+)?\s*:\s*`)
|
||||
raw = strings.TrimSpace(chNumRe.ReplaceAllString(raw, ""))
|
||||
|
||||
// Match a trailing relative date: "<n> <unit>[s] ago"
|
||||
dateRe := regexp.MustCompile(`\s*(\d+\s+(?:second|minute|hour|day|week|month|year)s?\s+ago)\s*$`)
|
||||
if m := dateRe.FindStringSubmatchIndex(raw); m != nil {
|
||||
return strings.TrimSpace(raw[:m[0]]), strings.TrimSpace(raw[m[2]:m[3]])
|
||||
}
|
||||
return raw, ""
|
||||
}
|
||||
|
||||
// ReadChapter returns the raw markdown content for chapter number n of slug.
|
||||
func (w *Writer) ReadChapter(slug string, n int) (string, error) {
|
||||
// Reconstruct path using the same bucketing formula as chapterPath.
|
||||
ref := scraper.ChapterRef{Number: n, Volume: 0}
|
||||
path := w.chapterPath(slug, ref)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("writer: read chapter %d: %w", n, err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
// ─── Ranking ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// RankingItem represents a single entry in the ranking.
|
||||
type RankingItem struct {
|
||||
Rank int `yaml:"rank" json:"rank"`
|
||||
Slug string `yaml:"slug" json:"slug"`
|
||||
Title string `yaml:"title" json:"title"`
|
||||
Author string `yaml:"author,omitempty" json:"author,omitempty"`
|
||||
Cover string `yaml:"cover,omitempty" json:"cover,omitempty"`
|
||||
Status string `yaml:"status,omitempty" json:"status,omitempty"`
|
||||
Genres []string `yaml:"genres,omitempty" json:"genres,omitempty"`
|
||||
SourceURL string `yaml:"source_url,omitempty" json:"source_url,omitempty"`
|
||||
}
|
||||
|
||||
// WriteRanking saves the ranking items as JSON to static/books/ranking.json.
|
||||
// This replaces the old markdown table format with a structured format that
|
||||
// is faster to read back (no custom parsing) and safe for titles containing "|".
|
||||
func (w *Writer) WriteRanking(items []RankingItem) error {
|
||||
path := filepath.Clean(w.rankingPath())
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("writer: mkdir %s: %w", dir, err)
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(items, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("writer: marshal ranking: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
return fmt.Errorf("writer: write ranking %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadRankingItems parses ranking.json into a slice of RankingItem.
|
||||
// Returns nil slice (not an error) when the file does not exist yet.
|
||||
func (w *Writer) ReadRankingItems() ([]RankingItem, error) {
|
||||
data, err := os.ReadFile(w.rankingPath())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("writer: read ranking: %w", err)
|
||||
}
|
||||
var items []RankingItem
|
||||
if err := json.Unmarshal(data, &items); err != nil {
|
||||
return nil, fmt.Errorf("writer: parse ranking json: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// RankingFileInfo returns os.FileInfo for the ranking.json file, if it exists.
|
||||
func (w *Writer) RankingFileInfo() (os.FileInfo, error) {
|
||||
return os.Stat(w.rankingPath())
|
||||
}
|
||||
|
||||
func (w *Writer) rankingPath() string {
|
||||
return filepath.Join(w.root, "ranking.json")
|
||||
}
|
||||
|
||||
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
|
||||
|
||||
// rankingCacheDir returns the directory that stores per-page HTML caches.
|
||||
func (w *Writer) rankingCacheDir() string {
|
||||
return filepath.Join(w.root, "_ranking_cache")
|
||||
}
|
||||
|
||||
// rankingPageCachePath returns the path for a cached ranking page HTML file.
|
||||
func (w *Writer) rankingPageCachePath(page int) string {
|
||||
return filepath.Join(w.rankingCacheDir(), fmt.Sprintf("page-%d.html", page))
|
||||
}
|
||||
|
||||
// WriteRankingPageCache persists raw HTML for the given ranking page number.
|
||||
func (w *Writer) WriteRankingPageCache(page int, html string) error {
|
||||
dir := w.rankingCacheDir()
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("writer: mkdir ranking cache %s: %w", dir, err)
|
||||
}
|
||||
path := w.rankingPageCachePath(page)
|
||||
if err := os.WriteFile(path, []byte(html), 0o644); err != nil {
|
||||
return fmt.Errorf("writer: write ranking page cache %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadRankingPageCache reads the cached HTML for the given ranking page.
|
||||
// Returns ("", nil) when no cache file exists yet.
|
||||
func (w *Writer) ReadRankingPageCache(page int) (string, error) {
|
||||
data, err := os.ReadFile(w.rankingPageCachePath(page))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return "", nil
|
||||
}
|
||||
return "", fmt.Errorf("writer: read ranking page cache page %d: %w", page, err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
// RankingPageCacheInfo returns os.FileInfo for a cached ranking page file.
|
||||
// Returns (nil, nil) when the file does not exist.
|
||||
func (w *Writer) RankingPageCacheInfo(page int) (os.FileInfo, error) {
|
||||
info, err := os.Stat(w.rankingPageCachePath(page))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// bookDir returns the root directory for a book slug.
|
||||
func (w *Writer) bookDir(slug string) string {
|
||||
return filepath.Join(w.root, slug)
|
||||
}
|
||||
|
||||
// AudioDir returns the directory used to cache generated MP3 files for a book.
|
||||
func (w *Writer) AudioDir(slug string) string {
|
||||
return filepath.Join(w.bookDir(slug), "audio")
|
||||
}
|
||||
|
||||
// AudioPath returns the full path for a cached chapter audio file.
|
||||
// The filename is keyed by chapter number, voice, and speed so that different
|
||||
// settings never collide. Speed is formatted to one decimal place (e.g. "1.0").
|
||||
func (w *Writer) AudioPath(slug string, n int, voice string, speed float64) string {
|
||||
safeVoice := sanitiseVoice(voice)
|
||||
filename := fmt.Sprintf("ch%d-%s-%.1f.mp3", n, safeVoice, speed)
|
||||
return filepath.Join(w.AudioDir(slug), filename)
|
||||
}
|
||||
|
||||
// AudioPartPath returns the path for an individual audio chunk generated during
|
||||
// chunked TTS. Part files are named ch{n}-{voice}-{speed}.part{p}.mp3 and are
|
||||
// deleted after they have been merged into the final AudioPath file.
|
||||
func (w *Writer) AudioPartPath(slug string, n int, voice string, speed float64, part int) string {
|
||||
safeVoice := sanitiseVoice(voice)
|
||||
filename := fmt.Sprintf("ch%d-%s-%.1f.part%d.mp3", n, safeVoice, speed, part)
|
||||
return filepath.Join(w.AudioDir(slug), filename)
|
||||
}
|
||||
|
||||
// sanitiseVoice converts a voice name into a string that is safe to embed in a
|
||||
// filename (only a-z, A-Z, 0-9, '_', '-' are kept; everything else becomes '_').
|
||||
func sanitiseVoice(voice string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '_'
|
||||
}, voice)
|
||||
}
|
||||
|
||||
// chapterPath computes the full file path for a chapter.
|
||||
//
|
||||
// vol-{volume}/{folderRange}/chapter-{number}.md
|
||||
//
|
||||
// Example: vol-0/1-50/chapter-1.md, vol-0/51-100/chapter-51.md
|
||||
func (w *Writer) chapterPath(slug string, ref scraper.ChapterRef) string {
|
||||
vol := ref.Volume // 0 == no volume grouping
|
||||
volDir := fmt.Sprintf("vol-%d", vol)
|
||||
|
||||
// Folder group: chapters 1-50 → "1-50", 51-100 → "51-100", …
|
||||
lo := ((ref.Number-1)/chaptersPerFolder)*chaptersPerFolder + 1
|
||||
hi := lo + chaptersPerFolder - 1
|
||||
rangeDir := fmt.Sprintf("%d-%d", lo, hi)
|
||||
|
||||
filename := fmt.Sprintf("chapter-%d.md", ref.Number)
|
||||
|
||||
return filepath.Join(w.bookDir(slug), volDir, rangeDir, filename)
|
||||
}
|
||||
BIN
scraper/scraper
BIN
scraper/scraper
Binary file not shown.
5
scraper/tools.go
Normal file
5
scraper/tools.go
Normal file
@@ -0,0 +1,5 @@
|
||||
//go:build tools
|
||||
|
||||
package tools
|
||||
|
||||
import _ "honnef.co/go/tools/cmd/staticcheck"
|
||||
13
scripts/.runner
Normal file
13
scripts/.runner
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"WARNING": "This file is automatically generated by act-runner. Do not edit it manually unless you know what you are doing. Removing this file will cause act runner to re-register as a new runner.",
|
||||
"id": 11,
|
||||
"uuid": "d5d04e0a-572c-46c0-83be-405508948391",
|
||||
"name": "runner-mac-1",
|
||||
"token": "ddf214ce148b4673a186f29cb684b407cb8c2ecc",
|
||||
"address": "https://gitea.kalekber.cc/",
|
||||
"labels": [
|
||||
"macos-latest:host",
|
||||
"macos-14:host"
|
||||
],
|
||||
"ephemeral": false
|
||||
}
|
||||
99
scripts/link-tooltip.user.js
Normal file
99
scripts/link-tooltip.user.js
Normal file
@@ -0,0 +1,99 @@
|
||||
// ==UserScript==
|
||||
// @name Link URL Tooltip
|
||||
// @namespace https://github.com/kalekber/libnovel-v2
|
||||
// @version 1.0.0
|
||||
// @description Show the destination URL near the cursor when hovering over any link
|
||||
// @author kalekber
|
||||
// @match *://*/*
|
||||
// @run-at document-idle
|
||||
// @grant none
|
||||
// ==/UserScript==
|
||||
|
||||
(function () {
|
||||
'use strict';
|
||||
|
||||
// --- Inject styles ---
|
||||
const style = document.createElement('style');
|
||||
style.textContent = `
|
||||
#lnk-tooltip {
|
||||
position: fixed;
|
||||
display: none;
|
||||
background-color: #333;
|
||||
color: #fff;
|
||||
padding: 5px 10px;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
font-family: monospace;
|
||||
pointer-events: none;
|
||||
z-index: 2147483647;
|
||||
white-space: nowrap;
|
||||
max-width: 600px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
box-shadow: 0 2px 6px rgba(0,0,0,0.4);
|
||||
}
|
||||
`;
|
||||
document.head.appendChild(style);
|
||||
|
||||
// --- Inject tooltip element ---
|
||||
const tooltip = document.createElement('div');
|
||||
tooltip.id = 'lnk-tooltip';
|
||||
document.body.appendChild(tooltip);
|
||||
|
||||
// --- Helpers ---
|
||||
function getAnchor(target) {
|
||||
// Walk up the DOM to find the nearest <a href="...">
|
||||
// (handles clicks on nested elements like <a><span>text</span></a>)
|
||||
return target.closest('a[href]');
|
||||
}
|
||||
|
||||
function show(anchor, clientX, clientY) {
|
||||
tooltip.textContent = anchor.href;
|
||||
tooltip.style.display = 'block';
|
||||
position(clientX, clientY);
|
||||
}
|
||||
|
||||
function hide() {
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
|
||||
function position(clientX, clientY) {
|
||||
const offset = 12;
|
||||
const tw = tooltip.offsetWidth;
|
||||
const th = tooltip.offsetHeight;
|
||||
const vw = window.innerWidth;
|
||||
const vh = window.innerHeight;
|
||||
|
||||
let x = clientX + offset;
|
||||
let y = clientY + offset;
|
||||
|
||||
// Flip horizontally if it would overflow the right edge
|
||||
if (x + tw > vw - 4) {
|
||||
x = clientX - tw - offset;
|
||||
}
|
||||
// Flip vertically if it would overflow the bottom edge
|
||||
if (y + th > vh - 4) {
|
||||
y = clientY - th - offset;
|
||||
}
|
||||
|
||||
tooltip.style.left = Math.max(0, x) + 'px';
|
||||
tooltip.style.top = Math.max(0, y) + 'px';
|
||||
}
|
||||
|
||||
// --- Event delegation on document ---
|
||||
document.addEventListener('mouseover', (e) => {
|
||||
const anchor = getAnchor(e.target);
|
||||
if (anchor) show(anchor, e.clientX, e.clientY);
|
||||
});
|
||||
|
||||
document.addEventListener('mousemove', (e) => {
|
||||
if (tooltip.style.display === 'block') {
|
||||
position(e.clientX, e.clientY);
|
||||
}
|
||||
});
|
||||
|
||||
document.addEventListener('mouseout', (e) => {
|
||||
const anchor = getAnchor(e.target);
|
||||
if (anchor) hide();
|
||||
});
|
||||
})();
|
||||
231
scripts/pb-init.sh
Executable file
231
scripts/pb-init.sh
Executable file
@@ -0,0 +1,231 @@
|
||||
#!/bin/sh
|
||||
# pb-init.sh — idempotent PocketBase collection bootstrap
|
||||
#
|
||||
# Creates all collections required by libnovel. Safe to re-run: POST returns
|
||||
# 400/422 when a collection already exists; both are treated as success.
|
||||
#
|
||||
# Required env vars (with defaults):
|
||||
# POCKETBASE_URL http://pocketbase:8090
|
||||
# POCKETBASE_ADMIN_EMAIL admin@libnovel.local
|
||||
# POCKETBASE_ADMIN_PASSWORD changeme123
|
||||
|
||||
set -e
|
||||
|
||||
PB_URL="${POCKETBASE_URL:-http://pocketbase:8090}"
|
||||
PB_EMAIL="${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
PB_PASSWORD="${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
|
||||
log() { echo "[pb-init] $*"; }
|
||||
|
||||
# ─── 1. Wait for PocketBase to be ready ──────────────────────────────────────
|
||||
log "waiting for PocketBase at $PB_URL ..."
|
||||
until wget -qO- "$PB_URL/api/health" > /dev/null 2>&1; do
|
||||
sleep 2
|
||||
done
|
||||
log "PocketBase is up"
|
||||
|
||||
# ─── 2. Authenticate and obtain a superuser token ────────────────────────────
|
||||
log "authenticating as $PB_EMAIL ..."
|
||||
AUTH_RESPONSE=$(wget -qO- \
|
||||
--header="Content-Type: application/json" \
|
||||
--post-data="{\"identity\":\"$PB_EMAIL\",\"password\":\"$PB_PASSWORD\"}" \
|
||||
"$PB_URL/api/collections/_superusers/auth-with-password")
|
||||
|
||||
TOKEN=$(echo "$AUTH_RESPONSE" | sed 's/.*"token":"\([^"]*\)".*/\1/')
|
||||
if [ -z "$TOKEN" ] || [ "$TOKEN" = "$AUTH_RESPONSE" ]; then
|
||||
log "ERROR: failed to obtain auth token. Response: $AUTH_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
log "auth token obtained"
|
||||
|
||||
# ─── 3. Helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
create_collection() {
|
||||
NAME="$1"
|
||||
BODY="$2"
|
||||
STATUS=$(wget -qSO- \
|
||||
--header="Content-Type: application/json" \
|
||||
--header="Authorization: Bearer $TOKEN" \
|
||||
--post-data="$BODY" \
|
||||
"$PB_URL/api/collections" 2>&1 | grep "^ HTTP/" | awk '{print $2}')
|
||||
case "$STATUS" in
|
||||
200|201) log "created collection: $NAME" ;;
|
||||
400|422) log "collection already exists (skipped): $NAME" ;;
|
||||
*) log "WARNING: unexpected status $STATUS for collection: $NAME" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ensure_field COLLECTION FIELD_NAME FIELD_TYPE
|
||||
#
|
||||
# Checks whether FIELD_NAME exists in COLLECTION's schema. If it is missing,
|
||||
# sends a PATCH with the full current fields list plus the new field appended.
|
||||
# Uses only busybox sh + wget + sed/awk — no python/jq required.
|
||||
ensure_field() {
|
||||
COLL="$1"
|
||||
FIELD_NAME="$2"
|
||||
FIELD_TYPE="$3"
|
||||
|
||||
SCHEMA=$(wget -qO- \
|
||||
--header="Authorization: Bearer $TOKEN" \
|
||||
"$PB_URL/api/collections/$COLL" 2>/dev/null)
|
||||
|
||||
# Check if the field already exists (look for "name":"<FIELD_NAME>" in the fields array)
|
||||
if echo "$SCHEMA" | grep -q "\"name\":\"$FIELD_NAME\""; then
|
||||
log "field $COLL.$FIELD_NAME already exists — skipping"
|
||||
return
|
||||
fi
|
||||
|
||||
COLLECTION_ID=$(echo "$SCHEMA" | sed 's/.*"id":"\([^"]*\)".*/\1/')
|
||||
if [ -z "$COLLECTION_ID" ] || [ "$COLLECTION_ID" = "$SCHEMA" ]; then
|
||||
log "WARNING: could not get id for collection $COLL — skipping ensure_field"
|
||||
return
|
||||
fi
|
||||
|
||||
# Extract current fields array (everything between the outermost [ ] of "fields":[...])
|
||||
# and append the new field object before the closing bracket.
|
||||
CURRENT_FIELDS=$(echo "$SCHEMA" | sed 's/.*"fields":\(\[.*\]\).*/\1/')
|
||||
# Strip the trailing ] and append the new field
|
||||
TRIMMED=$(echo "$CURRENT_FIELDS" | sed 's/]$//')
|
||||
NEW_FIELDS="${TRIMMED},{\"name\":\"${FIELD_NAME}\",\"type\":\"${FIELD_TYPE}\"}]"
|
||||
PATCH_BODY="{\"fields\":${NEW_FIELDS}}"
|
||||
|
||||
STATUS=$(wget -qSO- \
|
||||
--header="Content-Type: application/json" \
|
||||
--header="Authorization: Bearer $TOKEN" \
|
||||
--body-data="$PATCH_BODY" \
|
||||
--method=PATCH \
|
||||
"$PB_URL/api/collections/$COLLECTION_ID" 2>&1 | grep "^ HTTP/" | awk '{print $2}')
|
||||
case "$STATUS" in
|
||||
200|201) log "patched $COLL — added field: $FIELD_NAME ($FIELD_TYPE)" ;;
|
||||
*) log "WARNING: patch returned $STATUS when adding $FIELD_NAME to $COLL" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ─── 4. Create collections (idempotent — skips if already exist) ─────────────
|
||||
|
||||
create_collection "books" '{
|
||||
"name": "books",
|
||||
"type": "base",
|
||||
"fields": [
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "title", "type": "text", "required": true},
|
||||
{"name": "author", "type": "text"},
|
||||
{"name": "cover", "type": "text"},
|
||||
{"name": "status", "type": "text"},
|
||||
{"name": "genres", "type": "json"},
|
||||
{"name": "summary", "type": "text"},
|
||||
{"name": "total_chapters", "type": "number"},
|
||||
{"name": "source_url", "type": "text"},
|
||||
{"name": "ranking", "type": "number"},
|
||||
{"name": "meta_updated", "type": "date"}
|
||||
]
|
||||
}'
|
||||
|
||||
create_collection "chapters_idx" '{
|
||||
"name": "chapters_idx",
|
||||
"type": "base",
|
||||
"fields": [
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "number", "type": "number", "required": true},
|
||||
{"name": "title", "type": "text"},
|
||||
{"name": "date_label", "type": "text"}
|
||||
]
|
||||
}'
|
||||
|
||||
create_collection "ranking" '{
|
||||
"name": "ranking",
|
||||
"type": "base",
|
||||
"fields": [
|
||||
{"name": "rank", "type": "number", "required": true},
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "title", "type": "text"},
|
||||
{"name": "author", "type": "text"},
|
||||
{"name": "cover", "type": "text"},
|
||||
{"name": "status", "type": "text"},
|
||||
{"name": "genres", "type": "json"},
|
||||
{"name": "source_url", "type": "text"},
|
||||
{"name": "updated", "type": "date"}
|
||||
]
|
||||
}'
|
||||
|
||||
create_collection "progress" '{
|
||||
"name": "progress",
|
||||
"type": "base",
|
||||
"fields": [
|
||||
{"name": "session_id", "type": "text", "required": true},
|
||||
{"name": "user_id", "type": "text"},
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "chapter", "type": "number"},
|
||||
{"name": "updated", "type": "date"}
|
||||
]
|
||||
}'
|
||||
|
||||
create_collection "audio_cache" '{
|
||||
"name": "audio_cache",
|
||||
"type": "base",
|
||||
"fields": [
|
||||
{"name": "cache_key", "type": "text", "required": true},
|
||||
{"name": "filename", "type": "text"},
|
||||
{"name": "updated", "type": "date"}
|
||||
]
|
||||
}'
|
||||
|
||||
create_collection "app_users" '{
|
||||
"name": "app_users",
|
||||
"type": "base",
|
||||
"fields": [
|
||||
{"name": "username", "type": "text", "required": true},
|
||||
{"name": "password_hash", "type": "text", "required": true},
|
||||
{"name": "role", "type": "text"},
|
||||
{"name": "created", "type": "date"},
|
||||
{"name": "avatar_url", "type": "text"}
|
||||
]
|
||||
}'
|
||||
|
||||
create_collection "user_settings" '{
|
||||
"name": "user_settings",
|
||||
"type": "base",
|
||||
"fields": [
|
||||
{"name": "session_id", "type": "text", "required": true},
|
||||
{"name": "user_id", "type": "text"},
|
||||
{"name": "auto_next", "type": "bool"},
|
||||
{"name": "voice", "type": "text"},
|
||||
{"name": "speed", "type": "number"},
|
||||
{"name": "updated", "type": "date"}
|
||||
]
|
||||
}'
|
||||
|
||||
# ─── 5. Schema migrations (idempotent field additions) ───────────────────────
|
||||
# Ensures fields added after initial deploy are present in existing instances.
|
||||
|
||||
ensure_field "progress" "user_id" "text"
|
||||
ensure_field "progress" "audio_time" "number"
|
||||
ensure_field "user_settings" "user_id" "text"
|
||||
ensure_field "app_users" "avatar_url" "text"
|
||||
|
||||
create_collection "book_comments" '{
|
||||
"name": "book_comments",
|
||||
"type": "base",
|
||||
"fields": [
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "user_id", "type": "text"},
|
||||
{"name": "username", "type": "text"},
|
||||
{"name": "body", "type": "text", "required": true},
|
||||
{"name": "upvotes", "type": "number"},
|
||||
{"name": "downvotes", "type": "number"},
|
||||
{"name": "created", "type": "date"}
|
||||
]
|
||||
}'
|
||||
|
||||
create_collection "comment_votes" '{
|
||||
"name": "comment_votes",
|
||||
"type": "base",
|
||||
"fields": [
|
||||
{"name": "comment_id", "type": "text", "required": true},
|
||||
{"name": "user_id", "type": "text"},
|
||||
{"name": "session_id", "type": "text", "required": true},
|
||||
{"name": "vote", "type": "text", "required": true}
|
||||
]
|
||||
}'
|
||||
|
||||
log "all collections ready"
|
||||
39
scripts/runner-config-mac.yaml
Normal file
39
scripts/runner-config-mac.yaml
Normal file
@@ -0,0 +1,39 @@
|
||||
log:
|
||||
level: info
|
||||
|
||||
runner:
|
||||
file: .runner
|
||||
capacity: 1
|
||||
envs: {}
|
||||
env_file: .env
|
||||
timeout: 3h
|
||||
shutdown_timeout: 0s
|
||||
insecure: false
|
||||
fetch_timeout: 5s
|
||||
fetch_interval: 2s
|
||||
github_mirror: ''
|
||||
labels:
|
||||
- "macos-latest:host"
|
||||
- "macos-14:host"
|
||||
|
||||
cache:
|
||||
enabled: true
|
||||
dir: ""
|
||||
host: "__HOST_IP__"
|
||||
port: 8088
|
||||
external_server: ""
|
||||
|
||||
container:
|
||||
network: ""
|
||||
privileged: false
|
||||
options: ""
|
||||
workdir_parent: ""
|
||||
valid_volumes: []
|
||||
docker_host: ""
|
||||
force_pull: false
|
||||
force_rebuild: false
|
||||
require_docker: false
|
||||
docker_timeout: 0s
|
||||
|
||||
host:
|
||||
workdir_parent: ""
|
||||
109
scripts/runner-config.yaml
Normal file
109
scripts/runner-config.yaml
Normal file
@@ -0,0 +1,109 @@
|
||||
# Example configuration file, it's safe to copy this as the default config file without any modification.
|
||||
|
||||
# You don't have to copy this file to your instance,
|
||||
# just run `./act_runner generate-config > config.yaml` to generate a config file.
|
||||
|
||||
log:
|
||||
# The level of logging, can be trace, debug, info, warn, error, fatal
|
||||
level: info
|
||||
|
||||
runner:
|
||||
# Where to store the registration result.
|
||||
file: .runner
|
||||
# Execute how many tasks concurrently at the same time.
|
||||
capacity: 1
|
||||
# Extra environment variables to run jobs.
|
||||
envs:
|
||||
# Extra environment variables to run jobs from a file.
|
||||
# It will be ignored if it's empty or the file doesn't exist.
|
||||
env_file: .env
|
||||
# The timeout for a job to be finished.
|
||||
# Please note that the Gitea instance also has a timeout (3h by default) for the job.
|
||||
# So the job could be stopped by the Gitea instance if its timeout is shorter than this.
|
||||
timeout: 3h
|
||||
# The timeout for the runner to wait for running jobs to finish when shutting down.
|
||||
# Any running jobs that haven't finished after this timeout will be cancelled.
|
||||
shutdown_timeout: 0s
|
||||
# Whether skip verifying the TLS certificate of the Gitea instance.
|
||||
insecure: false
|
||||
# The timeout for fetching the job from the Gitea instance.
|
||||
fetch_timeout: 5s
|
||||
# The interval for fetching the job from the Gitea instance.
|
||||
fetch_interval: 2s
|
||||
# The github_mirror of a runner is used to specify the mirror address of the github that pulls the action repository.
|
||||
# It works when something like `uses: actions/checkout@v4` is used and DEFAULT_ACTIONS_URL is set to github,
|
||||
# and github_mirror is not empty. In this case,
|
||||
# it replaces https://github.com with the value here, which is useful for some special network environments.
|
||||
github_mirror: ''
|
||||
# The labels of a runner are used to determine which jobs the runner can run, and how to run them.
|
||||
# Like: "macos-arm64:host" or "ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest"
|
||||
# Find more images provided by Gitea at https://gitea.com/gitea/runner-images .
|
||||
# If it's empty when registering, it will ask for inputting labels.
|
||||
# If it's empty when execute `daemon`, will use labels in `.runner` file.
|
||||
labels:
|
||||
- "ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest"
|
||||
- "ubuntu-24.04:docker://docker.gitea.com/runner-images:ubuntu-24.04"
|
||||
- "ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04"
|
||||
|
||||
cache:
|
||||
# Enable cache server to use actions/cache.
|
||||
enabled: true
|
||||
# The directory to store the cache data.
|
||||
# If it's empty, the cache data will be stored in $HOME/.cache/actcache.
|
||||
dir: ""
|
||||
# The host of the cache server.
|
||||
# It's not for the address to listen, but the address to connect from job containers.
|
||||
# So 0.0.0.0 is a bad choice, leave it empty to detect automatically.
|
||||
host: ""
|
||||
# The port of the cache server.
|
||||
# 0 means to use a random available port.
|
||||
port: 8088
|
||||
# The external cache server URL. Valid only when enable is true.
|
||||
# If it's specified, act_runner will use this URL as the ACTIONS_CACHE_URL rather than start a server by itself.
|
||||
# The URL should generally end with "/".
|
||||
external_server: ""
|
||||
|
||||
container:
|
||||
# Specifies the network to which the container will connect.
|
||||
# Could be host, bridge or the name of a custom network.
|
||||
# If it's empty, act_runner will create a network automatically.
|
||||
network: ""
|
||||
# Whether to use privileged mode or not when launching task containers (privileged mode is required for Docker-in-Docker).
|
||||
privileged: false
|
||||
# Any other options to be used when the container is started (e.g., --add-host=my.gitea.url:host-gateway).
|
||||
options:
|
||||
|
||||
# The parent directory of a job's working directory.
|
||||
# NOTE: There is no need to add the first '/' of the path as act_runner will add it automatically.
|
||||
# If the path starts with '/', the '/' will be trimmed.
|
||||
# For example, if the parent directory is /path/to/my/dir, workdir_parent should be path/to/my/dir
|
||||
# If it's empty, /workspace will be used.
|
||||
workdir_parent:
|
||||
# Volumes (including bind mounts) can be mounted to containers. Glob syntax is supported, see https://github.com/gobwas/glob
|
||||
# You can specify multiple volumes. If the sequence is empty, no volumes can be mounted.
|
||||
# For example, if you only allow containers to mount the `data` volume and all the json files in `/src`, youshould change the config to:
|
||||
# valid_volumes:
|
||||
# - data
|
||||
# - /src/*.json
|
||||
# If you want to allow any volume, please use the following configuration:
|
||||
# valid_volumes:
|
||||
# - '**'
|
||||
valid_volumes: []
|
||||
# Overrides the docker client host with the specified one.
|
||||
# If it's empty, act_runner will find an available docker host automatically.
|
||||
# If it's "-", act_runner will find an available docker host automatically, but the docker host won't be mounted to the job containers and service containers.
|
||||
# If it's not empty or "-", the specified docker host will be used. An error will be returned if it doesn't work.
|
||||
docker_host: ""
|
||||
# Pull docker image(s) even if already present
|
||||
force_pull: false
|
||||
# Rebuild docker image(s) even if already present
|
||||
force_rebuild: false
|
||||
# Always require a reachable docker daemon, even if not required by act_runner
|
||||
require_docker: false
|
||||
# Timeout to wait for the docker daemon to be reachable, if docker is required by require_docker or act_runner
|
||||
docker_timeout: 0s
|
||||
|
||||
host:
|
||||
# The parent directory of a job's working directory.
|
||||
# If it's empty, $HOME/.cache/act/ will be used.
|
||||
workdir_parent:
|
||||
76
scripts/setup_runner.sh
Executable file
76
scripts/setup_runner.sh
Executable file
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# ── usage ─────────────────────────────────────────────────────────────────────
|
||||
usage() {
|
||||
echo "Usage: $0 <runner-name>"
|
||||
echo " runner-name: runner-node-1 | runner-node-2 | runner-node-3"
|
||||
exit 1
|
||||
}
|
||||
|
||||
[[ $# -ne 1 ]] && usage
|
||||
|
||||
RUNNER_NAME="$1"
|
||||
|
||||
# validate
|
||||
case "$RUNNER_NAME" in
|
||||
runner-node-1|runner-node-2|runner-node-3) ;;
|
||||
*) echo "ERROR: unknown runner name '$RUNNER_NAME'"; usage ;;
|
||||
esac
|
||||
|
||||
# ── config ────────────────────────────────────────────────────────────────────
|
||||
CACHE_PORT=8088
|
||||
GITEA_URL="https://gitea.kalekber.cc/"
|
||||
REGISTRATION_TOKEN="AboxpDKWx7gizwJ9xeheHVqKjj9J9N9BgyX96wvu"
|
||||
IMAGE="docker.io/gitea/act_runner:latest"
|
||||
DATA_DIR="$PWD/data/$RUNNER_NAME"
|
||||
CFG_PATH="$DATA_DIR/config.yaml"
|
||||
|
||||
# ── detect THIS machine's LAN IP ──────────────────────────────────────────────
|
||||
HOST_IP=$(ip route get 1.1.1.1 | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1); exit}')
|
||||
if [[ -z "$HOST_IP" ]]; then
|
||||
echo "ERROR: could not detect host LAN IP" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Host LAN IP: $HOST_IP"
|
||||
|
||||
# ── generate config.yaml ──────────────────────────────────────────────────────
|
||||
mkdir -p "$DATA_DIR"
|
||||
|
||||
docker run --rm --entrypoint="" "$IMAGE" \
|
||||
act_runner generate-config > "$CFG_PATH"
|
||||
|
||||
awk -v host="$HOST_IP" -v port="$CACHE_PORT" '
|
||||
/^cache:/ { in_cache=1 }
|
||||
in_cache && /enabled:/ { $0 = " enabled: true" }
|
||||
in_cache && /dir:/ { $0 = " dir: \"/data/cache\"" }
|
||||
in_cache && /host:/ { $0 = " host: \"" host "\"" }
|
||||
in_cache && /port:/ { $0 = " port: " port; in_cache=0 }
|
||||
{ print }
|
||||
' "$CFG_PATH" > "${CFG_PATH}.tmp" && mv "${CFG_PATH}.tmp" "$CFG_PATH"
|
||||
|
||||
echo "Config written to $CFG_PATH (cache $HOST_IP:$CACHE_PORT)"
|
||||
|
||||
# ── stop + remove old container if exists ────────────────────────────────────
|
||||
if docker inspect "$RUNNER_NAME" &>/dev/null; then
|
||||
echo "Removing existing $RUNNER_NAME..."
|
||||
docker stop "$RUNNER_NAME" || true
|
||||
docker rm "$RUNNER_NAME" || true
|
||||
fi
|
||||
|
||||
# ── start runner ──────────────────────────────────────────────────────────────
|
||||
docker run \
|
||||
-v "$DATA_DIR:/data" \
|
||||
-v "$CFG_PATH:/config.yaml" \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-e CONFIG_FILE=/config.yaml \
|
||||
-e GITEA_INSTANCE_URL="$GITEA_URL" \
|
||||
-e GITEA_RUNNER_REGISTRATION_TOKEN="$REGISTRATION_TOKEN" \
|
||||
-e GITEA_RUNNER_NAME="$RUNNER_NAME" \
|
||||
-p "${CACHE_PORT}:${CACHE_PORT}" \
|
||||
--restart unless-stopped \
|
||||
--name "$RUNNER_NAME" \
|
||||
-d "$IMAGE"
|
||||
|
||||
echo "Runner $RUNNER_NAME started"
|
||||
docker ps --filter "name=$RUNNER_NAME" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user