fix(runner): add heartbeat + translation polling to asynq mode
All checks were successful
CI / UI (push) Successful in 57s
CI / Backend (pull_request) Successful in 1m4s
CI / Backend (push) Successful in 1m45s
CI / UI (pull_request) Successful in 51s

Two bugs prevented asynq mode from working correctly on the homelab runner:

1. No healthcheck file: asynq mode never writes /tmp/runner.alive, so
   Docker healthcheck always fails. Added heartbeat goroutine that
   writes the file every StaleTaskThreshold (30s).

2. Translation tasks not dispatched: translation uses ClaimNextTranslationTask
   (PocketBase poll queue), not Redis/asynq. Audio + scrape use asynq mux,
   but translation sits in PocketBase forever. Added pollTranslationTasks()
   goroutine that polls PocketBase on the same PollInterval as the old
   poll() loop.

All Go tests pass (go test ./... in backend/).
This commit is contained in:
Admin
2026-03-29 20:22:37 +05:00
parent 98631df47a
commit 5da880d189

View File

@@ -13,6 +13,8 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"os"
"sync"
"time" "time"
"github.com/hibiken/asynq" "github.com/hibiken/asynq"
@@ -72,6 +74,44 @@ func (r *Runner) runAsynq(ctx context.Context) error {
r.deps.Log.Info("runner: asynq mode active", "redis_addr", r.cfg.RedisAddr) r.deps.Log.Info("runner: asynq mode active", "redis_addr", r.cfg.RedisAddr)
// ── Heartbeat goroutine ──────────────────────────────────────────────
// Write /tmp/runner.alive every 30s so Docker healthcheck passes in asynq mode.
// This mirrors the heartbeat file behavior from the poll() loop.
go func() {
heartbeatTick := time.NewTicker(r.cfg.StaleTaskThreshold)
defer heartbeatTick.Stop()
for {
select {
case <-ctx.Done():
return
case <-heartbeatTick.C:
if f, err := os.Create("/tmp/runner.alive"); err != nil {
r.deps.Log.Warn("runner: could not write heartbeat file", "err", err)
} else {
f.Close()
}
}
}
}()
// ── Translation polling goroutine ────────────────────────────────────
// Translation tasks live in PocketBase (not Redis), so we need a separate
// poll loop to claim and dispatch them. This runs alongside the Asynq server.
translationSem := make(chan struct{}, r.cfg.MaxConcurrentTranslation)
var translationWg sync.WaitGroup
go func() {
tick := time.NewTicker(r.cfg.PollInterval)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
r.pollTranslationTasks(ctx, translationSem, &translationWg)
}
}
}()
// Run catalogue refresh ticker in the background. // Run catalogue refresh ticker in the background.
go func() { go func() {
for { for {
@@ -93,6 +133,9 @@ func (r *Runner) runAsynq(ctx context.Context) error {
<-ctx.Done() <-ctx.Done()
r.deps.Log.Info("runner: context cancelled, shutting down asynq server") r.deps.Log.Info("runner: context cancelled, shutting down asynq server")
srv.Shutdown() srv.Shutdown()
// Wait for translation tasks to complete.
translationWg.Wait()
return nil return nil
} }
@@ -147,3 +190,47 @@ func (r *Runner) handleAudioTask(ctx context.Context, t *asynq.Task) error {
r.runAudioTask(ctx, task) r.runAudioTask(ctx, task)
return nil return nil
} }
// pollTranslationTasks claims all available translation tasks from PocketBase
// and dispatches them to goroutines. Translation tasks don't go through Redis/Asynq
// because they're stored in PocketBase, so we need this separate poll loop.
func (r *Runner) pollTranslationTasks(ctx context.Context, translationSem chan struct{}, wg *sync.WaitGroup) {
// Reap orphaned tasks (same logic as poll() in runner.go).
if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil {
r.deps.Log.Warn("runner: reap stale translation tasks failed", "err", err)
} else if n > 0 {
r.deps.Log.Info("runner: reaped stale translation tasks", "count", n)
}
translationLoop:
for {
if ctx.Err() != nil {
return
}
select {
case translationSem <- struct{}{}:
// Slot acquired — proceed to claim a task.
default:
// All slots busy; leave remaining pending tasks for next tick.
break translationLoop
}
task, ok, err := r.deps.Consumer.ClaimNextTranslationTask(ctx, r.cfg.WorkerID)
if err != nil {
<-translationSem
r.deps.Log.Error("runner: ClaimNextTranslationTask failed", "err", err)
break
}
if !ok {
<-translationSem
break
}
r.tasksRunning.Add(1)
wg.Add(1)
go func(t domain.TranslationTask) {
defer wg.Done()
defer func() { <-translationSem }()
defer r.tasksRunning.Add(-1)
r.runTranslationTask(ctx, t)
}(task)
}
}