refactor: split worker into paid/free via WORKER_PLAN env var

Replace dual-consumer priority WorkerPool with a single consumer per
worker process. WORKER_PLAN=paid|free selects the Kafka topic and
consumer group ID (dish-recognition-paid / dish-recognition-free).

docker-compose now runs worker-paid and worker-free as separate services
for independent scaling. Makefile dev target launches both workers locally.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
dbastrikin
2026-03-19 12:11:14 +02:00
parent 1afadf50a7
commit 1aaf20619d
4 changed files with 57 additions and 61 deletions

View File

@@ -4,50 +4,42 @@ import (
"context"
"log/slog"
"sync"
"time"
"github.com/food-ai/backend/internal/adapters/kafka"
)
const defaultWorkerCount = 5
// WorkerPool processes dish recognition jobs from Kafka with priority queuing.
// Paid jobs are processed before free jobs.
// WorkerPool processes dish recognition jobs from a single Kafka topic.
type WorkerPool struct {
jobRepo JobRepository
recognizer Recognizer
dishRepo DishRepository
paidConsumer *kafka.Consumer
freeConsumer *kafka.Consumer
workerCount int
paidJobs chan string
freeJobs chan string
jobRepo JobRepository
recognizer Recognizer
dishRepo DishRepository
consumer *kafka.Consumer
workerCount int
jobs chan string
}
// NewWorkerPool creates a WorkerPool with five workers.
// NewWorkerPool creates a WorkerPool with five workers consuming from a single consumer.
func NewWorkerPool(
jobRepo JobRepository,
recognizer Recognizer,
dishRepo DishRepository,
paidConsumer *kafka.Consumer,
freeConsumer *kafka.Consumer,
consumer *kafka.Consumer,
) *WorkerPool {
return &WorkerPool{
jobRepo: jobRepo,
recognizer: recognizer,
dishRepo: dishRepo,
paidConsumer: paidConsumer,
freeConsumer: freeConsumer,
workerCount: defaultWorkerCount,
paidJobs: make(chan string, 100),
freeJobs: make(chan string, 100),
jobRepo: jobRepo,
recognizer: recognizer,
dishRepo: dishRepo,
consumer: consumer,
workerCount: defaultWorkerCount,
jobs: make(chan string, 100),
}
}
// Start launches the Kafka feeder goroutines and all worker goroutines.
// Start launches the Kafka feeder goroutine and all worker goroutines.
func (pool *WorkerPool) Start(workerContext context.Context) {
go pool.paidConsumer.Run(workerContext, pool.paidJobs)
go pool.freeConsumer.Run(workerContext, pool.freeJobs)
go pool.consumer.Run(workerContext, pool.jobs)
for i := 0; i < pool.workerCount; i++ {
go pool.runWorker(workerContext)
}
@@ -55,26 +47,11 @@ func (pool *WorkerPool) Start(workerContext context.Context) {
func (pool *WorkerPool) runWorker(workerContext context.Context) {
for {
// Priority step: drain paid queue without blocking.
select {
case jobID := <-pool.paidJobs:
pool.processJob(workerContext, jobID)
continue
case <-workerContext.Done():
return
default:
}
// Fall back to either queue with a 100ms timeout.
select {
case jobID := <-pool.paidJobs:
pool.processJob(workerContext, jobID)
case jobID := <-pool.freeJobs:
case jobID := <-pool.jobs:
pool.processJob(workerContext, jobID)
case <-workerContext.Done():
return
case <-time.After(100 * time.Millisecond):
// nothing available; loop again
}
}
}