refactor: split worker into paid/free via WORKER_PLAN env var
Replace dual-consumer priority WorkerPool with a single consumer per worker process. WORKER_PLAN=paid|free selects the Kafka topic and consumer group ID (dish-recognition-paid / dish-recognition-free). docker-compose now runs worker-paid and worker-free as separate services for independent scaling. Makefile dev target launches both workers locally. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -4,50 +4,42 @@ import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/food-ai/backend/internal/adapters/kafka"
|
||||
)
|
||||
|
||||
const defaultWorkerCount = 5
|
||||
|
||||
// WorkerPool processes dish recognition jobs from Kafka with priority queuing.
|
||||
// Paid jobs are processed before free jobs.
|
||||
// WorkerPool processes dish recognition jobs from a single Kafka topic.
|
||||
type WorkerPool struct {
|
||||
jobRepo JobRepository
|
||||
recognizer Recognizer
|
||||
dishRepo DishRepository
|
||||
paidConsumer *kafka.Consumer
|
||||
freeConsumer *kafka.Consumer
|
||||
workerCount int
|
||||
paidJobs chan string
|
||||
freeJobs chan string
|
||||
jobRepo JobRepository
|
||||
recognizer Recognizer
|
||||
dishRepo DishRepository
|
||||
consumer *kafka.Consumer
|
||||
workerCount int
|
||||
jobs chan string
|
||||
}
|
||||
|
||||
// NewWorkerPool creates a WorkerPool with five workers.
|
||||
// NewWorkerPool creates a WorkerPool with five workers consuming from a single consumer.
|
||||
func NewWorkerPool(
|
||||
jobRepo JobRepository,
|
||||
recognizer Recognizer,
|
||||
dishRepo DishRepository,
|
||||
paidConsumer *kafka.Consumer,
|
||||
freeConsumer *kafka.Consumer,
|
||||
consumer *kafka.Consumer,
|
||||
) *WorkerPool {
|
||||
return &WorkerPool{
|
||||
jobRepo: jobRepo,
|
||||
recognizer: recognizer,
|
||||
dishRepo: dishRepo,
|
||||
paidConsumer: paidConsumer,
|
||||
freeConsumer: freeConsumer,
|
||||
workerCount: defaultWorkerCount,
|
||||
paidJobs: make(chan string, 100),
|
||||
freeJobs: make(chan string, 100),
|
||||
jobRepo: jobRepo,
|
||||
recognizer: recognizer,
|
||||
dishRepo: dishRepo,
|
||||
consumer: consumer,
|
||||
workerCount: defaultWorkerCount,
|
||||
jobs: make(chan string, 100),
|
||||
}
|
||||
}
|
||||
|
||||
// Start launches the Kafka feeder goroutines and all worker goroutines.
|
||||
// Start launches the Kafka feeder goroutine and all worker goroutines.
|
||||
func (pool *WorkerPool) Start(workerContext context.Context) {
|
||||
go pool.paidConsumer.Run(workerContext, pool.paidJobs)
|
||||
go pool.freeConsumer.Run(workerContext, pool.freeJobs)
|
||||
go pool.consumer.Run(workerContext, pool.jobs)
|
||||
for i := 0; i < pool.workerCount; i++ {
|
||||
go pool.runWorker(workerContext)
|
||||
}
|
||||
@@ -55,26 +47,11 @@ func (pool *WorkerPool) Start(workerContext context.Context) {
|
||||
|
||||
func (pool *WorkerPool) runWorker(workerContext context.Context) {
|
||||
for {
|
||||
// Priority step: drain paid queue without blocking.
|
||||
select {
|
||||
case jobID := <-pool.paidJobs:
|
||||
pool.processJob(workerContext, jobID)
|
||||
continue
|
||||
case <-workerContext.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
// Fall back to either queue with a 100ms timeout.
|
||||
select {
|
||||
case jobID := <-pool.paidJobs:
|
||||
pool.processJob(workerContext, jobID)
|
||||
case jobID := <-pool.freeJobs:
|
||||
case jobID := <-pool.jobs:
|
||||
pool.processJob(workerContext, jobID)
|
||||
case <-workerContext.Done():
|
||||
return
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
// nothing available; loop again
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user