Remove denormalized columns (name, calories, protein_g, fat_g, carbs_g) from meal_diary. Name is now resolved via JOIN with dishes/dish_translations; macros are computed as recipe.*_per_serving * portions at query time. - Add dish.Repository.FindOrCreateRecipe: finds or creates a minimal recipe stub seeded with AI-estimated macros - recognition/handler: resolve recipe_id synchronously per candidate; simplify enrichDishInBackground to translations-only - diary/handler: accept dish_id OR name; always resolve recipe_id via FindOrCreateRecipe before INSERT - diary/entity: DishID is now non-nullable string; CreateRequest drops macros - diary/repository: ListByDate and Create use JOIN to return computed macros - ai/types: add RecipeID field to DishCandidate - Update tests and wire_gen accordingly Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
428 lines
14 KiB
Go
428 lines
14 KiB
Go
package recognition
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/food-ai/backend/internal/adapters/ai"
|
|
"github.com/food-ai/backend/internal/domain/dish"
|
|
"github.com/food-ai/backend/internal/domain/ingredient"
|
|
"github.com/food-ai/backend/internal/infra/locale"
|
|
"github.com/food-ai/backend/internal/infra/middleware"
|
|
)
|
|
|
|
// DishRepository is the subset of dish.Repository used by this handler.
|
|
type DishRepository interface {
|
|
FindOrCreate(ctx context.Context, name string) (string, bool, error)
|
|
FindOrCreateRecipe(ctx context.Context, dishID string, calories, proteinG, fatG, carbsG float64) (string, bool, error)
|
|
UpsertTranslation(ctx context.Context, dishID, lang, name string) error
|
|
AddRecipe(ctx context.Context, dishID string, req dish.CreateRequest) (string, error)
|
|
}
|
|
|
|
// IngredientRepository is the subset of ingredient.Repository used by this handler.
|
|
type IngredientRepository interface {
|
|
FuzzyMatch(ctx context.Context, name string) (*ingredient.IngredientMapping, error)
|
|
Upsert(ctx context.Context, m *ingredient.IngredientMapping) (*ingredient.IngredientMapping, error)
|
|
UpsertTranslation(ctx context.Context, id, lang, name string) error
|
|
UpsertAliases(ctx context.Context, id, lang string, aliases []string) error
|
|
}
|
|
|
|
// Recognizer is the AI provider interface for image-based food recognition.
|
|
type Recognizer interface {
|
|
RecognizeReceipt(ctx context.Context, imageBase64, mimeType, lang string) (*ai.ReceiptResult, error)
|
|
RecognizeProducts(ctx context.Context, imageBase64, mimeType, lang string) ([]ai.RecognizedItem, error)
|
|
RecognizeDish(ctx context.Context, imageBase64, mimeType, lang string) (*ai.DishResult, error)
|
|
ClassifyIngredient(ctx context.Context, name string) (*ai.IngredientClassification, error)
|
|
GenerateRecipeForDish(ctx context.Context, dishName string) (*ai.Recipe, error)
|
|
TranslateDishName(ctx context.Context, name string) (map[string]string, error)
|
|
}
|
|
|
|
// Handler handles POST /ai/* recognition endpoints.
|
|
type Handler struct {
|
|
recognizer Recognizer
|
|
ingredientRepo IngredientRepository
|
|
dishRepo DishRepository
|
|
}
|
|
|
|
// NewHandler creates a new Handler.
|
|
func NewHandler(recognizer Recognizer, repo IngredientRepository, dishRepo DishRepository) *Handler {
|
|
return &Handler{recognizer: recognizer, ingredientRepo: repo, dishRepo: dishRepo}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Request / Response types
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// imageRequest is the common request body containing a single base64-encoded image.
|
|
type imageRequest struct {
|
|
ImageBase64 string `json:"image_base64"`
|
|
MimeType string `json:"mime_type"`
|
|
}
|
|
|
|
// imagesRequest is the request body for multi-image endpoints.
|
|
type imagesRequest struct {
|
|
Images []imageRequest `json:"images"`
|
|
}
|
|
|
|
// EnrichedItem is a recognized food item enriched with ingredient_mappings data.
|
|
type EnrichedItem struct {
|
|
Name string `json:"name"`
|
|
Quantity float64 `json:"quantity"`
|
|
Unit string `json:"unit"`
|
|
Category string `json:"category"`
|
|
Confidence float64 `json:"confidence"`
|
|
MappingID *string `json:"mapping_id"`
|
|
StorageDays int `json:"storage_days"`
|
|
}
|
|
|
|
// ReceiptResponse is the response for POST /ai/recognize-receipt.
|
|
type ReceiptResponse struct {
|
|
Items []EnrichedItem `json:"items"`
|
|
Unrecognized []ai.UnrecognizedItem `json:"unrecognized"`
|
|
}
|
|
|
|
// DishResponse is the response for POST /ai/recognize-dish.
|
|
type DishResponse = ai.DishResult
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Handlers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// RecognizeReceipt handles POST /ai/recognize-receipt.
|
|
// Body: {"image_base64": "...", "mime_type": "image/jpeg"}
|
|
func (h *Handler) RecognizeReceipt(w http.ResponseWriter, r *http.Request) {
|
|
userID := middleware.UserIDFromCtx(r.Context())
|
|
_ = userID // logged for tracing
|
|
|
|
var req imageRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.ImageBase64 == "" {
|
|
writeErrorJSON(w, http.StatusBadRequest, "image_base64 is required")
|
|
return
|
|
}
|
|
|
|
lang := locale.FromContext(r.Context())
|
|
result, err := h.recognizer.RecognizeReceipt(r.Context(), req.ImageBase64, req.MimeType, lang)
|
|
if err != nil {
|
|
slog.Error("recognize receipt", "err", err)
|
|
writeErrorJSON(w, http.StatusServiceUnavailable, "recognition failed, please try again")
|
|
return
|
|
}
|
|
|
|
enriched := h.enrichItems(r.Context(), result.Items)
|
|
writeJSON(w, http.StatusOK, ReceiptResponse{
|
|
Items: enriched,
|
|
Unrecognized: result.Unrecognized,
|
|
})
|
|
}
|
|
|
|
// RecognizeProducts handles POST /ai/recognize-products.
|
|
// Body: {"images": [{"image_base64": "...", "mime_type": "image/jpeg"}, ...]}
|
|
func (h *Handler) RecognizeProducts(w http.ResponseWriter, r *http.Request) {
|
|
var req imagesRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || len(req.Images) == 0 {
|
|
writeErrorJSON(w, http.StatusBadRequest, "at least one image is required")
|
|
return
|
|
}
|
|
if len(req.Images) > 3 {
|
|
req.Images = req.Images[:3] // cap at 3 photos as per spec
|
|
}
|
|
|
|
// Process each image in parallel.
|
|
lang := locale.FromContext(r.Context())
|
|
allItems := make([][]ai.RecognizedItem, len(req.Images))
|
|
var wg sync.WaitGroup
|
|
for i, img := range req.Images {
|
|
wg.Add(1)
|
|
go func(i int, img imageRequest) {
|
|
defer wg.Done()
|
|
items, err := h.recognizer.RecognizeProducts(r.Context(), img.ImageBase64, img.MimeType, lang)
|
|
if err != nil {
|
|
slog.Warn("recognize products from image", "index", i, "err", err)
|
|
return
|
|
}
|
|
allItems[i] = items
|
|
}(i, img)
|
|
}
|
|
wg.Wait()
|
|
|
|
merged := MergeAndDeduplicate(allItems)
|
|
enriched := h.enrichItems(r.Context(), merged)
|
|
writeJSON(w, http.StatusOK, map[string]any{"items": enriched})
|
|
}
|
|
|
|
// RecognizeDish handles POST /ai/recognize-dish.
|
|
// Body: {"image_base64": "...", "mime_type": "image/jpeg"}
|
|
func (h *Handler) RecognizeDish(w http.ResponseWriter, r *http.Request) {
|
|
var req imageRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.ImageBase64 == "" {
|
|
writeErrorJSON(w, http.StatusBadRequest, "image_base64 is required")
|
|
return
|
|
}
|
|
|
|
lang := locale.FromContext(r.Context())
|
|
result, err := h.recognizer.RecognizeDish(r.Context(), req.ImageBase64, req.MimeType, lang)
|
|
if err != nil {
|
|
slog.Error("recognize dish", "err", err)
|
|
writeErrorJSON(w, http.StatusServiceUnavailable, "recognition failed, please try again")
|
|
return
|
|
}
|
|
|
|
// Resolve dish_id and recipe_id for each candidate in parallel.
|
|
var mu sync.Mutex
|
|
var wg sync.WaitGroup
|
|
for i := range result.Candidates {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
candidate := result.Candidates[i]
|
|
dishID, created, findError := h.dishRepo.FindOrCreate(r.Context(), candidate.DishName)
|
|
if findError != nil {
|
|
slog.Warn("find or create dish", "name", candidate.DishName, "err", findError)
|
|
return
|
|
}
|
|
mu.Lock()
|
|
result.Candidates[i].DishID = &dishID
|
|
mu.Unlock()
|
|
if created {
|
|
go h.enrichDishInBackground(dishID, candidate.DishName)
|
|
}
|
|
|
|
recipeID, _, recipeError := h.dishRepo.FindOrCreateRecipe(
|
|
r.Context(), dishID,
|
|
candidate.Calories, candidate.ProteinG, candidate.FatG, candidate.CarbsG,
|
|
)
|
|
if recipeError != nil {
|
|
slog.Warn("find or create recipe", "dish_id", dishID, "err", recipeError)
|
|
return
|
|
}
|
|
mu.Lock()
|
|
result.Candidates[i].RecipeID = &recipeID
|
|
mu.Unlock()
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
|
|
writeJSON(w, http.StatusOK, result)
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// enrichItems matches each recognized item against ingredient_mappings.
|
|
// Items without a match trigger a Gemini classification call and upsert into the DB.
|
|
func (h *Handler) enrichItems(ctx context.Context, items []ai.RecognizedItem) []EnrichedItem {
|
|
result := make([]EnrichedItem, 0, len(items))
|
|
for _, item := range items {
|
|
enriched := EnrichedItem{
|
|
Name: item.Name,
|
|
Quantity: item.Quantity,
|
|
Unit: item.Unit,
|
|
Category: item.Category,
|
|
Confidence: item.Confidence,
|
|
StorageDays: 7, // sensible default
|
|
}
|
|
|
|
mapping, err := h.ingredientRepo.FuzzyMatch(ctx, item.Name)
|
|
if err != nil {
|
|
slog.Warn("fuzzy match ingredient", "name", item.Name, "err", err)
|
|
}
|
|
|
|
if mapping != nil {
|
|
// Found existing mapping — use its canonical data.
|
|
id := mapping.ID
|
|
enriched.MappingID = &id
|
|
if mapping.DefaultUnit != nil {
|
|
enriched.Unit = *mapping.DefaultUnit
|
|
}
|
|
if mapping.StorageDays != nil {
|
|
enriched.StorageDays = *mapping.StorageDays
|
|
}
|
|
if mapping.Category != nil {
|
|
enriched.Category = *mapping.Category
|
|
}
|
|
} else {
|
|
// No mapping — ask AI to classify and save for future reuse.
|
|
classification, err := h.recognizer.ClassifyIngredient(ctx, item.Name)
|
|
if err != nil {
|
|
slog.Warn("classify unknown ingredient", "name", item.Name, "err", err)
|
|
} else {
|
|
saved := h.saveClassification(ctx, classification)
|
|
if saved != nil {
|
|
id := saved.ID
|
|
enriched.MappingID = &id
|
|
}
|
|
enriched.Category = classification.Category
|
|
enriched.Unit = classification.DefaultUnit
|
|
enriched.StorageDays = classification.StorageDays
|
|
}
|
|
}
|
|
result = append(result, enriched)
|
|
}
|
|
return result
|
|
}
|
|
|
|
// saveClassification upserts an AI-produced ingredient classification into the DB.
|
|
func (h *Handler) saveClassification(ctx context.Context, c *ai.IngredientClassification) *ingredient.IngredientMapping {
|
|
if c == nil || c.CanonicalName == "" {
|
|
return nil
|
|
}
|
|
|
|
m := &ingredient.IngredientMapping{
|
|
CanonicalName: c.CanonicalName,
|
|
Category: strPtr(c.Category),
|
|
DefaultUnit: strPtr(c.DefaultUnit),
|
|
CaloriesPer100g: c.CaloriesPer100g,
|
|
ProteinPer100g: c.ProteinPer100g,
|
|
FatPer100g: c.FatPer100g,
|
|
CarbsPer100g: c.CarbsPer100g,
|
|
StorageDays: intPtr(c.StorageDays),
|
|
}
|
|
|
|
saved, err := h.ingredientRepo.Upsert(ctx, m)
|
|
if err != nil {
|
|
slog.Warn("upsert classified ingredient", "name", c.CanonicalName, "err", err)
|
|
return nil
|
|
}
|
|
|
|
if len(c.Aliases) > 0 {
|
|
if err := h.ingredientRepo.UpsertAliases(ctx, saved.ID, "en", c.Aliases); err != nil {
|
|
slog.Warn("upsert ingredient aliases", "id", saved.ID, "err", err)
|
|
}
|
|
}
|
|
|
|
for _, t := range c.Translations {
|
|
if err := h.ingredientRepo.UpsertTranslation(ctx, saved.ID, t.Lang, t.Name); err != nil {
|
|
slog.Warn("upsert ingredient translation", "id", saved.ID, "lang", t.Lang, "err", err)
|
|
}
|
|
if len(t.Aliases) > 0 {
|
|
if err := h.ingredientRepo.UpsertAliases(ctx, saved.ID, t.Lang, t.Aliases); err != nil {
|
|
slog.Warn("upsert ingredient translation aliases", "id", saved.ID, "lang", t.Lang, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
return saved
|
|
}
|
|
|
|
// enrichDishInBackground generates name translations for a newly created dish stub.
|
|
// Recipe creation is handled synchronously in RecognizeDish.
|
|
// Runs as a fire-and-forget goroutine so it never blocks the HTTP response.
|
|
func (h *Handler) enrichDishInBackground(dishID, dishName string) {
|
|
enrichContext := context.Background()
|
|
|
|
translations, translateError := h.recognizer.TranslateDishName(enrichContext, dishName)
|
|
if translateError != nil {
|
|
slog.Warn("translate dish name", "name", dishName, "err", translateError)
|
|
return
|
|
}
|
|
for lang, translatedName := range translations {
|
|
if upsertError := h.dishRepo.UpsertTranslation(enrichContext, dishID, lang, translatedName); upsertError != nil {
|
|
slog.Warn("upsert dish translation", "dish_id", dishID, "lang", lang, "err", upsertError)
|
|
}
|
|
}
|
|
}
|
|
|
|
// aiRecipeToCreateRequest converts an AI-generated recipe into a dish.CreateRequest.
|
|
func aiRecipeToCreateRequest(recipe *ai.Recipe) dish.CreateRequest {
|
|
ingredients := make([]dish.IngredientInput, len(recipe.Ingredients))
|
|
for i, ingredient := range recipe.Ingredients {
|
|
ingredients[i] = dish.IngredientInput{
|
|
Name: ingredient.Name, Amount: ingredient.Amount, Unit: ingredient.Unit,
|
|
}
|
|
}
|
|
steps := make([]dish.StepInput, len(recipe.Steps))
|
|
for i, step := range recipe.Steps {
|
|
steps[i] = dish.StepInput{
|
|
Number: step.Number, Description: step.Description, TimerSeconds: step.TimerSeconds,
|
|
}
|
|
}
|
|
return dish.CreateRequest{
|
|
Name: recipe.Title,
|
|
Description: recipe.Description,
|
|
CuisineSlug: recipe.Cuisine,
|
|
ImageURL: recipe.ImageURL,
|
|
Tags: recipe.Tags,
|
|
Source: "ai",
|
|
Difficulty: recipe.Difficulty,
|
|
PrepTimeMin: recipe.PrepTimeMin,
|
|
CookTimeMin: recipe.CookTimeMin,
|
|
Servings: recipe.Servings,
|
|
Calories: recipe.Nutrition.Calories,
|
|
Protein: recipe.Nutrition.ProteinG,
|
|
Fat: recipe.Nutrition.FatG,
|
|
Carbs: recipe.Nutrition.CarbsG,
|
|
Ingredients: ingredients,
|
|
Steps: steps,
|
|
}
|
|
}
|
|
|
|
// MergeAndDeduplicate combines results from multiple images.
|
|
// Items sharing the same name (case-insensitive) have their quantities summed.
|
|
func MergeAndDeduplicate(batches [][]ai.RecognizedItem) []ai.RecognizedItem {
|
|
seen := make(map[string]*ai.RecognizedItem)
|
|
var order []string
|
|
|
|
for _, batch := range batches {
|
|
for i := range batch {
|
|
item := &batch[i]
|
|
key := normalizeName(item.Name)
|
|
if existing, ok := seen[key]; ok {
|
|
existing.Quantity += item.Quantity
|
|
// Keep the higher confidence estimate.
|
|
if item.Confidence > existing.Confidence {
|
|
existing.Confidence = item.Confidence
|
|
}
|
|
} else {
|
|
seen[key] = item
|
|
order = append(order, key)
|
|
}
|
|
}
|
|
}
|
|
|
|
result := make([]ai.RecognizedItem, 0, len(order))
|
|
for _, key := range order {
|
|
result = append(result, *seen[key])
|
|
}
|
|
return result
|
|
}
|
|
|
|
func normalizeName(s string) string {
|
|
return strings.ToLower(strings.TrimSpace(s))
|
|
}
|
|
|
|
func strPtr(s string) *string {
|
|
if s == "" {
|
|
return nil
|
|
}
|
|
return &s
|
|
}
|
|
|
|
func intPtr(n int) *int {
|
|
return &n
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// HTTP helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type errorResponse struct {
|
|
Error string `json:"error"`
|
|
}
|
|
|
|
func writeErrorJSON(w http.ResponseWriter, status int, msg string) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(errorResponse{Error: msg})
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, status int, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(v)
|
|
}
|