acrius/food-ai
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: meal tracking, dish recognition UX improvements, English AI prompts

Browse Source 
Backend:
- Translate all recognition prompts (receipt, products, dish) from Russian to English
- Add lang parameter to Recognizer interface and pass locale.FromContext in handlers
- DishResult type uses candidates array for multi-candidate responses

Client:
- Add meal tracking: diary provider, date selector, meal type model
- DishResult parser: backward-compatible with legacy flat format and new candidates format
- DishResultScreen: sticky bottom button, full-width portion/meal-type inputs,
  КБЖУ disclaimer moved under nutrition card, add date field to diary POST body
- Recognition prompts now return dish/product names in user's preferred language
- Onboarding, profile, home screen visual updates

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
dbastrikin
2026-03-17 14:29:36 +02:00
parent 2a95bcd53c
commit 87ef2097fc
16 changed files with 1269 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
backend/internal/adapters/ai/types.go
View File

@@ -94,16 +94,20 @@ type ReceiptResult struct {
Unrecognized []UnrecognizedItem `json:"unrecognized"`
}
// DishResult is the result of dish recognition.
// DishCandidate is a single dish recognition candidate with estimated nutrition.
type DishCandidate struct {
DishName string `json:"dish_name"`
WeightGrams int `json:"weight_grams"`
Calories float64 `json:"calories"`
ProteinG float64 `json:"protein_g"`
FatG float64 `json:"fat_g"`
CarbsG float64 `json:"carbs_g"`
Confidence float64 `json:"confidence"`
}
// DishResult is the result of dish recognition with multiple ranked candidates.
type DishResult struct {
DishName string `json:"dish_name"`
WeightGrams int `json:"weight_grams"`
Calories float64 `json:"calories"`
ProteinG float64 `json:"protein_g"`
FatG float64 `json:"fat_g"`
CarbsG float64 `json:"carbs_g"`
Confidence float64 `json:"confidence"`
SimilarDishes []string `json:"similar_dishes"`
Candidates []DishCandidate `json:"candidates"`
}
// IngredientTranslation holds the localized name and aliases for one language.

135
backend/internal/adapters/openai/recognition.go
View File

@@ -9,30 +9,43 @@ import (
"github.com/food-ai/backend/internal/adapters/ai"
)
// RecognizeReceipt uses the vision model to extract food items from a receipt photo.
func (c *Client) RecognizeReceipt(ctx context.Context, imageBase64, mimeType string) (*ai.ReceiptResult, error) {
prompt := `Ты — OCR-система для чеков из продуктовых магазинов.
// langNames maps ISO 639-1 codes to English language names used in AI prompts.
var langNames = map[string]string{
"en": "English", "ru": "Russian", "es": "Spanish",
"de": "German", "fr": "French", "it": "Italian",
"pt": "Portuguese", "zh": "Chinese", "ja": "Japanese",
"ko": "Korean", "ar": "Arabic", "hi": "Hindi",
}
Проанализируй фото чека и извлеки список продуктов питания.
Для каждого продукта определи:
- name: название на русском языке (убери артикулы, коды, лишние символы)
- quantity: количество (число)
- unit: единица (г, кг, мл, л, шт, уп)
// RecognizeReceipt uses the vision model to extract food items from a receipt photo.
func (c *Client) RecognizeReceipt(ctx context.Context, imageBase64, mimeType, lang string) (*ai.ReceiptResult, error) {
langName := langNames[lang]
if langName == "" {
langName = "English"
}
prompt := fmt.Sprintf(`You are an OCR system for grocery receipts.
Analyse the receipt photo and extract a list of food products.
For each product determine:
- name: product name (remove article codes, extra symbols)
- quantity: amount (number)
- unit: unit (g, kg, ml, l, pcs, pack)
- category: dairy | meat | produce | bakery | frozen | beverages | other
- confidence: 0.0–1.0
Позиции, которые не являются едой (бытовая химия, табак, алкоголь) — пропусти.
Позиции с нечитаемым текстом — добавь в unrecognized.
Skip items that are not food (household chemicals, tobacco, alcohol).
Items with unreadable text — add to unrecognized.
Верни ТОЛЬКО валидный JSON без markdown:
Return all text fields (name) in %s.
Return ONLY valid JSON without markdown:
{
"items": [
{"name": "Молоко 2.5%", "quantity": 1, "unit": "л", "category": "dairy", "confidence": 0.95}
{"name": "...", "quantity": 1, "unit": "l", "category": "dairy", "confidence": 0.95}
],
"unrecognized": [
{"raw_text": "ТОВ АРТИК 1ШТ", "price": 89.0}
{"raw_text": "...", "price": 89.0}
]
}`
}`, langName)
text, err := c.generateVisionContent(ctx, prompt, imageBase64, mimeType)
if err != nil {
@@ -53,25 +66,30 @@ func (c *Client) RecognizeReceipt(ctx context.Context, imageBase64, mimeType str
}
// RecognizeProducts uses the vision model to identify food items in a photo (fridge, shelf, etc.).
func (c *Client) RecognizeProducts(ctx context.Context, imageBase64, mimeType string) ([]ai.RecognizedItem, error) {
prompt := `Ты — система распознавания продуктов питания.
func (c *Client) RecognizeProducts(ctx context.Context, imageBase64, mimeType, lang string) ([]ai.RecognizedItem, error) {
langName := langNames[lang]
if langName == "" {
langName = "English"
}
prompt := fmt.Sprintf(`You are a food product recognition system.
Посмотри на фото и определи все видимые продукты питания.
Для каждого продукта оцени:
- name: название на русском языке
- quantity: приблизительное количество (число)
- unit: единица (г, кг, мл, л, шт)
Look at the photo and identify all visible food products.
For each product estimate:
- name: product name
- quantity: approximate amount (number)
- unit: unit (g, kg, ml, l, pcs)
- category: dairy | meat | produce | bakery | frozen | beverages | other
- confidence: 0.0–1.0
Только продукты питания. Пустые упаковки и несъедобные предметы — пропусти.
Food products only. Skip empty packaging and inedible objects.
Верни ТОЛЬКО валидный JSON без markdown:
Return all text fields (name) in %s.
Return ONLY valid JSON without markdown:
{
"items": [
{"name": "Яйца", "quantity": 10, "unit": "шт", "category": "dairy", "confidence": 0.9}
{"name": "...", "quantity": 10, "unit": "pcs", "category": "dairy", "confidence": 0.9}
]
}`
}`, langName)
text, err := c.generateVisionContent(ctx, prompt, imageBase64, mimeType)
if err != nil {
@@ -91,28 +109,49 @@ func (c *Client) RecognizeProducts(ctx context.Context, imageBase64, mimeType st
}
// RecognizeDish uses the vision model to identify a dish and estimate its nutritional content.
func (c *Client) RecognizeDish(ctx context.Context, imageBase64, mimeType string) (*ai.DishResult, error) {
prompt := `Ты — диетолог и кулинарный эксперт.
// Returns 3–5 ranked candidates so the user can correct mis-identifications.
func (c *Client) RecognizeDish(ctx context.Context, imageBase64, mimeType, lang string) (*ai.DishResult, error) {
langName := langNames[lang]
if langName == "" {
langName = "English"
}
prompt := fmt.Sprintf(`You are a dietitian and culinary expert.
Посмотри на фото блюда и определи:
- dish_name: название блюда на русском языке
- weight_grams: приблизительный вес порции в граммах
- calories: калорийность порции (приблизительно)
- protein_g, fat_g, carbs_g: БЖУ на порцию
- confidence: 0.0–1.0
- similar_dishes: до 3 похожих блюд (для поиска рецептов)
Look at the dish photo and suggest 3 to 5 possible dishes it could be.
Even if the first option is obvious, add 2–4 alternative dishes with lower confidence.
For each candidate specify:
- dish_name: dish name
- weight_grams: approximate portion weight in grams (estimate from photo)
- calories: calories for this portion (kcal)
- protein_g, fat_g, carbs_g: macros for this portion (grams)
- confidence: certainty 0.0–1.0
Верни ТОЛЬКО валидный JSON без markdown:
Sort candidates by descending confidence. First — most likely.
Return dish_name values in %s.
Return ONLY valid JSON without markdown:
{
"dish_name": "Паста Карбонара",
"weight_grams": 350,
"calories": 520,
"protein_g": 22,
"fat_g": 26,
"carbs_g": 48,
"confidence": 0.85,
"similar_dishes": ["Паста с беконом", "Спагетти"]
}`
"candidates": [
{
"dish_name": "...",
"weight_grams": 350,
"calories": 520,
"protein_g": 22,
"fat_g": 26,
"carbs_g": 48,
"confidence": 0.88
},
{
"dish_name": "...",
"weight_grams": 350,
"calories": 540,
"protein_g": 20,
"fat_g": 28,
"carbs_g": 49,
"confidence": 0.65
}
]
}`, langName)
text, err := c.generateVisionContent(ctx, prompt, imageBase64, mimeType)
if err != nil {
@@ -120,11 +159,11 @@ func (c *Client) RecognizeDish(ctx context.Context, imageBase64, mimeType string
}
var result ai.DishResult
if err := parseJSON(text, &result); err != nil {
return nil, fmt.Errorf("parse dish result: %w", err)
if parseError := parseJSON(text, &result); parseError != nil {
return nil, fmt.Errorf("parse dish result: %w", parseError)
}
if result.SimilarDishes == nil {
result.SimilarDishes = []string{}
if result.Candidates == nil {
result.Candidates = []ai.DishCandidate{}
}
return &result, nil
}

16
backend/internal/domain/recognition/handler.go
View File

@@ -9,6 +9,7 @@ import (
"sync"
"github.com/food-ai/backend/internal/adapters/ai"
"github.com/food-ai/backend/internal/infra/locale"
"github.com/food-ai/backend/internal/infra/middleware"
"github.com/food-ai/backend/internal/domain/ingredient"
)
@@ -23,9 +24,9 @@ type IngredientRepository interface {
// Recognizer is the AI provider interface for image-based food recognition.
type Recognizer interface {
RecognizeReceipt(ctx context.Context, imageBase64, mimeType string) (*ai.ReceiptResult, error)
RecognizeProducts(ctx context.Context, imageBase64, mimeType string) ([]ai.RecognizedItem, error)
RecognizeDish(ctx context.Context, imageBase64, mimeType string) (*ai.DishResult, error)
RecognizeReceipt(ctx context.Context, imageBase64, mimeType, lang string) (*ai.ReceiptResult, error)
RecognizeProducts(ctx context.Context, imageBase64, mimeType, lang string) ([]ai.RecognizedItem, error)
RecognizeDish(ctx context.Context, imageBase64, mimeType, lang string) (*ai.DishResult, error)
ClassifyIngredient(ctx context.Context, name string) (*ai.IngredientClassification, error)
}
@@ -91,7 +92,8 @@ func (h *Handler) RecognizeReceipt(w http.ResponseWriter, r *http.Request) {
return
}
result, err := h.recognizer.RecognizeReceipt(r.Context(), req.ImageBase64, req.MimeType)
lang := locale.FromContext(r.Context())
result, err := h.recognizer.RecognizeReceipt(r.Context(), req.ImageBase64, req.MimeType, lang)
if err != nil {
slog.Error("recognize receipt", "err", err)
writeErrorJSON(w, http.StatusServiceUnavailable, "recognition failed, please try again")
@@ -118,13 +120,14 @@ func (h *Handler) RecognizeProducts(w http.ResponseWriter, r *http.Request) {
}
// Process each image in parallel.
lang := locale.FromContext(r.Context())
allItems := make([][]ai.RecognizedItem, len(req.Images))
var wg sync.WaitGroup
for i, img := range req.Images {
wg.Add(1)
go func(i int, img imageRequest) {
defer wg.Done()
items, err := h.recognizer.RecognizeProducts(r.Context(), img.ImageBase64, img.MimeType)
items, err := h.recognizer.RecognizeProducts(r.Context(), img.ImageBase64, img.MimeType, lang)
if err != nil {
slog.Warn("recognize products from image", "index", i, "err", err)
return
@@ -148,7 +151,8 @@ func (h *Handler) RecognizeDish(w http.ResponseWriter, r *http.Request) {
return
}
result, err := h.recognizer.RecognizeDish(r.Context(), req.ImageBase64, req.MimeType)
lang := locale.FromContext(r.Context())
result, err := h.recognizer.RecognizeDish(r.Context(), req.ImageBase64, req.MimeType, lang)
if err != nil {
slog.Error("recognize dish", "err", err)
writeErrorJSON(w, http.StatusServiceUnavailable, "recognition failed, please try again")

2
backend/internal/domain/user/repository.go
View File

@@ -112,7 +112,7 @@ func buildUpdateQuery(id string, req UpdateProfileRequest) (string, []interface{
argIdx++
}
if req.Preferences != nil {
setClauses = append(setClauses, fmt.Sprintf("preferences = $%d", argIdx))
setClauses = append(setClauses, fmt.Sprintf("preferences = preferences || $%d::jsonb", argIdx))
args = append(args, string(*req.Preferences))
argIdx++
}