feat: add import-off make target for OpenFoodFacts import

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
dbastrikin
2026-03-21 13:07:56 +02:00
parent 31a4179ac6
commit fb4fe54090
2 changed files with 17 additions and 1 deletions

View File

@@ -1,4 +1,4 @@
.PHONY: run run-worker-paid run-worker-free dev dev-infra-up dev-infra-down test test-integration lint migrate-up migrate-down migrate-create migrate-status docker-up docker-down docker-logs docker-logs-worker .PHONY: run run-worker-paid run-worker-free dev dev-infra-up dev-infra-down test test-integration lint migrate-up migrate-down migrate-create migrate-status docker-up docker-down docker-logs docker-logs-worker import-off
ifneq (,$(wildcard .env)) ifneq (,$(wildcard .env))
include .env include .env
@@ -54,6 +54,17 @@ migrate-create:
migrate-status: migrate-status:
goose -dir migrations postgres "$(DATABASE_URL)" status goose -dir migrations postgres "$(DATABASE_URL)" status
# OpenFoodFacts import
# Usage: make import-off file=openfoodfacts-products.jsonl.gz
# Optional: make import-off file=... limit=1000 batch=500 min-scans=1
import-off:
go run ./cmd/importoff \
-file=$(file) \
-dsn="$(DATABASE_URL)" \
$(if $(limit),-limit=$(limit)) \
$(if $(batch),-batch=$(batch)) \
$(if $(min-scans),-min-scans=$(min-scans))
# Docker # Docker
docker-up: docker-up:
docker compose up -d docker compose up -d

View File

@@ -116,6 +116,7 @@ make run-worker-free # Воркер free-очереди
| `make migrate-down` | Откатить последнюю миграцию | | `make migrate-down` | Откатить последнюю миграцию |
| `make migrate-status` | Статус миграций | | `make migrate-status` | Статус миграций |
| `make migrate-create name=<name>` | Создать новую миграцию | | `make migrate-create name=<name>` | Создать новую миграцию |
| `make import-off file=<path>` | Импортировать каталог из OpenFoodFacts JSONL/GZ |
## Импорт каталога продуктов ## Импорт каталога продуктов
@@ -125,6 +126,10 @@ make run-worker-free # Воркер free-очереди
# Скачать дамп (~66 ГБ, ~4.4 млн записей) # Скачать дамп (~66 ГБ, ~4.4 млн записей)
# https://world.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz # https://world.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz
# Через make (DSN берётся из .env)
make import-off file=openfoodfacts-products.jsonl.gz
# Или напрямую с дополнительными флагами
go run ./cmd/importoff \ go run ./cmd/importoff \
-file=openfoodfacts-products.jsonl.gz \ -file=openfoodfacts-products.jsonl.gz \
-dsn="$DATABASE_URL" \ -dsn="$DATABASE_URL" \