feat: add import-off make target for OpenFoodFacts import
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,4 @@
|
|||||||
.PHONY: run run-worker-paid run-worker-free dev dev-infra-up dev-infra-down test test-integration lint migrate-up migrate-down migrate-create migrate-status docker-up docker-down docker-logs docker-logs-worker
|
.PHONY: run run-worker-paid run-worker-free dev dev-infra-up dev-infra-down test test-integration lint migrate-up migrate-down migrate-create migrate-status docker-up docker-down docker-logs docker-logs-worker import-off
|
||||||
|
|
||||||
ifneq (,$(wildcard .env))
|
ifneq (,$(wildcard .env))
|
||||||
include .env
|
include .env
|
||||||
@@ -54,6 +54,17 @@ migrate-create:
|
|||||||
migrate-status:
|
migrate-status:
|
||||||
goose -dir migrations postgres "$(DATABASE_URL)" status
|
goose -dir migrations postgres "$(DATABASE_URL)" status
|
||||||
|
|
||||||
|
# OpenFoodFacts import
|
||||||
|
# Usage: make import-off file=openfoodfacts-products.jsonl.gz
|
||||||
|
# Optional: make import-off file=... limit=1000 batch=500 min-scans=1
|
||||||
|
import-off:
|
||||||
|
go run ./cmd/importoff \
|
||||||
|
-file=$(file) \
|
||||||
|
-dsn="$(DATABASE_URL)" \
|
||||||
|
$(if $(limit),-limit=$(limit)) \
|
||||||
|
$(if $(batch),-batch=$(batch)) \
|
||||||
|
$(if $(min-scans),-min-scans=$(min-scans))
|
||||||
|
|
||||||
# Docker
|
# Docker
|
||||||
docker-up:
|
docker-up:
|
||||||
docker compose up -d
|
docker compose up -d
|
||||||
|
|||||||
@@ -116,6 +116,7 @@ make run-worker-free # Воркер free-очереди
|
|||||||
| `make migrate-down` | Откатить последнюю миграцию |
|
| `make migrate-down` | Откатить последнюю миграцию |
|
||||||
| `make migrate-status` | Статус миграций |
|
| `make migrate-status` | Статус миграций |
|
||||||
| `make migrate-create name=<name>` | Создать новую миграцию |
|
| `make migrate-create name=<name>` | Создать новую миграцию |
|
||||||
|
| `make import-off file=<path>` | Импортировать каталог из OpenFoodFacts JSONL/GZ |
|
||||||
|
|
||||||
## Импорт каталога продуктов
|
## Импорт каталога продуктов
|
||||||
|
|
||||||
@@ -125,6 +126,10 @@ make run-worker-free # Воркер free-очереди
|
|||||||
# Скачать дамп (~66 ГБ, ~4.4 млн записей)
|
# Скачать дамп (~66 ГБ, ~4.4 млн записей)
|
||||||
# https://world.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz
|
# https://world.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz
|
||||||
|
|
||||||
|
# Через make (DSN берётся из .env)
|
||||||
|
make import-off file=openfoodfacts-products.jsonl.gz
|
||||||
|
|
||||||
|
# Или напрямую с дополнительными флагами
|
||||||
go run ./cmd/importoff \
|
go run ./cmd/importoff \
|
||||||
-file=openfoodfacts-products.jsonl.gz \
|
-file=openfoodfacts-products.jsonl.gz \
|
||||||
-dsn="$DATABASE_URL" \
|
-dsn="$DATABASE_URL" \
|
||||||
|
|||||||
Reference in New Issue
Block a user