From 5bf8a012ce03eb02fe1a647d350f9e254f6660df Mon Sep 17 00:00:00 2001 From: BXY0414 <1079823339@qq.com> Date: Fri, 13 Mar 2026 18:55:33 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0docker=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 9 ++ DOCKER_PREFECT_USAGE.md | 86 +++++++++++++++ Dockerfile | 19 ++++ docker-compose.yaml | 13 +++ docker-entrypoint.sh | 19 ++++ prefect_gmv_scheduler.py | 228 +++++++++++++++++++++++++++++++++++++++ requirements-docker.txt | 4 + 7 files changed, 378 insertions(+) create mode 100644 .dockerignore create mode 100644 DOCKER_PREFECT_USAGE.md create mode 100644 Dockerfile create mode 100644 docker-compose.yaml create mode 100644 docker-entrypoint.sh create mode 100644 prefect_gmv_scheduler.py create mode 100644 requirements-docker.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..248ee96 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.gitignore +.vscode +__pycache__ +*.pyc +*.pyo +*.pyd +.DS_Store +project.zip diff --git a/DOCKER_PREFECT_USAGE.md b/DOCKER_PREFECT_USAGE.md new file mode 100644 index 0000000..5dfe212 --- /dev/null +++ b/DOCKER_PREFECT_USAGE.md @@ -0,0 +1,86 @@ +# Docker + Prefect + +## 1. 构建镜像 + +```bash +cd /Users/idss/YXT_Project/gmv-analytics +docker build -t gmv-analytics-prefect:latest . +``` + +## 2. 单次执行 + +```bash +docker run --rm \ + --env-file /Users/idss/YXT_Project/gmv-analytics/.env \ + -e PROJECT_MAPPING_FILE=/app/店铺映射表.xlsx \ + gmv-analytics-prefect:latest \ + python prefect_gmv_scheduler.py +``` + +## 3. 启动每天 09:00 的 Prefect 调度 + +这会连接 `.env` 中已经配置的 `PREFECT_API_URL`，并在 Prefect 中启动一个 deployment。 + +```bash +docker run --rm \ + --env-file /Users/idss/YXT_Project/gmv-analytics/.env \ + -e PROJECT_MAPPING_FILE=/app/店铺映射表.xlsx \ + gmv-analytics-prefect:latest +``` + +容器默认执行： + +```bash +python prefect_gmv_scheduler.py --serve +``` + +也可以直接用 Compose： + +```bash +cd /Users/idss/YXT_Project/gmv-analytics +docker compose up -d --build +``` + +停止： + +```bash +docker compose down +``` + +## 4. 如果你是用 Prefect worker 跑容器 + +先保证你的 Prefect worker 已经连到同一个 `PREFECT_API_URL`。 + +推荐命令示例： + +```bash +prefect worker start --pool default-agent-pool --type process +``` + +如果你后面要改成真正的 Docker work pool，再在 Prefect UI 中把 deployment 的基础设施切到 Docker，并指定镜像： + +```text +gmv-analytics-prefect:latest +``` + +## 5. 常用覆盖参数 + +单次 dry-run： + +```bash +docker run --rm \ + --env-file /Users/idss/YXT_Project/gmv-analytics/.env \ + -e PROJECT_MAPPING_FILE=/app/店铺映射表.xlsx \ + gmv-analytics-prefect:latest \ + python prefect_gmv_scheduler.py --dry-run +``` + +指定时间范围： + +```bash +docker run --rm \ + --env-file /Users/idss/YXT_Project/gmv-analytics/.env \ + -e PROJECT_MAPPING_FILE=/app/店铺映射表.xlsx \ + gmv-analytics-prefect:latest \ + python prefect_gmv_scheduler.py --start-date 2026-03-01 --end-date 2026-03-12 +``` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9196a85 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + TZ=Asia/Shanghai + +WORKDIR /app + +COPY requirements-docker.txt /app/requirements-docker.txt + +RUN pip install --no-cache-dir --upgrade pip \ + && pip install --no-cache-dir -r /app/requirements-docker.txt + +COPY . /app + +RUN chmod +x /app/docker-entrypoint.sh + +ENTRYPOINT ["/app/docker-entrypoint.sh"] +CMD ["python", "prefect_gmv_scheduler.py", "--serve"] diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..01bc4d6 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + gmv-prefect-scheduler: + build: + context: . + dockerfile: Dockerfile + image: gmv-analytics-prefect:latest + container_name: gmv-prefect-scheduler + restart: unless-stopped + env_file: + - .env + environment: + PROJECT_MAPPING_FILE: /app/店铺映射表.xlsx + command: ["python", "prefect_gmv_scheduler.py", "--serve"] diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 0000000..9755dfa --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu + +APP_DIR="/app" +DEFAULT_MAPPING_FILE="$APP_DIR/店铺映射表.xlsx" + +if [ -f "$APP_DIR/.env" ]; then + set -a + . "$APP_DIR/.env" + set +a +fi + +if [ -f "$DEFAULT_MAPPING_FILE" ]; then + if [ -z "${PROJECT_MAPPING_FILE:-}" ] || [ ! -f "${PROJECT_MAPPING_FILE}" ]; then + export PROJECT_MAPPING_FILE="$DEFAULT_MAPPING_FILE" + fi +fi + +exec "$@" diff --git a/prefect_gmv_scheduler.py b/prefect_gmv_scheduler.py new file mode 100644 index 0000000..b4d3265 --- /dev/null +++ b/prefect_gmv_scheduler.py @@ -0,0 +1,228 @@ +import argparse +import os +from dataclasses import dataclass +from datetime import date +from pathlib import Path + + +def bootstrap_prefect_env() -> None: + env_file = Path(__file__).resolve().parent / ".env" + if not env_file.exists(): + return + for raw_line in env_file.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + if key.startswith("PREFECT_"): + os.environ.setdefault(key, value.strip()) + + +bootstrap_prefect_env() + +from prefect import flow, task +from prefect.logging import get_run_logger + +from project_bitable_common import ( + get_base_url, + get_default_end_date, + get_default_start_date, + get_feishu_chat_id, + get_feishu_report_image_path, + get_group_webhook_url, + get_mapping_file, + get_store_field_name, + iter_dates, + load_dotenv, + log_step, + run_gmv_sync, + run_metric_sync, + send_feishu_app_card, + send_feishu_sync_card, + validate_date, +) + + +@dataclass(frozen=True) +class SyncConfig: + start_date: str + end_date: str + base_url: str + mapping_file: str + store_field_name: str + chat_id: str + report_image_path: str + group_webhook: str + table_map_file: str + dry_run: bool + skip_missing_tables: bool + + +def load_sync_config( + start_date: str | None = None, + end_date: str | None = None, + dry_run: bool = False, + skip_missing_tables: bool = False, + table_map_file: str = "", +) -> SyncConfig: + load_dotenv() + return SyncConfig( + start_date=start_date or get_default_start_date(), + end_date=end_date or get_default_end_date(), + base_url=get_base_url(), + mapping_file=str(get_mapping_file()), + store_field_name=get_store_field_name(), + chat_id=get_feishu_chat_id(), + report_image_path=get_feishu_report_image_path(), + group_webhook=get_group_webhook_url(), + table_map_file=table_map_file, + dry_run=dry_run, + skip_missing_tables=skip_missing_tables, + ) + + +@task(name="sync-sales") +def sync_sales_for_day(stat_date_text: str, config: SyncConfig) -> None: + stat_date = validate_date(stat_date_text) + run_metric_sync( + metric_name="sales", + start_date=stat_date, + end_date=stat_date, + base_url=config.base_url, + mapping_file=Path(config.mapping_file), + store_field_name=config.store_field_name, + table_map_file=config.table_map_file, + dry_run=config.dry_run, + skip_missing_tables=config.skip_missing_tables, + ) + + +@task(name="sync-refund") +def sync_refund_for_day(stat_date_text: str, config: SyncConfig) -> None: + stat_date = validate_date(stat_date_text) + run_metric_sync( + metric_name="refund", + start_date=stat_date, + end_date=stat_date, + base_url=config.base_url, + mapping_file=Path(config.mapping_file), + store_field_name=config.store_field_name, + table_map_file=config.table_map_file, + dry_run=config.dry_run, + skip_missing_tables=config.skip_missing_tables, + ) + + +@task(name="sync-gmv") +def sync_gmv_for_day(stat_date_text: str, config: SyncConfig) -> None: + stat_date = validate_date(stat_date_text) + run_gmv_sync( + start_date=stat_date, + end_date=stat_date, + base_url=config.base_url, + store_field_name=config.store_field_name, + dry_run=config.dry_run, + skip_missing_tables=config.skip_missing_tables, + ) + + +@task(name="notify-feishu") +def notify_feishu(title: str, config: SyncConfig) -> None: + if config.dry_run: + log_step("dry-run 模式下跳过飞书群推送") + return + start_date = validate_date(config.start_date) + end_date = validate_date(config.end_date) + if config.chat_id: + send_feishu_app_card( + chat_id=config.chat_id, + title=title, + base_url=config.base_url, + start_date=start_date, + end_date=end_date, + image_path=config.report_image_path, + ) + return + send_feishu_sync_card( + webhook_url=config.group_webhook, + title=title, + base_url=config.base_url, + start_date=start_date, + end_date=end_date, + ) + + +@flow(name="gmv-daily-sync") +def gmv_daily_sync_flow( + start_date: str | None = None, + end_date: str | None = None, + dry_run: bool = False, + skip_missing_tables: bool = False, + table_map_file: str = "", +) -> None: + logger = get_run_logger() + config = load_sync_config( + start_date=start_date, + end_date=end_date, + dry_run=dry_run, + skip_missing_tables=skip_missing_tables, + table_map_file=table_map_file, + ) + + start_date_value = validate_date(config.start_date) + end_date_value = validate_date(config.end_date) + if end_date_value < start_date_value: + raise ValueError("end-date 不能早于 start-date") + + logger.info("Prefect 调度开始: %s -> %s", config.start_date, config.end_date) + try: + for stat_date in iter_dates(start_date_value, end_date_value): + stat_date_text = stat_date.isoformat() + logger.info("开始处理日期: %s", stat_date_text) + sync_sales_for_day(stat_date_text, config) + sync_refund_for_day(stat_date_text, config) + sync_gmv_for_day(stat_date_text, config) + logger.info("日期处理完成: %s", stat_date_text) + except Exception: + notify_feishu.with_options(name="notify-feishu-failed")("GMV数据同步失败", config) + raise + + notify_feishu("GMV数据同步", config) + logger.info("Prefect 调度完成") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Prefect 调度：每天上午 9 点执行 GMV 数据同步。") + parser.add_argument("--start-date", default="", help="开始日期，格式 YYYY-MM-DD，默认读取 .env 中的 SYNC_START_DATE") + parser.add_argument("--end-date", default="", help="结束日期，格式 YYYY-MM-DD，默认昨天") + parser.add_argument("--table-map-file", default="", help="可选，项目到表名的映射 JSON 文件路径") + parser.add_argument("--dry-run", action="store_true", help="只输出将要更新的内容，不调用飞书写接口") + parser.add_argument("--skip-missing-tables", action="store_true", help="找不到对应月份的飞书表时跳过") + parser.add_argument("--serve", action="store_true", help="启动 Prefect deployment，并按每天上午 9 点定时执行") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + flow_kwargs = { + "start_date": args.start_date or None, + "end_date": args.end_date or None, + "dry_run": args.dry_run, + "skip_missing_tables": args.skip_missing_tables, + "table_map_file": args.table_map_file, + } + if args.serve: + gmv_daily_sync_flow.serve( + name="gmv-daily-sync-9am", + cron="0 9 * * *", + timezone="Asia/Shanghai", + parameters=flow_kwargs, + tags=["gmv", "feishu", "bitable"], + ) + return + gmv_daily_sync_flow(**flow_kwargs) + + +if __name__ == "__main__": + main() diff --git a/requirements-docker.txt b/requirements-docker.txt new file mode 100644 index 0000000..98de86d --- /dev/null +++ b/requirements-docker.txt @@ -0,0 +1,4 @@ +openpyxl>=3.1.2 +prefect>=3.0.0 +psycopg2-binary>=2.9.9 +requests>=2.31.0