Prefect 同步跳过缺失表并推送飞书提醒
This commit is contained in:
@@ -39,6 +39,7 @@ from project_bitable_common import (
|
||||
load_dotenv,
|
||||
log_step,
|
||||
run_daily_sync_for_date,
|
||||
send_missing_table_notification,
|
||||
send_sync_notification,
|
||||
validate_date,
|
||||
)
|
||||
@@ -72,7 +73,7 @@ def load_sync_config(
|
||||
start_date: str | None = None,
|
||||
end_date: str | None = None,
|
||||
dry_run: bool = False,
|
||||
skip_missing_tables: bool = False,
|
||||
skip_missing_tables: bool = True,
|
||||
table_map_file: str = "",
|
||||
) -> SyncConfig:
|
||||
load_dotenv()
|
||||
@@ -174,7 +175,7 @@ def gmv_daily_data_sync_flow(
|
||||
start_date: str | None = None,
|
||||
end_date: str | None = None,
|
||||
dry_run: bool = False,
|
||||
skip_missing_tables: bool = False,
|
||||
skip_missing_tables: bool = True,
|
||||
table_map_file: str = "",
|
||||
) -> None:
|
||||
config = load_sync_config(
|
||||
@@ -205,6 +206,14 @@ def gmv_daily_data_sync_flow(
|
||||
run_daily_sync_for_date(context, stat_date)
|
||||
if not config.dry_run:
|
||||
mark_sync_window_completed(sync_window)
|
||||
if context.missing_tables:
|
||||
log_step(f"Prefect 数据同步发现缺失表: {len(context.missing_tables)} 个")
|
||||
send_missing_table_notification(
|
||||
context,
|
||||
context.missing_tables,
|
||||
sync_window.start_date,
|
||||
sync_window.end_date,
|
||||
)
|
||||
log_step("Prefect 数据同步完成")
|
||||
|
||||
|
||||
@@ -213,7 +222,7 @@ def gmv_daily_notify_flow(
|
||||
start_date: str | None = None,
|
||||
end_date: str | None = None,
|
||||
dry_run: bool = False,
|
||||
skip_missing_tables: bool = False,
|
||||
skip_missing_tables: bool = True,
|
||||
table_map_file: str = "",
|
||||
) -> None:
|
||||
config = load_sync_config(
|
||||
@@ -235,7 +244,12 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--end-date", default="", help="结束日期,格式 YYYY-MM-DD,默认昨天")
|
||||
parser.add_argument("--table-map-file", default="", help="可选,项目到表名的映射 JSON 文件路径")
|
||||
parser.add_argument("--dry-run", action="store_true", help="只输出将要更新的内容,不调用飞书写接口")
|
||||
parser.add_argument("--skip-missing-tables", action="store_true", help="找不到对应月份的飞书表时跳过")
|
||||
parser.add_argument(
|
||||
"--skip-missing-tables",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=True,
|
||||
help="找不到对应月份的飞书表时跳过并在同步结束后推送提醒(默认开启)",
|
||||
)
|
||||
parser.add_argument("--serve", action="store_true", help="启动 Prefect deployments,按 8 点同步、9 点通知 定时执行")
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
|
||||
Reference in New Issue
Block a user