diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1367188 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +.vscode/ +.env diff --git a/generate_project_gmv_csv.py b/generate_project_gmv_csv.py index f4bdb17..8700314 100644 --- a/generate_project_gmv_csv.py +++ b/generate_project_gmv_csv.py @@ -1,10 +1,10 @@ import argparse from project_bitable_common import ( - DEFAULT_END_DATE, - DEFAULT_START_DATE, - DEFAULT_STORE_FIELD_NAME, + get_default_end_date, + get_default_start_date, get_base_url, + get_store_field_name, load_dotenv, log_step, run_gmv_sync, @@ -14,10 +14,10 @@ from project_bitable_common import ( def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="从飞书多维表格读取销售额和退款额,计算 GMV 并写回。") - parser.add_argument("--start-date", default=DEFAULT_START_DATE, help="开始日期,格式 YYYY-MM-DD") - parser.add_argument("--end-date", default=DEFAULT_END_DATE, help="结束日期,格式 YYYY-MM-DD") + parser.add_argument("--start-date", default=get_default_start_date(), help="开始日期,格式 YYYY-MM-DD") + parser.add_argument("--end-date", default=get_default_end_date(), help="结束日期,格式 YYYY-MM-DD,默认昨天") parser.add_argument("--base-url", default=get_base_url(), help="飞书多维表格地址") - parser.add_argument("--store-field-name", default=DEFAULT_STORE_FIELD_NAME, help="飞书中的店铺字段名") + parser.add_argument("--store-field-name", default=get_store_field_name(), help="飞书中的店铺字段名") parser.add_argument("--dry-run", action="store_true", help="只输出将要更新的内容,不调用飞书写接口") parser.add_argument("--skip-missing-tables", action="store_true", help="找不到对应月份的飞书表时跳过") return parser.parse_args() diff --git a/generate_project_refund_csv.py b/generate_project_refund_csv.py index 38baa3f..5ed63f8 100644 --- a/generate_project_refund_csv.py +++ b/generate_project_refund_csv.py @@ -2,11 +2,11 @@ import argparse from pathlib import Path from project_bitable_common import ( - DEFAULT_END_DATE, - DEFAULT_START_DATE, - DEFAULT_STORE_FIELD_NAME, + get_default_end_date, + get_default_start_date, get_base_url, get_mapping_file, + get_store_field_name, load_dotenv, log_step, run_metric_sync, @@ -16,11 +16,11 @@ from project_bitable_common import ( def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="从 PostgreSQL 拉取退款额并写入飞书多维表格。") - parser.add_argument("--start-date", default=DEFAULT_START_DATE, help="开始日期,格式 YYYY-MM-DD") - parser.add_argument("--end-date", default=DEFAULT_END_DATE, help="结束日期,格式 YYYY-MM-DD") + parser.add_argument("--start-date", default=get_default_start_date(), help="开始日期,格式 YYYY-MM-DD") + parser.add_argument("--end-date", default=get_default_end_date(), help="结束日期,格式 YYYY-MM-DD,默认昨天") parser.add_argument("--base-url", default=get_base_url(), help="飞书多维表格地址") parser.add_argument("--mapping-file", default=str(get_mapping_file()), help="店铺映射表 xlsx") - parser.add_argument("--store-field-name", default=DEFAULT_STORE_FIELD_NAME, help="飞书中的店铺字段名") + parser.add_argument("--store-field-name", default=get_store_field_name(), help="飞书中的店铺字段名") parser.add_argument("--table-map-file", default="", help="可选,项目到表名的映射 JSON 文件路径") parser.add_argument("--dry-run", action="store_true", help="只输出将要更新的内容,不调用飞书写接口") parser.add_argument("--skip-missing-tables", action="store_true", help="找不到对应月份的飞书表时跳过") diff --git a/generate_project_sales_csv.py b/generate_project_sales_csv.py index 3a652b0..3d6771e 100644 --- a/generate_project_sales_csv.py +++ b/generate_project_sales_csv.py @@ -2,11 +2,11 @@ import argparse from pathlib import Path from project_bitable_common import ( - DEFAULT_END_DATE, - DEFAULT_START_DATE, - DEFAULT_STORE_FIELD_NAME, + get_default_end_date, + get_default_start_date, get_base_url, get_mapping_file, + get_store_field_name, load_dotenv, log_step, run_metric_sync, @@ -16,11 +16,11 @@ from project_bitable_common import ( def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="从 PostgreSQL 拉取销售额并写入飞书多维表格。") - parser.add_argument("--start-date", default=DEFAULT_START_DATE, help="开始日期,格式 YYYY-MM-DD") - parser.add_argument("--end-date", default=DEFAULT_END_DATE, help="结束日期,格式 YYYY-MM-DD") + parser.add_argument("--start-date", default=get_default_start_date(), help="开始日期,格式 YYYY-MM-DD") + parser.add_argument("--end-date", default=get_default_end_date(), help="结束日期,格式 YYYY-MM-DD,默认昨天") parser.add_argument("--base-url", default=get_base_url(), help="飞书多维表格地址") parser.add_argument("--mapping-file", default=str(get_mapping_file()), help="店铺映射表 xlsx") - parser.add_argument("--store-field-name", default=DEFAULT_STORE_FIELD_NAME, help="飞书中的店铺字段名") + parser.add_argument("--store-field-name", default=get_store_field_name(), help="飞书中的店铺字段名") parser.add_argument("--table-map-file", default="", help="可选,项目到表名的映射 JSON 文件路径") parser.add_argument("--dry-run", action="store_true", help="只输出将要更新的内容,不调用飞书写接口") parser.add_argument("--skip-missing-tables", action="store_true", help="找不到对应月份的飞书表时跳过") diff --git a/project_bitable_common.py b/project_bitable_common.py index aca42cc..9f12c57 100644 --- a/project_bitable_common.py +++ b/project_bitable_common.py @@ -17,9 +17,8 @@ from openpyxl import load_workbook BASE_DIR = Path(__file__).resolve().parent ENV_FILE = BASE_DIR / ".env" DEFAULT_MAPPING_FILE = BASE_DIR / "店铺映射表.xlsx" -DEFAULT_START_DATE = "2026-03-01" -DEFAULT_END_DATE = "2026-03-01" -DEFAULT_STORE_FIELD_NAME = "业务店铺名称" +DEFAULT_START_DATE = "2026-02-25" +DEFAULT_STORE_FIELD_NAME = "店铺名称" STORE_FIELD_CANDIDATES = ["业务店铺名称", "店铺名称", "门店名称", "店铺", "名称"] SUPPORTED_PROJECTS = ["大眼睛", "羽心堂", "MMMJ", "EM", "春回盏", "BROKEN ZOO"] @@ -278,6 +277,42 @@ def get_mapping_file() -> Path: return Path(os.getenv("PROJECT_MAPPING_FILE", str(DEFAULT_MAPPING_FILE))) +def get_store_field_name() -> str: + return os.getenv("FEISHU_STORE_FIELD_NAME", DEFAULT_STORE_FIELD_NAME) + + +def get_default_start_date() -> str: + return os.getenv("SYNC_START_DATE", DEFAULT_START_DATE) + + +def get_default_end_date() -> str: + return (date.today() - timedelta(days=1)).isoformat() + + +def get_group_webhook_url() -> str: + candidates = [ + "FEISHU_BOT_WEBHOOK", + "FEISHU_ROBOT_WEBHOOK", + "FEISHU_WEBHOOK", + "LARK_BOT_WEBHOOK", + "LARK_ROBOT_WEBHOOK", + "GROUP_WEBHOOK", + ] + for key in candidates: + value = os.getenv(key) + if value: + return value + return "" + + +def get_feishu_chat_id() -> str: + return os.getenv("FEISHU_CHAT_ID", "") + + +def get_feishu_report_image_path() -> str: + return os.getenv("FEISHU_REPORT_IMAGE_PATH", "") + + def resolve_store_field_name(fields: list[dict[str, Any]], preferred_field_name: str) -> str: field_names = [str(item.get("field_name") or "") for item in fields] if preferred_field_name in field_names: @@ -330,7 +365,15 @@ def load_store_mapping(path: Path) -> dict[str, str]: wb = load_workbook(path, data_only=True) ws = wb.active mapping: dict[str, str] = {} - for business_name, bi_name in ws.iter_rows(min_row=2, values_only=True): + for row in ws.iter_rows(min_row=2, values_only=True): + if not row: + continue + if len(row) >= 3: + _, business_name, bi_name = row[:3] + elif len(row) >= 2: + business_name, bi_name = row[:2] + else: + continue if not business_name or not bi_name: continue mapping[str(bi_name).strip()] = str(business_name).strip() @@ -610,6 +653,146 @@ def load_table_name_overrides(path_text: str) -> dict[str, str]: return {str(key): str(value) for key, value in payload.items()} +def send_feishu_sync_card( + webhook_url: str, + title: str, + base_url: str, + start_date: date, + end_date: date, +) -> None: + if not webhook_url: + log_step("未配置飞书群机器人 webhook,跳过群消息推送") + return + card = { + "msg_type": "interactive", + "card": { + "config": {"wide_screen_mode": True}, + "header": { + "template": "blue", + "title": {"tag": "plain_text", "content": title}, + }, + "elements": [ + { + "tag": "div", + "text": { + "tag": "lark_md", + "content": ( + f"**同步范围**\n{start_date.isoformat()} 至 {end_date.isoformat()}\n\n" + f"**多维表格**\n[点击打开多维表格]({base_url})" + ), + }, + } + ], + }, + } + log_step("开始推送飞书群卡片消息") + resp = requests.post(webhook_url, json=card, timeout=30) + resp.raise_for_status() + payload = resp.json() + if payload.get("code") not in (0, None): + raise RuntimeError(f"飞书群消息推送失败: {payload}") + log_step("飞书群卡片消息推送完成") + + +def upload_feishu_image(image_path: str) -> str: + if not image_path: + return "" + path = Path(image_path) + if not path.exists(): + raise FileNotFoundError(f"未找到预览图文件: {path}") + token = get_feishu_token() + headers = {"Authorization": f"Bearer {token}"} + with path.open("rb") as image_file: + files = {"image": (path.name, image_file, "application/octet-stream")} + data = {"image_type": "message"} + log_step(f"开始上传飞书消息图片: {path}") + resp = requests.post("https://open.feishu.cn/open-apis/im/v1/images", headers=headers, data=data, files=files, timeout=60) + resp.raise_for_status() + payload = resp.json() + if payload.get("code") != 0: + raise RuntimeError(f"飞书图片上传失败: {payload}") + image_key = ((payload.get("data") or {}).get("image_key")) or "" + if not image_key: + raise RuntimeError(f"飞书图片上传成功但未返回 image_key: {payload}") + log_step(f"飞书消息图片上传完成: image_key={image_key}") + return image_key + + +def send_feishu_app_card( + chat_id: str, + title: str, + base_url: str, + start_date: date, + end_date: date, + image_path: str = "", +) -> None: + if not chat_id: + log_step("未配置 FEISHU_CHAT_ID,跳过应用机器人卡片推送") + return + token = get_feishu_token() + image_key = upload_feishu_image(image_path) if image_path else "" + elements: list[dict[str, Any]] = [ + { + "tag": "div", + "text": { + "tag": "lark_md", + "content": ( + f"**同步范围**\n{start_date.isoformat()} 至 {end_date.isoformat()}\n\n" + f"**多维表格**\n[{title}]({base_url})" + ), + }, + } + ] + if image_key: + elements.append({"tag": "img", "img_key": image_key, "alt": {"tag": "plain_text", "content": "报表预览图"}}) + elements.append( + { + "tag": "action", + "actions": [ + { + "tag": "button", + "text": {"tag": "plain_text", "content": "打开仪表盘"}, + "type": "primary", + "url": base_url, + } + ], + } + ) + message = { + "receive_id": chat_id, + "msg_type": "interactive", + "content": json.dumps( + { + "config": {"wide_screen_mode": True}, + "header": {"template": "blue", "title": {"tag": "plain_text", "content": title}}, + "elements": elements, + }, + ensure_ascii=False, + ), + } + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json; charset=utf-8", + } + log_step(f"开始通过应用机器人发送群卡片: chat_id={chat_id}") + resp = requests.post( + "https://open.feishu.cn/open-apis/im/v1/messages?receive_id_type=chat_id", + headers=headers, + json=message, + timeout=60, + ) + try: + payload = resp.json() + except ValueError: + resp.raise_for_status() + raise RuntimeError(f"应用机器人发送群卡片失败,返回非 JSON: status={resp.status_code}, body={resp.text[:500]}") + if resp.status_code >= 400: + raise RuntimeError(f"应用机器人发送群卡片失败: status={resp.status_code}, payload={payload}") + if payload.get("code") != 0: + raise RuntimeError(f"应用机器人发送群卡片失败: {payload}") + log_step("应用机器人群卡片发送完成") + + def find_target_table( tables: list[TableInfo], project: str, @@ -916,4 +1099,3 @@ def run_gmv_sync( preferred_store_field_name=store_field_name, dry_run=dry_run, ) - diff --git a/run_project_bitable_pipeline.py b/run_project_bitable_pipeline.py index 4008f22..e486900 100644 --- a/run_project_bitable_pipeline.py +++ b/run_project_bitable_pipeline.py @@ -2,26 +2,35 @@ import argparse from pathlib import Path from project_bitable_common import ( - DEFAULT_END_DATE, - DEFAULT_START_DATE, - DEFAULT_STORE_FIELD_NAME, + get_default_end_date, + get_default_start_date, get_base_url, + get_feishu_chat_id, + get_feishu_report_image_path, + get_group_webhook_url, get_mapping_file, + get_store_field_name, + iter_dates, load_dotenv, log_step, run_gmv_sync, run_metric_sync, + send_feishu_app_card, + send_feishu_sync_card, validate_date, ) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="按 销售 -> 退款 -> GMV 顺序执行飞书多维表格同步。") - parser.add_argument("--start-date", default=DEFAULT_START_DATE, help="开始日期,格式 YYYY-MM-DD") - parser.add_argument("--end-date", default=DEFAULT_END_DATE, help="结束日期,格式 YYYY-MM-DD") + parser.add_argument("--start-date", default=get_default_start_date(), help="开始日期,格式 YYYY-MM-DD") + parser.add_argument("--end-date", default=get_default_end_date(), help="结束日期,格式 YYYY-MM-DD,默认昨天") parser.add_argument("--base-url", default=get_base_url(), help="飞书多维表格地址") parser.add_argument("--mapping-file", default=str(get_mapping_file()), help="店铺映射表 xlsx") - parser.add_argument("--store-field-name", default=DEFAULT_STORE_FIELD_NAME, help="飞书中的店铺字段名") + parser.add_argument("--store-field-name", default=get_store_field_name(), help="飞书中的店铺字段名") + parser.add_argument("--chat-id", default=get_feishu_chat_id(), help="飞书群 chat_id") + parser.add_argument("--report-image-path", default=get_feishu_report_image_path(), help="飞书卡片预览图本地路径") + parser.add_argument("--group-webhook", default=get_group_webhook_url(), help="飞书群机器人 webhook") parser.add_argument("--table-map-file", default="", help="可选,项目到表名的映射 JSON 文件路径") parser.add_argument("--dry-run", action="store_true", help="只输出将要更新的内容,不调用飞书写接口") parser.add_argument("--skip-missing-tables", action="store_true", help="找不到对应月份的飞书表时跳过") @@ -36,37 +45,58 @@ def main() -> None: if end_date < start_date: raise ValueError("end-date 不能早于 start-date") - log_step("流水线开始: 销售 -> 退款 -> GMV") - run_metric_sync( - metric_name="sales", - start_date=start_date, - end_date=end_date, - base_url=args.base_url, - mapping_file=Path(args.mapping_file), - store_field_name=args.store_field_name, - table_map_file=args.table_map_file, - dry_run=args.dry_run, - skip_missing_tables=args.skip_missing_tables, - ) - run_metric_sync( - metric_name="refund", - start_date=start_date, - end_date=end_date, - base_url=args.base_url, - mapping_file=Path(args.mapping_file), - store_field_name=args.store_field_name, - table_map_file=args.table_map_file, - dry_run=args.dry_run, - skip_missing_tables=args.skip_missing_tables, - ) - run_gmv_sync( - start_date=start_date, - end_date=end_date, - base_url=args.base_url, - store_field_name=args.store_field_name, - dry_run=args.dry_run, - skip_missing_tables=args.skip_missing_tables, - ) + log_step("流水线开始: 按天执行 销售 -> 退款 -> GMV") + for stat_date in iter_dates(start_date, end_date): + log_step(f"开始处理日期: {stat_date.isoformat()}") + run_metric_sync( + metric_name="sales", + start_date=stat_date, + end_date=stat_date, + base_url=args.base_url, + mapping_file=Path(args.mapping_file), + store_field_name=args.store_field_name, + table_map_file=args.table_map_file, + dry_run=args.dry_run, + skip_missing_tables=args.skip_missing_tables, + ) + run_metric_sync( + metric_name="refund", + start_date=stat_date, + end_date=stat_date, + base_url=args.base_url, + mapping_file=Path(args.mapping_file), + store_field_name=args.store_field_name, + table_map_file=args.table_map_file, + dry_run=args.dry_run, + skip_missing_tables=args.skip_missing_tables, + ) + run_gmv_sync( + start_date=stat_date, + end_date=stat_date, + base_url=args.base_url, + store_field_name=args.store_field_name, + dry_run=args.dry_run, + skip_missing_tables=args.skip_missing_tables, + ) + log_step(f"日期处理完成: {stat_date.isoformat()}") + if not args.dry_run: + if args.chat_id: + send_feishu_app_card( + chat_id=args.chat_id, + title="GMV数据同步", + base_url=args.base_url, + start_date=start_date, + end_date=end_date, + image_path=args.report_image_path, + ) + else: + send_feishu_sync_card( + webhook_url=args.group_webhook, + title="GMV数据同步", + base_url=args.base_url, + start_date=start_date, + end_date=end_date, + ) log_step("流水线完成") diff --git a/店铺映射表.xlsx b/店铺映射表.xlsx index 5af1e52..b30ff3e 100644 Binary files a/店铺映射表.xlsx and b/店铺映射表.xlsx differ