920 lines
35 KiB
Python
920 lines
35 KiB
Python
import json
|
|
import os
|
|
import re
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass
|
|
from datetime import date, datetime, timedelta
|
|
from decimal import Decimal
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
|
|
import psycopg2
|
|
import requests
|
|
from openpyxl import load_workbook
|
|
|
|
|
|
BASE_DIR = Path(__file__).resolve().parent
|
|
ENV_FILE = BASE_DIR / ".env"
|
|
DEFAULT_MAPPING_FILE = BASE_DIR / "店铺映射表.xlsx"
|
|
DEFAULT_START_DATE = "2026-03-01"
|
|
DEFAULT_END_DATE = "2026-03-01"
|
|
DEFAULT_STORE_FIELD_NAME = "业务店铺名称"
|
|
STORE_FIELD_CANDIDATES = ["业务店铺名称", "店铺名称", "门店名称", "店铺", "名称"]
|
|
SUPPORTED_PROJECTS = ["大眼睛", "羽心堂", "MMMJ", "EM", "春回盏", "BROKEN ZOO"]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MetricConfig:
|
|
name: str
|
|
description: str
|
|
sql_template: str
|
|
field_suffix: str
|
|
|
|
|
|
METRIC_CONFIGS = {
|
|
"sales": MetricConfig(
|
|
name="sales",
|
|
description="销售额",
|
|
field_suffix="总销售额",
|
|
sql_template="""
|
|
SET TIMEZONE = 'Asia/Shanghai';
|
|
|
|
WITH base_data AS (
|
|
SELECT
|
|
f.支付时间::DATE AS 统计日期,
|
|
f.店铺主键,
|
|
s.店铺名称 AS 原始店铺名称,
|
|
p.品牌名称,
|
|
f.业务模式,
|
|
f.实付分摊金额 AS 指标金额
|
|
FROM analytics_layer.fact_sales f
|
|
LEFT JOIN analytics_layer.dim_shop s
|
|
ON f.店铺主键 = s.店铺主键
|
|
LEFT JOIN analytics_layer.dim_product p
|
|
ON f.商品主键 = p.商品主键
|
|
AND f.统一规格编码 = p.统一规格编码
|
|
WHERE f.支付时间 >= %(start_ts)s
|
|
AND f.支付时间 < %(end_ts_exclusive)s
|
|
),
|
|
brand_split AS (
|
|
SELECT
|
|
统计日期,
|
|
店铺主键,
|
|
原始店铺名称,
|
|
业务模式,
|
|
CASE
|
|
WHEN 店铺主键 IN (1,28,29) AND 品牌名称 LIKE '羽心堂'
|
|
THEN 原始店铺名称 || '_羽心堂'
|
|
WHEN 店铺主键 IN (1,28,29) AND 品牌名称 LIKE '春回盏'
|
|
THEN 原始店铺名称 || '_春回盏'
|
|
WHEN 店铺主键 IN (1,28,29) AND 品牌名称 LIKE 'Mama Mujer'
|
|
THEN 原始店铺名称 || '_Mama Mujer'
|
|
WHEN 店铺主键 IN (1,28,29) AND 品牌名称 LIKE 'ERBA MIGINA'
|
|
THEN 原始店铺名称 || '_ERBA MIGINA'
|
|
WHEN 店铺主键 IN (1,28,29) AND 品牌名称 LIKE 'BROKEN ZOO'
|
|
THEN 原始店铺名称 || '_BROKEN ZOO'
|
|
WHEN 店铺主键 IN (1,28,29)
|
|
THEN 原始店铺名称 || '_其他'
|
|
WHEN 店铺主键 IN (3,9) AND 品牌名称 LIKE '羽心堂'
|
|
THEN 原始店铺名称 || '_羽心堂'
|
|
WHEN 店铺主键 IN (3,9) AND 品牌名称 LIKE 'Mama Mujer'
|
|
THEN 原始店铺名称 || '_Mama Mujer'
|
|
WHEN 店铺主键 IN (3,9)
|
|
THEN 原始店铺名称 || '_其他'
|
|
WHEN 店铺主键 IS NULL OR 原始店铺名称 IS NULL
|
|
THEN '分销/SEEDING'
|
|
ELSE 原始店铺名称
|
|
END AS 店铺名称,
|
|
指标金额
|
|
FROM base_data
|
|
),
|
|
shop_brand_detail AS (
|
|
SELECT
|
|
统计日期,
|
|
店铺主键,
|
|
店铺名称,
|
|
SUM(指标金额) AS 指标值
|
|
FROM brand_split
|
|
GROUP BY 1,2,3
|
|
)
|
|
SELECT
|
|
统计日期,
|
|
店铺名称,
|
|
指标值
|
|
FROM shop_brand_detail
|
|
WHERE NOT (
|
|
店铺主键 IN (1,9,28,29)
|
|
AND 店铺名称 LIKE '%%_其他'
|
|
)
|
|
|
|
UNION ALL
|
|
|
|
SELECT
|
|
统计日期,
|
|
原始店铺名称 AS 店铺名称,
|
|
SUM(指标金额)
|
|
FROM base_data
|
|
WHERE 店铺主键 IN (1,28,29,3)
|
|
GROUP BY 1,2
|
|
|
|
UNION ALL
|
|
|
|
SELECT
|
|
统计日期,
|
|
CASE
|
|
WHEN 业务模式 IN ('三方计算','三方分销')
|
|
THEN 原始店铺名称 || '_三方'
|
|
ELSE 原始店铺名称 || '_自营'
|
|
END AS 店铺名称,
|
|
SUM(指标金额)
|
|
FROM base_data
|
|
WHERE 店铺主键 = 1
|
|
GROUP BY 1,2
|
|
|
|
ORDER BY 统计日期 DESC, 指标值 DESC;
|
|
""",
|
|
),
|
|
"refund": MetricConfig(
|
|
name="refund",
|
|
description="退款额",
|
|
field_suffix="退款额",
|
|
sql_template="""
|
|
SET TIMEZONE = 'Asia/Shanghai';
|
|
|
|
WITH base_data AS (
|
|
SELECT
|
|
f.完结时间::DATE AS 统计日期,
|
|
f.店铺主键,
|
|
s.店铺名称 AS 原始店铺名称,
|
|
p.品牌名称,
|
|
f.退款金额 AS 指标金额
|
|
FROM analytics_layer.fact_returns f
|
|
LEFT JOIN analytics_layer.dim_shop s
|
|
ON f.店铺主键 = s.店铺主键
|
|
LEFT JOIN analytics_layer.dim_product p
|
|
ON f.商品主键 = p.商品主键
|
|
AND f.统一规格编码 = p.统一规格编码
|
|
WHERE f.完结时间 >= %(start_ts)s
|
|
AND f.完结时间 < %(end_ts_exclusive)s
|
|
),
|
|
brand_split AS (
|
|
SELECT
|
|
统计日期,
|
|
店铺主键,
|
|
原始店铺名称,
|
|
CASE
|
|
WHEN 店铺主键 IN (1,28,29) AND 品牌名称 LIKE '羽心堂'
|
|
THEN 原始店铺名称 || '_羽心堂'
|
|
WHEN 店铺主键 IN (1,28,29) AND 品牌名称 LIKE '春回盏'
|
|
THEN 原始店铺名称 || '_春回盏'
|
|
WHEN 店铺主键 IN (1,28,29) AND 品牌名称 LIKE 'Mama Mujer'
|
|
THEN 原始店铺名称 || '_Mama Mujer'
|
|
WHEN 店铺主键 IN (1,28,29) AND 品牌名称 LIKE 'ERBA MIGINA'
|
|
THEN 原始店铺名称 || '_ERBA MIGINA'
|
|
WHEN 店铺主键 IN (1,28,29) AND 品牌名称 LIKE 'BROKEN ZOO'
|
|
THEN 原始店铺名称 || '_BROKEN ZOO'
|
|
WHEN 店铺主键 IN (1,28,29)
|
|
THEN 原始店铺名称 || '_其他'
|
|
WHEN 店铺主键 IN (3,9) AND 品牌名称 LIKE '羽心堂'
|
|
THEN 原始店铺名称 || '_羽心堂'
|
|
WHEN 店铺主键 IN (3,9) AND 品牌名称 LIKE 'Mama Mujer'
|
|
THEN 原始店铺名称 || '_Mama Mujer'
|
|
WHEN 店铺主键 IN (3,9)
|
|
THEN 原始店铺名称 || '_其他'
|
|
WHEN 店铺主键 IS NULL OR 原始店铺名称 IS NULL
|
|
THEN '分销/SEEDING'
|
|
ELSE 原始店铺名称
|
|
END AS 店铺名称,
|
|
指标金额
|
|
FROM base_data
|
|
),
|
|
shop_brand_detail AS (
|
|
SELECT
|
|
统计日期,
|
|
店铺主键,
|
|
店铺名称,
|
|
SUM(指标金额) AS 指标值
|
|
FROM brand_split
|
|
GROUP BY 1,2,3
|
|
)
|
|
SELECT
|
|
统计日期,
|
|
店铺名称,
|
|
指标值
|
|
FROM shop_brand_detail
|
|
WHERE NOT (
|
|
店铺主键 IN (1,9,28,29)
|
|
AND 店铺名称 LIKE '%%_其他'
|
|
)
|
|
|
|
UNION ALL
|
|
|
|
SELECT
|
|
统计日期,
|
|
原始店铺名称 AS 店铺名称,
|
|
SUM(指标金额)
|
|
FROM base_data
|
|
WHERE 店铺主键 IN (1,28,29,3)
|
|
GROUP BY 1,2
|
|
|
|
ORDER BY 统计日期 DESC, 指标值 DESC;
|
|
""",
|
|
),
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MetricRow:
|
|
stat_date: date
|
|
bi_store_name: str
|
|
amount: Decimal
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TableInfo:
|
|
table_id: str
|
|
name: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SyncResult:
|
|
sql_only_stores: list[str]
|
|
feishu_only_stores: list[str]
|
|
|
|
|
|
def log_step(message: str) -> None:
|
|
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
print(f"[{now}] {message}", flush=True)
|
|
|
|
|
|
def preview_list(values: list[Any], limit: int = 20) -> str:
|
|
if len(values) <= limit:
|
|
return str(values)
|
|
return f"{values[:limit]} ... 共{len(values)}项"
|
|
|
|
|
|
def load_dotenv(env_file: Path = ENV_FILE) -> None:
|
|
if not env_file.exists():
|
|
log_step(f"未找到 .env 文件，跳过加载: {env_file}")
|
|
return
|
|
log_step(f"加载环境变量: {env_file}")
|
|
for raw_line in env_file.read_text(encoding="utf-8").splitlines():
|
|
line = raw_line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
os.environ.setdefault(key.strip(), value.strip())
|
|
|
|
|
|
def get_base_url() -> str:
|
|
base_url = os.getenv("FEISHU_BASE_URL")
|
|
if not base_url:
|
|
raise ValueError("缺少环境变量 FEISHU_BASE_URL")
|
|
return base_url
|
|
|
|
|
|
def get_mapping_file() -> Path:
|
|
return Path(os.getenv("PROJECT_MAPPING_FILE", str(DEFAULT_MAPPING_FILE)))
|
|
|
|
|
|
def resolve_store_field_name(fields: list[dict[str, Any]], preferred_field_name: str) -> str:
|
|
field_names = [str(item.get("field_name") or "") for item in fields]
|
|
if preferred_field_name in field_names:
|
|
return preferred_field_name
|
|
for candidate in STORE_FIELD_CANDIDATES:
|
|
if candidate in field_names:
|
|
return candidate
|
|
raise ValueError(f"未找到店铺名称字段: preferred={preferred_field_name}, 可用字段={field_names}")
|
|
|
|
|
|
def validate_date(value: str) -> date:
|
|
return datetime.strptime(value, "%Y-%m-%d").date()
|
|
|
|
|
|
def normalize_text(value: str) -> str:
|
|
return re.sub(r"[\s_\-（）()]+", "", value or "").lower()
|
|
|
|
|
|
def classify_project(store_name: str) -> str:
|
|
name = store_name or ""
|
|
prioritized_rules = [
|
|
("MMMJ", ["Mama", "mmmj", "MMMJ"]),
|
|
("羽心堂", ["羽心堂"]),
|
|
("EM", ["EM", "ERBA MIGINA"]),
|
|
("春回盏", ["春回盏"]),
|
|
("BROKEN ZOO", ["BROKEN ZOO"]),
|
|
]
|
|
for project, keywords in prioritized_rules:
|
|
if any(keyword in name for keyword in keywords):
|
|
return project
|
|
if "大眼睛" in name or "大西瓜籽妈妈" in name:
|
|
return "大眼睛"
|
|
return ""
|
|
|
|
|
|
def get_project_table_aliases(project: str) -> list[str]:
|
|
aliases = {
|
|
"大眼睛": ["大眼睛", "大眼睛买买买"],
|
|
"羽心堂": ["羽心堂"],
|
|
"MMMJ": ["MMMJ", "Mama Mujer", "MAMA MUJER"],
|
|
"EM": ["EM", "ERBA MIGINA"],
|
|
"春回盏": ["春回盏"],
|
|
"BROKEN ZOO": ["BROKEN ZOO", "BROKENZOO", "葆福颂"],
|
|
}
|
|
return aliases.get(project, [project])
|
|
|
|
|
|
def load_store_mapping(path: Path) -> dict[str, str]:
|
|
log_step(f"加载店铺映射表: {path}")
|
|
wb = load_workbook(path, data_only=True)
|
|
ws = wb.active
|
|
mapping: dict[str, str] = {}
|
|
for business_name, bi_name in ws.iter_rows(min_row=2, values_only=True):
|
|
if not business_name or not bi_name:
|
|
continue
|
|
mapping[str(bi_name).strip()] = str(business_name).strip()
|
|
log_step(f"店铺映射加载完成: {len(mapping)} 条")
|
|
return mapping
|
|
|
|
|
|
def resolve_business_store_name(bi_store_name: str, store_mapping: dict[str, str]) -> str:
|
|
return store_mapping.get(bi_store_name, "")
|
|
|
|
|
|
def get_db_connection() -> psycopg2.extensions.connection:
|
|
required = [
|
|
"ANALYTICS_DB_HOST",
|
|
"ANALYTICS_DB_PORT",
|
|
"ANALYTICS_DB_USER",
|
|
"ANALYTICS_DB_PASSWORD",
|
|
"ANALYTICS_DB_NAME",
|
|
]
|
|
missing = [key for key in required if not os.getenv(key)]
|
|
if missing:
|
|
raise ValueError(f"缺少数据库环境变量: {', '.join(missing)}")
|
|
log_step(
|
|
f"连接 PostgreSQL: host={os.environ['ANALYTICS_DB_HOST']} port={os.environ['ANALYTICS_DB_PORT']} db={os.environ['ANALYTICS_DB_NAME']}"
|
|
)
|
|
return psycopg2.connect(
|
|
host=os.environ["ANALYTICS_DB_HOST"],
|
|
port=os.environ["ANALYTICS_DB_PORT"],
|
|
user=os.environ["ANALYTICS_DB_USER"],
|
|
password=os.environ["ANALYTICS_DB_PASSWORD"],
|
|
dbname=os.environ["ANALYTICS_DB_NAME"],
|
|
)
|
|
|
|
|
|
def fetch_metric_rows(start_date: date, end_date: date, metric_config: MetricConfig) -> list[MetricRow]:
|
|
start_ts = f"{start_date.isoformat()} 00:00:00"
|
|
end_ts_exclusive = f"{(end_date + timedelta(days=1)).isoformat()} 00:00:00"
|
|
log_step(f"开始查询{metric_config.description}: {start_ts} -> {end_ts_exclusive} (右开区间)")
|
|
with get_db_connection() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(metric_config.sql_template, {"start_ts": start_ts, "end_ts_exclusive": end_ts_exclusive})
|
|
rows = cur.fetchall()
|
|
log_step(f"{metric_config.description}查询完成: {len(rows)} 行")
|
|
return [
|
|
MetricRow(stat_date=row[0], bi_store_name=(row[1] or "").strip(), amount=row[2] or Decimal("0"))
|
|
for row in rows
|
|
]
|
|
|
|
|
|
def build_project_metrics(
|
|
rows: list[MetricRow],
|
|
store_mapping: dict[str, str],
|
|
metric_config: MetricConfig,
|
|
) -> dict[str, dict[tuple[int, int], dict[date, dict[str, Decimal]]]]:
|
|
log_step(f"开始按项目/月/日期聚合{metric_config.description}")
|
|
grouped: dict[str, dict[tuple[int, int], dict[date, dict[str, Decimal]]]] = defaultdict(
|
|
lambda: defaultdict(lambda: defaultdict(dict))
|
|
)
|
|
for row in rows:
|
|
business_store_name = resolve_business_store_name(row.bi_store_name, store_mapping)
|
|
if not business_store_name:
|
|
continue
|
|
project = classify_project(business_store_name)
|
|
if not project:
|
|
continue
|
|
month_key = (row.stat_date.year, row.stat_date.month)
|
|
current_amount = grouped[project][month_key][row.stat_date].get(business_store_name, Decimal("0"))
|
|
grouped[project][month_key][row.stat_date][business_store_name] = current_amount + row.amount
|
|
project_count = len(grouped)
|
|
month_count = sum(len(months) for months in grouped.values())
|
|
log_step(f"{metric_config.description}聚合完成: 项目={project_count}，项目月份组合={month_count}")
|
|
return grouped
|
|
|
|
|
|
def build_target_project_months(
|
|
start_date: date,
|
|
end_date: date,
|
|
grouped_metrics: dict[str, dict[tuple[int, int], dict[date, dict[str, Decimal]]]],
|
|
) -> dict[str, dict[tuple[int, int], dict[date, dict[str, Decimal]]]]:
|
|
result = {
|
|
project: {month_key: dict(date_map) for month_key, date_map in month_dict.items()}
|
|
for project, month_dict in grouped_metrics.items()
|
|
}
|
|
current = start_date
|
|
while current <= end_date:
|
|
month_key = (current.year, current.month)
|
|
for project in SUPPORTED_PROJECTS:
|
|
project_months = result.setdefault(project, {})
|
|
metrics_by_date = project_months.setdefault(month_key, {})
|
|
metrics_by_date.setdefault(current, {})
|
|
current += timedelta(days=1)
|
|
return result
|
|
|
|
|
|
def get_feishu_token() -> str:
|
|
app_id = os.getenv("FEISHU_APP_ID")
|
|
app_secret = os.getenv("FEISHU_APP_SECRET")
|
|
if not app_id or not app_secret:
|
|
raise ValueError("缺少飞书环境变量 FEISHU_APP_ID / FEISHU_APP_SECRET")
|
|
log_step("开始获取飞书 tenant_access_token")
|
|
resp = requests.post(
|
|
"https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal",
|
|
json={"app_id": app_id, "app_secret": app_secret},
|
|
timeout=30,
|
|
)
|
|
resp.raise_for_status()
|
|
payload = resp.json()
|
|
if payload.get("code") != 0:
|
|
raise RuntimeError(f"获取飞书 token 失败: {payload}")
|
|
log_step("飞书 tenant_access_token 获取成功")
|
|
return payload["tenant_access_token"]
|
|
|
|
|
|
class FeishuBitableClient:
|
|
def __init__(self, base_url: str) -> None:
|
|
self.base_url = base_url
|
|
self.app_token = self.extract_app_token(base_url)
|
|
log_step(f"解析飞书 Base URL 成功: app_token={self.app_token}")
|
|
self.token = get_feishu_token()
|
|
self._fields_cache: dict[str, list[dict[str, Any]]] = {}
|
|
self.session = requests.Session()
|
|
self.session.headers.update(
|
|
{
|
|
"Authorization": f"Bearer {self.token}",
|
|
"Content-Type": "application/json; charset=utf-8",
|
|
}
|
|
)
|
|
|
|
@staticmethod
|
|
def extract_app_token(base_url: str) -> str:
|
|
path = urlparse(base_url).path.strip("/")
|
|
match = re.search(r"base/([^/]+)", path)
|
|
if not match:
|
|
raise ValueError(f"无法从链接中解析 app_token: {base_url}")
|
|
return match.group(1)
|
|
|
|
def request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]:
|
|
url = f"https://open.feishu.cn{path}"
|
|
params = kwargs.get("params")
|
|
if params:
|
|
log_step(f"请求飞书接口: {method} {path} params={params}")
|
|
else:
|
|
log_step(f"请求飞书接口: {method} {path}")
|
|
resp = self.session.request(method, url, timeout=60, **kwargs)
|
|
try:
|
|
payload = resp.json()
|
|
except ValueError:
|
|
resp.raise_for_status()
|
|
raise RuntimeError(f"飞书接口返回了非 JSON 响应: status={resp.status_code}, body={resp.text[:500]}")
|
|
if resp.status_code >= 400:
|
|
error = payload.get("error") or {}
|
|
raise RuntimeError(
|
|
f"飞书接口 HTTP {resp.status_code} {path}: code={payload.get('code')} msg={payload.get('msg')} "
|
|
f"log_id={error.get('log_id', '')} payload={payload}"
|
|
)
|
|
if payload.get("code") != 0:
|
|
error = payload.get("error") or {}
|
|
raise RuntimeError(
|
|
f"飞书接口失败 {path}: code={payload.get('code')} msg={payload.get('msg')} "
|
|
f"log_id={error.get('log_id', '')} payload={payload}"
|
|
)
|
|
return payload.get("data") or {}
|
|
|
|
def list_tables(self) -> list[TableInfo]:
|
|
log_step("开始获取多维表格数据表列表")
|
|
items: list[TableInfo] = []
|
|
page_token = ""
|
|
while True:
|
|
params: dict[str, Any] = {}
|
|
if page_token:
|
|
params["page_token"] = page_token
|
|
data = self.request("GET", f"/open-apis/bitable/v1/apps/{self.app_token}/tables", params=params)
|
|
for item in data.get("items") or []:
|
|
items.append(TableInfo(table_id=item["table_id"], name=item["name"]))
|
|
if not data.get("has_more", False):
|
|
break
|
|
page_token = data.get("page_token") or ""
|
|
if not page_token:
|
|
break
|
|
log_step(f"飞书返回的数据表名称: {preview_list([item.name for item in items])}")
|
|
log_step(f"数据表列表获取完成: {len(items)} 个")
|
|
return items
|
|
|
|
def list_fields(self, table_id: str) -> list[dict[str, Any]]:
|
|
if table_id in self._fields_cache:
|
|
log_step(f"命中字段缓存: table_id={table_id}, 字段数={len(self._fields_cache[table_id])}")
|
|
return self._fields_cache[table_id]
|
|
log_step(f"开始获取字段列表: table_id={table_id}")
|
|
items: list[dict[str, Any]] = []
|
|
page_token = ""
|
|
seen_tokens: set[str] = set()
|
|
page_index = 0
|
|
while True:
|
|
page_index += 1
|
|
params = {"page_size": 100}
|
|
if page_token:
|
|
params["page_token"] = page_token
|
|
if page_token in seen_tokens:
|
|
raise RuntimeError(f"字段列表分页出现重复 page_token，已中断: table_id={table_id}, page_token={page_token}")
|
|
seen_tokens.add(page_token)
|
|
log_step(f"获取字段列表分页: table_id={table_id}, page={page_index}, page_token={page_token or '<first>'}")
|
|
data = self.request(
|
|
"GET",
|
|
f"/open-apis/bitable/v1/apps/{self.app_token}/tables/{table_id}/fields",
|
|
params=params,
|
|
)
|
|
items.extend(data.get("items") or [])
|
|
has_more = data.get("has_more", False)
|
|
next_page_token = data.get("page_token") or ""
|
|
log_step(
|
|
f"字段列表分页完成: table_id={table_id}, page={page_index}, 累计字段数={len(items)}, "
|
|
f"has_more={has_more}, next_page_token={next_page_token or '<empty>'}"
|
|
)
|
|
if not has_more:
|
|
break
|
|
if not next_page_token:
|
|
raise RuntimeError(f"字段列表分页返回 has_more=True 但 page_token 为空: table_id={table_id}, 已获取字段数={len(items)}")
|
|
page_token = next_page_token
|
|
if page_index >= 100:
|
|
raise RuntimeError(f"字段列表分页超过 100 页，已中断: table_id={table_id}")
|
|
self._fields_cache[table_id] = items
|
|
log_step(f"飞书返回的字段名称: table_id={table_id}, fields={preview_list([item.get('field_name', '') for item in items])}")
|
|
log_step(f"字段列表获取完成: table_id={table_id}, 字段数={len(items)}")
|
|
return items
|
|
|
|
def ensure_number_field(self, table_id: str, field_name: str) -> None:
|
|
log_step(f"检查字段是否存在: table_id={table_id}, field={field_name}")
|
|
fields = self.list_fields(table_id)
|
|
if any(item.get("field_name") == field_name for item in fields):
|
|
log_step(f"字段已存在，跳过创建: table_id={table_id}, field={field_name}")
|
|
return
|
|
log_step(f"开始创建数字字段: table_id={table_id}, field={field_name}")
|
|
self.request(
|
|
"POST",
|
|
f"/open-apis/bitable/v1/apps/{self.app_token}/tables/{table_id}/fields",
|
|
json={"field_name": field_name, "type": 2},
|
|
)
|
|
self._fields_cache.pop(table_id, None)
|
|
log_step(f"字段创建完成: table_id={table_id}, field={field_name}")
|
|
|
|
def list_records(self, table_id: str) -> list[dict[str, Any]]:
|
|
log_step(f"开始获取记录列表: table_id={table_id}")
|
|
items: list[dict[str, Any]] = []
|
|
page_token = ""
|
|
while True:
|
|
params = {"page_size": 500}
|
|
if page_token:
|
|
params["page_token"] = page_token
|
|
data = self.request(
|
|
"GET",
|
|
f"/open-apis/bitable/v1/apps/{self.app_token}/tables/{table_id}/records",
|
|
params=params,
|
|
)
|
|
items.extend(data.get("items") or [])
|
|
page_token = data.get("page_token") or ""
|
|
if not page_token:
|
|
break
|
|
log_step(f"记录列表获取完成: table_id={table_id}, 记录数={len(items)}")
|
|
return items
|
|
|
|
def update_record(self, table_id: str, record_id: str, fields: dict[str, Any]) -> None:
|
|
log_step(f"更新记录: table_id={table_id}, record_id={record_id}, fields={list(fields.keys())}")
|
|
self.request(
|
|
"PUT",
|
|
f"/open-apis/bitable/v1/apps/{self.app_token}/tables/{table_id}/records/{record_id}",
|
|
json={"fields": fields},
|
|
)
|
|
|
|
|
|
def load_table_name_overrides(path_text: str) -> dict[str, str]:
|
|
if not path_text:
|
|
return {}
|
|
path = Path(path_text)
|
|
if not path.exists():
|
|
raise FileNotFoundError(f"未找到表名映射文件: {path}")
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
return {str(key): str(value) for key, value in payload.items()}
|
|
|
|
|
|
def find_target_table(
|
|
tables: list[TableInfo],
|
|
project: str,
|
|
year: int,
|
|
month: int,
|
|
overrides: dict[str, str] | None = None,
|
|
) -> TableInfo:
|
|
log_step(f"开始匹配目标表: 项目={project}, 月份={year:04d}-{month:02d}")
|
|
overrides = overrides or {}
|
|
override_key = f"{project}:{year:04d}-{month:02d}"
|
|
if override_key in overrides:
|
|
target_name = overrides[override_key]
|
|
for table in tables:
|
|
if table.name == target_name:
|
|
log_step(f"命中表名映射: {override_key} -> {table.name}")
|
|
return table
|
|
raise ValueError(f"表名映射命中了 {target_name}，但飞书中不存在该表")
|
|
normalized_projects = [normalize_text(alias) for alias in get_project_table_aliases(project)]
|
|
month_tokens = {
|
|
str(month),
|
|
f"{month}月",
|
|
f"{month:02d}月",
|
|
f"{year:04d}{month:02d}",
|
|
f"{year:04d}年{month:02d}月",
|
|
f"{year:04d}-{month:02d}",
|
|
f"{year:04d}_{month:02d}",
|
|
f"{year:04d}.{month:02d}",
|
|
}
|
|
log_step(f"项目表名别名: 项目={project}, aliases={get_project_table_aliases(project)}")
|
|
candidates = []
|
|
project_related_tables = []
|
|
for table in tables:
|
|
normalized_name = normalize_text(table.name)
|
|
matched_project = any(alias in normalized_name for alias in normalized_projects)
|
|
if matched_project:
|
|
project_related_tables.append(table.name)
|
|
if not matched_project:
|
|
continue
|
|
if any(normalize_text(token) in normalized_name for token in month_tokens):
|
|
candidates.append(table)
|
|
if len(candidates) == 1:
|
|
log_step(f"匹配到目标表: 项目={project}, 月份={year:04d}-{month:02d}, 表={candidates[0].name}")
|
|
return candidates[0]
|
|
if not candidates:
|
|
all_table_names = [table.name for table in tables]
|
|
raise ValueError(
|
|
f"未找到匹配表: 项目={project}, 月份={year:04d}-{month:02d}, 项目相关表={project_related_tables or '[]'}, 全部表={all_table_names}"
|
|
)
|
|
raise ValueError(
|
|
f"匹配到多个表，请提供 table_map_file 指定: 项目={project}, 月份={year:04d}-{month:02d}, 表={', '.join(t.name for t in candidates)}"
|
|
)
|
|
|
|
|
|
def decimal_to_number(value: Decimal) -> float:
|
|
return float(value.quantize(Decimal("0.01")))
|
|
|
|
|
|
def get_metric_field_name(stat_date: date, metric_config: MetricConfig) -> str:
|
|
return stat_date.strftime(f"%m-%d{metric_config.field_suffix}")
|
|
|
|
|
|
def parse_number(value: Any) -> float | None:
|
|
if value in (None, ""):
|
|
return None
|
|
if isinstance(value, bool):
|
|
return None
|
|
if isinstance(value, (int, float)):
|
|
return round(float(value), 2)
|
|
if isinstance(value, str):
|
|
text = value.strip().replace(",", "")
|
|
if not text:
|
|
return None
|
|
try:
|
|
return round(float(text), 2)
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def iter_dates(start_date: date, end_date: date) -> list[date]:
|
|
dates = []
|
|
current = start_date
|
|
while current <= end_date:
|
|
dates.append(current)
|
|
current = date.fromordinal(current.toordinal() + 1)
|
|
return dates
|
|
|
|
|
|
def sync_metric_table_for_month(
|
|
client: FeishuBitableClient,
|
|
table: TableInfo,
|
|
store_field_name: str,
|
|
metrics_by_date: dict[date, dict[str, Decimal]],
|
|
dry_run: bool,
|
|
metric_config: MetricConfig,
|
|
) -> SyncResult:
|
|
log_step(f"开始同步数据表: {table.name}")
|
|
table_fields = client.list_fields(table.table_id)
|
|
store_field_name = resolve_store_field_name(table_fields, store_field_name)
|
|
log_step(f"识别到店铺字段: 表={table.name}, field={store_field_name}")
|
|
records = client.list_records(table.table_id)
|
|
record_by_store: dict[str, dict[str, Any]] = {}
|
|
preview_store_fields: list[str] = []
|
|
for item in records:
|
|
fields = item.get("fields", {})
|
|
store_name = fields.get(store_field_name)
|
|
if isinstance(store_name, list):
|
|
store_name = store_name[0] if store_name else ""
|
|
if not store_name:
|
|
continue
|
|
normalized_store_name = str(store_name).strip()
|
|
normalized_store_key = normalize_text(normalized_store_name)
|
|
preview_store_fields.append(normalized_store_name)
|
|
record_by_store[normalized_store_key] = item
|
|
log_step(f"飞书返回的记录店铺名称预览: table_id={table.table_id}, stores={preview_list(preview_store_fields)}")
|
|
|
|
feishu_store_names = set(record_by_store)
|
|
sql_store_names: set[str] = set()
|
|
sql_store_names_raw: set[str] = set()
|
|
for store_metrics in metrics_by_date.values():
|
|
sql_store_names_raw.update(store_metrics.keys())
|
|
sql_store_names.update(normalize_text(name) for name in store_metrics.keys())
|
|
all_store_names = feishu_store_names | sql_store_names
|
|
|
|
sql_name_map = {normalize_text(name): name for name in sql_store_names_raw}
|
|
feishu_name_map = {normalize_text(name): name for name in preview_store_fields}
|
|
sql_only_stores = sorted(sql_name_map[name] for name in (sql_store_names - feishu_store_names))
|
|
feishu_only_stores = sorted(feishu_name_map[name] for name in (feishu_store_names - sql_store_names))
|
|
|
|
log_step(f"SQL 聚合出的店铺名称: 表={table.name}, stores={preview_list(sorted(sql_store_names_raw))}")
|
|
log_step(f"待处理店铺数: 表={table.name}, 店铺数={len(all_store_names)}, 日期数={len(metrics_by_date)}")
|
|
log_step(
|
|
f"店铺匹配情况: 表={table.name}, SQL有但飞书无={preview_list(sql_only_stores)}, "
|
|
f"飞书有但SQL无={preview_list(feishu_only_stores)}"
|
|
)
|
|
|
|
for stat_date in sorted(metrics_by_date):
|
|
field_name = get_metric_field_name(stat_date, metric_config)
|
|
if dry_run:
|
|
print(f"[DRY-RUN] 表 {table.name} 将新增字段: {field_name}")
|
|
else:
|
|
client.ensure_number_field(table.table_id, field_name)
|
|
for store_key in sorted(all_store_names):
|
|
store_name = sql_name_map.get(store_key) or feishu_name_map.get(store_key) or store_key
|
|
amount = None
|
|
for raw_name, raw_amount in metrics_by_date[stat_date].items():
|
|
if normalize_text(raw_name) == store_key:
|
|
amount = raw_amount
|
|
break
|
|
field_value = decimal_to_number(amount) if amount is not None else None
|
|
existing = record_by_store.get(store_key)
|
|
if dry_run:
|
|
print(
|
|
f"[DRY-RUN] 表={table.name} 店铺={store_name} 字段={field_name} 值={field_value if field_value is not None else ''}"
|
|
)
|
|
continue
|
|
if existing:
|
|
client.update_record(table.table_id, existing["record_id"], {field_name: field_value})
|
|
continue
|
|
log_step(f"跳过新增记录: 表={table.name}, 店铺={store_name}, 字段={field_name}, 原因=飞书中不存在该店铺行")
|
|
log_step(f"数据表同步完成: {table.name}")
|
|
return SyncResult(sql_only_stores=sql_only_stores, feishu_only_stores=feishu_only_stores)
|
|
|
|
|
|
def run_metric_sync(
|
|
metric_name: str,
|
|
start_date: date,
|
|
end_date: date,
|
|
base_url: str,
|
|
mapping_file: Path,
|
|
store_field_name: str,
|
|
table_map_file: str,
|
|
dry_run: bool,
|
|
skip_missing_tables: bool,
|
|
) -> None:
|
|
metric_config = METRIC_CONFIGS[metric_name]
|
|
mapping = load_store_mapping(mapping_file)
|
|
metric_rows = fetch_metric_rows(start_date, end_date, metric_config)
|
|
grouped_metrics = build_project_metrics(metric_rows, mapping, metric_config)
|
|
grouped_metrics = build_target_project_months(start_date, end_date, grouped_metrics)
|
|
table_overrides = load_table_name_overrides(table_map_file)
|
|
if table_overrides:
|
|
log_step(f"加载表名映射完成: {len(table_overrides)} 条")
|
|
client = FeishuBitableClient(base_url)
|
|
tables = client.list_tables()
|
|
unmatched_summary: list[tuple[str, str, str, SyncResult]] = []
|
|
for project in sorted(grouped_metrics):
|
|
for (year, month), metrics_by_date in sorted(grouped_metrics[project].items()):
|
|
try:
|
|
table = find_target_table(tables, project, year, month, table_overrides)
|
|
except ValueError as exc:
|
|
if not skip_missing_tables:
|
|
raise
|
|
log_step(f"跳过: {exc}")
|
|
continue
|
|
log_step(f"同步项目={project} 月份={year:04d}-{month:02d} -> 表={table.name}")
|
|
sync_result = sync_metric_table_for_month(
|
|
client=client,
|
|
table=table,
|
|
store_field_name=store_field_name,
|
|
metrics_by_date=metrics_by_date,
|
|
dry_run=dry_run,
|
|
metric_config=metric_config,
|
|
)
|
|
unmatched_summary.append((project, f"{year:04d}-{month:02d}", table.name, sync_result))
|
|
log_step("未匹配店铺汇总开始")
|
|
for project, month, table_name, result in unmatched_summary:
|
|
log_step(
|
|
f"未匹配店铺汇总: 项目={project}, 月份={month}, 表={table_name}, "
|
|
f"SQL有但飞书无={result.sql_only_stores}, 飞书有但SQL无={result.feishu_only_stores}"
|
|
)
|
|
|
|
|
|
def get_sales_field_name(stat_date: date) -> str:
|
|
return get_metric_field_name(stat_date, METRIC_CONFIGS["sales"])
|
|
|
|
|
|
def get_refund_field_name(stat_date: date) -> str:
|
|
return get_metric_field_name(stat_date, METRIC_CONFIGS["refund"])
|
|
|
|
|
|
def get_gmv_field_name(stat_date: date) -> str:
|
|
return stat_date.strftime("%m-%d GMV")
|
|
|
|
|
|
def sync_gmv_table_for_day(
|
|
client: FeishuBitableClient,
|
|
table: TableInfo,
|
|
stat_date: date,
|
|
preferred_store_field_name: str,
|
|
dry_run: bool,
|
|
) -> None:
|
|
log_step(f"开始计算 GMV: 表={table.name}, 日期={stat_date.isoformat()}")
|
|
fields = client.list_fields(table.table_id)
|
|
store_field_name = resolve_store_field_name(fields, preferred_store_field_name)
|
|
sales_field_name = get_sales_field_name(stat_date)
|
|
refund_field_name = get_refund_field_name(stat_date)
|
|
gmv_field_name = get_gmv_field_name(stat_date)
|
|
field_names = [str(item.get("field_name") or "") for item in fields]
|
|
log_step(
|
|
f"GMV 字段检查: 表={table.name}, 店铺字段={store_field_name}, 销售字段={sales_field_name in field_names}, "
|
|
f"退款字段={refund_field_name in field_names}, GMV字段={gmv_field_name in field_names}"
|
|
)
|
|
if sales_field_name not in field_names or refund_field_name not in field_names:
|
|
log_step(f"跳过 GMV 计算: 表={table.name}, 原因=缺少销售额或退款额字段")
|
|
return
|
|
if not dry_run:
|
|
client.ensure_number_field(table.table_id, gmv_field_name)
|
|
records = client.list_records(table.table_id)
|
|
preview_stores = []
|
|
updated_count = 0
|
|
skipped_empty_count = 0
|
|
for item in records:
|
|
record_id = item["record_id"]
|
|
record_fields = item.get("fields", {})
|
|
store_name = record_fields.get(store_field_name)
|
|
if isinstance(store_name, list):
|
|
store_name = store_name[0] if store_name else ""
|
|
store_name = str(store_name or "").strip()
|
|
if store_name:
|
|
preview_stores.append(store_name)
|
|
sales_value = parse_number(record_fields.get(sales_field_name))
|
|
refund_value = parse_number(record_fields.get(refund_field_name))
|
|
if sales_value is None and refund_value is None:
|
|
skipped_empty_count += 1
|
|
continue
|
|
gmv_value = round((sales_value or 0.0) - (refund_value or 0.0), 2)
|
|
if dry_run:
|
|
print(
|
|
f"[DRY-RUN] 表={table.name} 店铺={store_name} 字段={gmv_field_name} "
|
|
f"销售额={sales_value if sales_value is not None else ''} "
|
|
f"退款额={refund_value if refund_value is not None else ''} GMV={gmv_value}"
|
|
)
|
|
else:
|
|
client.update_record(table.table_id, record_id, {gmv_field_name: gmv_value})
|
|
updated_count += 1
|
|
log_step(f"飞书记录店铺预览: 表={table.name}, stores={preview_list(preview_stores)}")
|
|
log_step(f"GMV 计算完成: 表={table.name}, 更新记录数={updated_count}, 双空跳过={skipped_empty_count}")
|
|
|
|
|
|
def run_gmv_sync(
|
|
start_date: date,
|
|
end_date: date,
|
|
base_url: str,
|
|
store_field_name: str,
|
|
dry_run: bool,
|
|
skip_missing_tables: bool,
|
|
) -> None:
|
|
client = FeishuBitableClient(base_url)
|
|
tables = client.list_tables()
|
|
for stat_date in iter_dates(start_date, end_date):
|
|
for project in SUPPORTED_PROJECTS:
|
|
try:
|
|
table = find_target_table(tables, project, stat_date.year, stat_date.month)
|
|
except ValueError as exc:
|
|
if not skip_missing_tables:
|
|
raise
|
|
log_step(f"跳过: {exc}")
|
|
continue
|
|
sync_gmv_table_for_day(
|
|
client=client,
|
|
table=table,
|
|
stat_date=stat_date,
|
|
preferred_store_field_name=store_field_name,
|
|
dry_run=dry_run,
|
|
)
|
|
|