BXY0414/gmv-analytics
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

调整

Browse Source
This commit is contained in:
BXY0414
2026-03-24 16:33:14 +08:00
parent a9c7b7ead5
commit aae76442ec
7 changed files with 499 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
project_bitable_common.py
+98 -71
View File
@@ -124,7 +124,7 @@ UNION ALL
SELECT
统计日期,
CASE
WHEN 业务模式 IN ('三方计算','三方分销')
WHEN 业务模式 IN ('三方结算','三方分销')
THEN 原始店铺名称 || '_三方'
ELSE 原始店铺名称 || '_自营'
END AS 店铺名称,
@@ -664,44 +664,24 @@ class FeishuBitableClient:
self.request(
"POST",
f"/open-apis/bitable/v1/apps/{self.app_token}/tables/{table_id}/fields",
json={"field_name": field_name, "type": 2},
json={"field_name": field_name, "type": 2, "property": {"formatter": "0.00"}},
)
self._fields_cache.pop(table_id, None)
log_step(f"字段创建完成: table_id={table_id}, field={field_name}")
return True
def try_list_field_groups(self, table_id: str) -> list[dict[str, Any]] | None:
path = f"/open-apis/bitable/v1/apps/{self.app_token}/tables/{table_id}/field_groups"
url = f"https://open.feishu.cn{path}"
log_debug(f"尝试读取字段编组: table_id={table_id}")
resp = self.session.request("GET", url, timeout=60)
try:
payload = resp.json()
except ValueError:
log_debug(f"字段编组接口返回非 JSON，跳过读取: table_id={table_id}, status={resp.status_code}")
return None
if resp.status_code == 404:
log_debug(f"字段编组读取接口不可用: table_id={table_id}, payload={payload}")
return None
if resp.status_code >= 400 or payload.get("code") != 0:
log_debug(f"字段编组读取失败，跳过比对: table_id={table_id}, payload={payload}")
return None
data = payload.get("data") or {}
items = data.get("items") or data.get("field_groups") or []
log_debug(f"字段编组读取成功: table_id={table_id}, 编组数={len(items)}")
return items
def create_field_groups(self, table_id: str, field_groups: list[dict[str, Any]]) -> None:
def create_field_groups(self, table_id: str, field_groups: list[dict[str, Any]]) -> dict[str, Any]:
log_step(
f"开始创建字段编组: table_id={table_id}, "
f"groups={[(group.get('name') or '', len(group.get('children') or [])) for group in field_groups]}"
)
self.request(
data = self.request(
"POST",
f"/open-apis/bitable/v1/apps/{self.app_token}/tables/{table_id}/field_groups",
json={"field_groups": field_groups},
)
log_step(f"字段编组创建请求完成: table_id={table_id}")
return data
def list_records(self, table_id: str) -> list[dict[str, Any]]:
if table_id in self._records_cache:
@@ -1000,6 +980,7 @@ def build_field_group_payload(
store_field_name: str,
allowed_field_names: set[str] | None = None,
) -> list[dict[str, Any]]:
summary_fields: list[dict[str, str]] = []
gmv_fields: list[dict[str, str]] = []
other_fields: list[dict[str, str]] = []
for field in fields:
@@ -1012,11 +993,16 @@ def build_field_group_payload(
if allowed_field_names is not None and field_name not in allowed_field_names:
continue
child = {"type": "field", "id": field_id}
if field_name == get_summary_gmv_field_name():
summary_fields.append(child)
continue
if "gmv" in field_name.lower():
gmv_fields.append(child)
else:
other_fields.append(child)
groups: list[dict[str, Any]] = []
if summary_fields:
groups.append({"name": "汇总金额", "children": summary_fields})
if gmv_fields:
groups.append({"name": "GMV编组", "children": gmv_fields})
if other_fields:
@@ -1024,22 +1010,13 @@ def build_field_group_payload(
return groups
def normalize_field_groups(field_groups: list[dict[str, Any]]) -> dict[str, set[str]]:
normalized: dict[str, set[str]] = {}
for group in field_groups:
name = str(group.get("name") or "").strip()
if not name:
continue
normalized[name] = {
str(child.get("id") or "").strip()
for child in (group.get("children") or [])
if str(child.get("id") or "").strip()
}
return normalized
def get_field_group_order(field_groups: list[dict[str, Any]]) -> list[str]:
return [str(group.get("name") or "").strip() for group in field_groups if str(group.get("name") or "").strip()]
def get_incremental_field_names_for_day(stat_date: date) -> set[str]:
return {
get_summary_gmv_field_name(),
get_gmv_field_name(stat_date),
get_sales_field_name(stat_date),
get_refund_field_name(stat_date),
}
def ensure_field_groups_for_table(
@@ -1057,21 +1034,6 @@ def ensure_field_groups_for_table(
return
desired_summary = {group["name"]: len(group.get("children") or []) for group in desired_groups}
log_step(f"目标字段编组: 表={table.name}, summary={desired_summary}")
existing_groups = client.try_list_field_groups(table.table_id)
if existing_groups is not None:
existing_normalized = normalize_field_groups(existing_groups)
desired_normalized = normalize_field_groups(desired_groups)
existing_order = get_field_group_order(existing_groups)
desired_order = get_field_group_order(desired_groups)
if existing_normalized == desired_normalized and existing_order == desired_order:
log_step(f"字段编组已存在且一致，跳过创建: 表={table.name}")
return
log_debug(
f"字段编组存在差异: 表={table.name}, "
f"existing={ {name: len(ids) for name, ids in existing_normalized.items()} }, "
f"desired={ {name: len(ids) for name, ids in desired_normalized.items()} }, "
f"existing_order={existing_order}, desired_order={desired_order}"
)
if dry_run:
print(f"[DRY-RUN] 表={table.name} 将创建/更新字段编组: {desired_summary}")
return
@@ -1087,6 +1049,12 @@ def ensure_field_groups_for_table(
"原因=应用缺少 base:field_group:create 权限，请在飞书开放平台开通后重试"
)
return
if "1254117" in error_text or "The name of field group is duplicated" in error_text:
log_step(
f"跳过字段编组创建: 表={table.name}, "
"原因=字段编组已存在同名分组，按幂等场景处理"
)
return
if "1254122" in error_text or "Child belongs to multiple field groups" in error_text:
log_step(f"检测到字段已被其他编组占用，尝试逐个创建编组: 表={table.name}")
for group in desired_groups:
@@ -1143,6 +1111,31 @@ def sync_field_groups_for_range(
log_step("字段编组同步完成")
def rebuild_field_groups_for_range_with_context(
context: DailySyncContext,
start_date: date,
end_date: date,
) -> None:
log_step(f"开始重建字段编组: {start_date.isoformat()} -> {end_date.isoformat()}")
for (year, month), _ in sorted(build_group_dates_by_month(start_date, end_date).items()):
for project in SUPPORTED_PROJECTS:
try:
table = find_target_table(context.tables, project, year, month, context.table_overrides)
except ValueError as exc:
if context.config.skip_missing_tables:
log_step(f"跳过字段编组重建: {exc}")
continue
raise
ensure_field_groups_for_table(
client=context.client,
table=table,
store_field_name=context.config.store_field_name,
allowed_field_names=None,
dry_run=context.config.dry_run,
)
log_step("字段编组重建完成")
def sync_field_groups_for_day_with_context(
context: DailySyncContext,
stat_date: date,
@@ -1155,11 +1148,7 @@ def sync_field_groups_for_day_with_context(
if allowed_field_names_override is not None:
allowed_field_names = set(allowed_field_names_override)
elif not force_full_rebuild and stat_date.day != 1:
allowed_field_names = {
get_gmv_field_name(stat_date),
get_sales_field_name(stat_date),
get_refund_field_name(stat_date),
}
allowed_field_names = get_incremental_field_names_for_day(stat_date)
log_step(f"增量编组目标字段: 日期={stat_date.isoformat()}, fields={sorted(allowed_field_names)}")
for project in SUPPORTED_PROJECTS:
try:
@@ -1355,19 +1344,24 @@ def get_gmv_field_name(stat_date: date) -> str:
return stat_date.strftime("%m-%d GMV")
def get_summary_gmv_field_name() -> str:
return "汇总GMV"
def sync_gmv_table_for_day(
client: FeishuBitableClient,
table: TableInfo,
stat_date: date,
preferred_store_field_name: str,
dry_run: bool,
) -> bool:
) -> set[str]:
log_step(f"开始计算 GMV: 表={table.name}, 日期={stat_date.isoformat()}")
fields = client.list_fields(table.table_id)
store_field_name = resolve_store_field_name(fields, preferred_store_field_name)
sales_field_name = get_sales_field_name(stat_date)
refund_field_name = get_refund_field_name(stat_date)
gmv_field_name = get_gmv_field_name(stat_date)
summary_gmv_field_name = get_summary_gmv_field_name()
field_names = [str(item.get("field_name") or "") for item in fields]
log_debug(
f"GMV 字段检查: 表={table.name}, 店铺字段={store_field_name}, 销售字段={sales_field_name in field_names}, "
@@ -1375,10 +1369,20 @@ def sync_gmv_table_for_day(
)
if sales_field_name not in field_names or refund_field_name not in field_names:
log_step(f"跳过 GMV 计算: 表={table.name}, 原因=缺少销售额或退款额字段")
return False
created_field = False
return set()
created_fields: set[str] = set()
if not dry_run:
created_field = client.ensure_number_field(table.table_id, gmv_field_name)
if client.ensure_number_field(table.table_id, gmv_field_name):
created_fields.add(gmv_field_name)
if client.ensure_number_field(table.table_id, summary_gmv_field_name):
created_fields.add(summary_gmv_field_name)
fields = client.list_fields(table.table_id)
field_names = [str(item.get("field_name") or "") for item in fields]
month_gmv_field_names = sorted(
field_name
for field_name in field_names
if field_name.endswith(" GMV") and field_name != summary_gmv_field_name
)
records = client.list_records(table.table_id)
preview_stores = []
updated_count = 0
@@ -1398,18 +1402,27 @@ def sync_gmv_table_for_day(
skipped_empty_count += 1
continue
gmv_value = round((sales_value or 0.0) - (refund_value or 0.0), 2)
summary_gmv_value = round(
sum(parse_number(record_fields.get(name)) or 0.0 for name in month_gmv_field_names if name != gmv_field_name) + gmv_value,
2,
)
if dry_run:
print(
f"[DRY-RUN] 表={table.name} 店铺={store_name} 字段={gmv_field_name} "
f"销售额={sales_value if sales_value is not None else ''} "
f"退款额={refund_value if refund_value is not None else ''} GMV={gmv_value}"
f"退款额={refund_value if refund_value is not None else ''} GMV={gmv_value} "
f"{summary_gmv_field_name}={summary_gmv_value}"
)
else:
client.update_record(table.table_id, record_id, {gmv_field_name: gmv_value})
client.update_record(
table.table_id,
record_id,
{gmv_field_name: gmv_value, summary_gmv_field_name: summary_gmv_value},
)
updated_count += 1
log_debug(f"飞书记录店铺预览: 表={table.name}, stores={preview_list(preview_stores)}")
log_step(f"GMV 计算完成: 表={table.name}, 更新记录数={updated_count}, 双空跳过={skipped_empty_count}")
return created_field
return created_fields
def run_gmv_sync_with_context(
@@ -1434,8 +1447,7 @@ def run_gmv_sync_with_context(
preferred_store_field_name=context.config.store_field_name,
dry_run=context.config.dry_run,
)
if created:
created_fields.add(get_gmv_field_name(stat_date))
created_fields.update(created)
return MetricSyncArtifacts(created_fields=created_fields)
@@ -1460,6 +1472,17 @@ def run_gmv_sync(
run_gmv_sync_with_context(context, start_date, end_date)
def run_daily_data_sync_for_date(context: DailySyncContext, stat_date: date) -> None:
log_step(f"开始处理日期: {stat_date.isoformat()}")
created_fields: set[str] = set()
for metric_name in ("sales", "refund"):
artifacts = run_metric_sync_with_context(context, metric_name, stat_date, stat_date)
created_fields.update(artifacts.created_fields)
gmv_artifacts = run_gmv_sync_with_context(context, stat_date, stat_date)
created_fields.update(gmv_artifacts.created_fields)
log_step(f"日期处理完成: {stat_date.isoformat()}")
def run_daily_sync_for_date(context: DailySyncContext, stat_date: date) -> None:
log_step(f"开始处理日期: {stat_date.isoformat()}")
created_fields: set[str] = set()
@@ -1471,9 +1494,13 @@ def run_daily_sync_for_date(context: DailySyncContext, stat_date: date) -> None:
should_sync_field_groups = stat_date.day == 1 or bool(created_fields)
if should_sync_field_groups:
if stat_date.day == 1:
sync_reason = "每月1号完整编组"
else:
sync_reason = f"新增字段={sorted(created_fields)}"
log_step(
f"触发字段编组同步: 日期={stat_date.isoformat()}, "
f"原因={'每月1号完整编组' if stat_date.day == 1 else f'新增字段={sorted(created_fields)}'}"
f"原因={sync_reason}"
)
sync_field_groups_for_day_with_context(
context=context,