添加 委外批次进数据库、委外批次入飞书
This commit is contained in:
@@ -0,0 +1,685 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate YXT batch inventory report by joining Raw batch stock with Analytics shop stock."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
import psycopg2
|
||||
|
||||
from inventory_sync.feishu import FeishuApiError, FeishuClient
|
||||
from inventory_sync.settings import build_app_config
|
||||
from inventory_sync.settings import load_env_candidates, require_config
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
LOG_DIR = PROJECT_ROOT / "logs"
|
||||
REPORT_DIR = PROJECT_ROOT / "reports"
|
||||
LOG_FILE = LOG_DIR / "generate_yxt_batch_inventory_report.log"
|
||||
LOGGER_NAME = "yxt_batch_inventory_report"
|
||||
BATCH_FEISHU_TABLE_NAME = "批次库存"
|
||||
SENSITIVE_PATTERN = re.compile(r"(?i)(password|passwd|secret|token|authorization)\s*[:=]\s*([^\s,;]+)")
|
||||
|
||||
RAW_BATCH_SQL = """
|
||||
WITH latest_batch AS (
|
||||
SELECT MAX(created_at) AS latest_created_at
|
||||
FROM raw_data.wangdian_batch_inventory
|
||||
),
|
||||
latest_raw AS (
|
||||
SELECT r.*
|
||||
FROM raw_data.wangdian_batch_inventory r
|
||||
JOIN latest_batch lb
|
||||
ON r.created_at = lb.latest_created_at
|
||||
)
|
||||
SELECT
|
||||
r.payload -> 'inventory_data' ->> '仓库' AS "仓库",
|
||||
NULLIF(r.payload -> 'inventory_data' ->> '保质期', '')::integer AS "保质期",
|
||||
r.payload -> 'inventory_data' ->> '批次号' AS "批次号",
|
||||
r.payload -> 'inventory_data' ->> '规格码' AS "规格码",
|
||||
NULLIF(r.payload -> 'inventory_data' ->> '冻结库存', '')::numeric(18,4) AS "冻结库存",
|
||||
NULLIF(
|
||||
r.payload -> 'inventory_data' ->> '创建时间',
|
||||
'0000-00-00 00:00:00'
|
||||
)::timestamp AS "创建时间",
|
||||
r.payload -> 'inventory_data' ->> '商家编码' AS "商家编码",
|
||||
NULLIF(r.payload -> 'inventory_data' ->> '库存日期', '0000-00-00')::date AS "库存日期",
|
||||
NULLIF(r.payload -> 'inventory_data' ->> '次品标记', '')::integer AS "次品标记",
|
||||
NULLIF(r.payload -> 'inventory_data' ->> '正常库存', '')::numeric(18,4) AS "正常库存",
|
||||
NULLIF(
|
||||
r.payload -> 'inventory_data' ->> '生产日期',
|
||||
'0000-00-00 00:00:00'
|
||||
)::timestamp AS "生产日期",
|
||||
r.payload -> 'inventory_data' ->> '规格名称' AS "规格名称",
|
||||
r.payload -> 'inventory_data' ->> '货品名称' AS "货品名称",
|
||||
NULLIF(
|
||||
r.payload -> 'inventory_data' ->> '过期日期',
|
||||
'0000-00-00 00:00:00'
|
||||
)::timestamp AS "过期日期",
|
||||
NULLIF(r.payload -> 'inventory_data' ->> '距离到期天数', '')::integer AS "距离到期天数",
|
||||
r.payload -> 'inventory_data' ->> '保质期计算方式' AS "保质期计算方式"
|
||||
FROM latest_raw r
|
||||
WHERE COALESCE(
|
||||
r.payload -> 'inventory_data' ->> '货品名称',
|
||||
''
|
||||
) NOT LIKE '%测试%'
|
||||
ORDER BY
|
||||
r.payload -> 'inventory_data' ->> '商家编码',
|
||||
r.payload -> 'inventory_data' ->> '仓库',
|
||||
r.payload -> 'inventory_data' ->> '批次号';
|
||||
"""
|
||||
|
||||
ANALYTICS_LOCK_SQL_TEMPLATE = """
|
||||
WITH raw_codes AS (
|
||||
SELECT code AS "统一规格编码"
|
||||
FROM (
|
||||
VALUES {code_values}
|
||||
) AS t(code)
|
||||
),
|
||||
|
||||
latest_shop_stock AS (
|
||||
SELECT
|
||||
t."统一规格编码",
|
||||
t."店铺主键",
|
||||
SUM(t."可发库存") AS "可发库存"
|
||||
FROM (
|
||||
SELECT
|
||||
f."统一规格编码",
|
||||
f."店铺主键",
|
||||
f."仓库主键",
|
||||
f."快照时间",
|
||||
f."可发库存",
|
||||
f."状态",
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY
|
||||
f."统一规格编码",
|
||||
f."店铺主键",
|
||||
f."仓库主键"
|
||||
ORDER BY f."快照时间" DESC
|
||||
) AS rn
|
||||
FROM analytics_layer.fact_shop_stock_snapshot f
|
||||
WHERE f."数据来源系统" = '旺店通店铺'
|
||||
AND NULLIF(TRIM(f."统一规格编码"), '') IS NOT NULL
|
||||
AND TRIM(f."统一规格编码") IN (
|
||||
SELECT "统一规格编码" FROM raw_codes
|
||||
)
|
||||
) t
|
||||
WHERE t.rn = 1
|
||||
AND t."状态" <> '已失效'
|
||||
GROUP BY
|
||||
t."统一规格编码",
|
||||
t."店铺主键"
|
||||
),
|
||||
|
||||
product_dedup AS (
|
||||
SELECT DISTINCT ON (TRIM("统一规格编码"))
|
||||
TRIM("统一规格编码") AS "统一规格编码",
|
||||
"品牌名称",
|
||||
"商品名称",
|
||||
"一级分类"
|
||||
FROM analytics_layer.dim_product
|
||||
WHERE NULLIF(TRIM("统一规格编码"), '') IS NOT NULL
|
||||
AND TRIM("统一规格编码") IN (
|
||||
SELECT "统一规格编码" FROM raw_codes
|
||||
)
|
||||
ORDER BY TRIM("统一规格编码"), "商品主键" DESC
|
||||
),
|
||||
|
||||
sales_30d AS (
|
||||
SELECT
|
||||
TRIM(fs."统一规格编码") AS "统一规格编码",
|
||||
SUM(fs."销售数量") AS "近30天销售数量"
|
||||
FROM analytics_layer.fact_sales fs
|
||||
WHERE NULLIF(TRIM(fs."统一规格编码"), '') IS NOT NULL
|
||||
AND TRIM(fs."统一规格编码") IN (
|
||||
SELECT "统一规格编码" FROM raw_codes
|
||||
)
|
||||
AND fs."发货时间" >= now() - INTERVAL '30 days'
|
||||
AND fs."发货时间" < now()
|
||||
GROUP BY TRIM(fs."统一规格编码")
|
||||
),
|
||||
|
||||
shop_pivot AS (
|
||||
SELECT
|
||||
l."统一规格编码",
|
||||
SUM(CASE WHEN s."店铺名称" = '有赞_大眼睛买买买商店' THEN l."可发库存" ELSE 0 END) AS "有赞_大眼睛买买买商店",
|
||||
SUM(CASE WHEN s."店铺名称" = '店铺-有赞-麦漾' THEN l."可发库存" ELSE 0 END) AS "店铺-有赞-麦漾",
|
||||
SUM(CASE WHEN s."店铺名称" = '店铺-天猫-羽心堂旗舰店' THEN l."可发库存" ELSE 0 END) AS "店铺-天猫-羽心堂旗舰店",
|
||||
SUM(CASE WHEN s."店铺名称" = '店铺-抖音-羽心堂旗舰店' THEN l."可发库存" ELSE 0 END) AS "店铺-抖音-羽心堂旗舰店"
|
||||
FROM latest_shop_stock l
|
||||
LEFT JOIN analytics_layer.dim_shop s
|
||||
ON l."店铺主键" = s."店铺主键"
|
||||
GROUP BY l."统一规格编码"
|
||||
)
|
||||
|
||||
SELECT
|
||||
rc."统一规格编码",
|
||||
pd."品牌名称",
|
||||
pd."商品名称",
|
||||
pd."一级分类",
|
||||
COALESCE(sp."有赞_大眼睛买买买商店", 0) AS "有赞_大眼睛买买买商店",
|
||||
COALESCE(sp."店铺-有赞-麦漾", 0) AS "店铺-有赞-麦漾",
|
||||
COALESCE(sp."店铺-天猫-羽心堂旗舰店", 0) AS "店铺-天猫-羽心堂旗舰店",
|
||||
COALESCE(sp."店铺-抖音-羽心堂旗舰店", 0) AS "店铺-抖音-羽心堂旗舰店",
|
||||
COALESCE(sa."近30天销售数量", 0) AS "近30天销售数量"
|
||||
FROM raw_codes rc
|
||||
LEFT JOIN product_dedup pd
|
||||
ON rc."统一规格编码" = pd."统一规格编码"
|
||||
LEFT JOIN shop_pivot sp
|
||||
ON rc."统一规格编码" = sp."统一规格编码"
|
||||
LEFT JOIN sales_30d sa
|
||||
ON rc."统一规格编码" = sa."统一规格编码"
|
||||
ORDER BY rc."统一规格编码";
|
||||
"""
|
||||
|
||||
WAREHOUSE_FILTER = {
|
||||
"杭州飞蚁仓",
|
||||
"飞蚁-大眼睛仓",
|
||||
"北京配思云仓",
|
||||
"上海顺丰松江仓",
|
||||
}
|
||||
|
||||
logger = logging.getLogger(LOGGER_NAME)
|
||||
|
||||
LOG_FIELD_LABELS = {
|
||||
"available_fields": "可用字段",
|
||||
"created_records": "新增记录数",
|
||||
"database": "数据库",
|
||||
"deleted_records": "删除记录数",
|
||||
"elapsed_ms": "耗时毫秒",
|
||||
"env_path": "环境文件",
|
||||
"existing_records": "已有记录数",
|
||||
"filtered_rows": "筛选后行数",
|
||||
"join_key": "关联字段",
|
||||
"log_file": "日志文件",
|
||||
"matched_rows": "匹配行数",
|
||||
"output": "输出文件",
|
||||
"raw_rows": "Raw行数",
|
||||
"report_rows": "报表行数",
|
||||
"rows": "行数",
|
||||
"table": "表",
|
||||
"table_id": "表ID",
|
||||
"table_name": "表名",
|
||||
"unmatched_rows": "未匹配行数",
|
||||
"warehouse_count": "仓库数量",
|
||||
}
|
||||
|
||||
|
||||
def sanitize_value(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
return SENSITIVE_PATTERN.sub(r"\1=***", value)
|
||||
return value
|
||||
|
||||
|
||||
def configure_logging() -> None:
|
||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
formatter = logging.Formatter(
|
||||
fmt="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
stream_handler = logging.StreamHandler(sys.stdout)
|
||||
stream_handler.setFormatter(formatter)
|
||||
file_handler = RotatingFileHandler(LOG_FILE, maxBytes=5 * 1024 * 1024, backupCount=5, encoding="utf-8")
|
||||
file_handler.setFormatter(formatter)
|
||||
root_logger = logging.getLogger()
|
||||
root_logger.handlers.clear()
|
||||
root_logger.setLevel(logging.INFO)
|
||||
root_logger.addHandler(stream_handler)
|
||||
root_logger.addHandler(file_handler)
|
||||
|
||||
|
||||
def log_event(level: int, event: str, **fields: Any) -> None:
|
||||
safe_event = sanitize_value(event)
|
||||
details = " ".join(
|
||||
f"{LOG_FIELD_LABELS.get(key, key)}={sanitize_value(value)!r}" for key, value in fields.items()
|
||||
)
|
||||
logger.log(level, "事件=%s%s", safe_event, f" {details}" if details else "")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DbConfig:
|
||||
host: str
|
||||
port: str
|
||||
user: str
|
||||
password: str
|
||||
name: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FeishuFieldSpec:
|
||||
field_name: str # 飞书字段名
|
||||
source_name: str # DataFrame 列名
|
||||
field_type: int
|
||||
formatter: str | None = None
|
||||
|
||||
|
||||
BATCH_FEISHU_PRIMARY_FIELD = "库存日期"
|
||||
BATCH_FEISHU_FIELDS = [
|
||||
FeishuFieldSpec("货品编码", "商家编码", 1),
|
||||
FeishuFieldSpec("货品名称", "货品名称", 1),
|
||||
FeishuFieldSpec("批次号", "批次号", 1),
|
||||
FeishuFieldSpec("生产日期", "生产日期", 1),
|
||||
FeishuFieldSpec("过期日期", "过期日期", 1),
|
||||
FeishuFieldSpec("批次库存", "正常库存", 2, "0"),
|
||||
FeishuFieldSpec("有赞_大眼睛买买买商店", "有赞_大眼睛买买买商店", 2, "0"),
|
||||
FeishuFieldSpec("店铺-有赞-麦漾", "店铺-有赞-麦漾", 2, "0"),
|
||||
FeishuFieldSpec("店铺-天猫-羽心堂旗舰店", "店铺-天猫-羽心堂旗舰店", 2, "0"),
|
||||
FeishuFieldSpec("店铺-抖音-羽心堂旗舰店", "店铺-抖音-羽心堂旗舰店", 2, "0"),
|
||||
FeishuFieldSpec("保质期", "保质期", 2, "0"),
|
||||
FeishuFieldSpec("距离到期天数", "距离到期天数", 2, "0"),
|
||||
FeishuFieldSpec("效期比例", "效期比例", 2, "0.00"),
|
||||
FeishuFieldSpec("近30天销售数量", "近30天销售数量", 2, "0"),
|
||||
FeishuFieldSpec("分类", "一级分类", 1),
|
||||
]
|
||||
|
||||
|
||||
def load_db_config(prefix: str) -> tuple[DbConfig, Path | None]:
|
||||
config_map, env_path = load_env_candidates()
|
||||
return (
|
||||
DbConfig(
|
||||
host=require_config(config_map, f"{prefix}_DB_HOST"),
|
||||
port=require_config(config_map, f"{prefix}_DB_PORT"),
|
||||
user=require_config(config_map, f"{prefix}_DB_USER"),
|
||||
password=require_config(config_map, f"{prefix}_DB_PASSWORD"),
|
||||
name=require_config(config_map, f"{prefix}_DB_NAME"),
|
||||
),
|
||||
env_path,
|
||||
)
|
||||
|
||||
|
||||
def format_feishu_value(column: str, value: Any) -> Any:
|
||||
if pd.isna(value):
|
||||
return None
|
||||
if isinstance(value, pd.Timestamp):
|
||||
if column == "库存日期":
|
||||
return value.strftime("%Y-%m-%d")
|
||||
return value.strftime("%Y-%m-%d %H:%M:%S")
|
||||
if isinstance(value, datetime):
|
||||
if column == "库存日期":
|
||||
return value.strftime("%Y-%m-%d")
|
||||
return value.strftime("%Y-%m-%d %H:%M:%S")
|
||||
if column in {
|
||||
"正常库存",
|
||||
"有赞_大眼睛买买买商店",
|
||||
"店铺-有赞-麦漾",
|
||||
"店铺-天猫-羽心堂旗舰店",
|
||||
"店铺-抖音-羽心堂旗舰店",
|
||||
"近30天销售数量",
|
||||
"保质期",
|
||||
"距离到期天数",
|
||||
"效期比例",
|
||||
}:
|
||||
return float(value)
|
||||
return str(value)
|
||||
|
||||
|
||||
def build_batch_feishu_records(df: pd.DataFrame) -> list[dict[str, Any]]:
|
||||
records: list[dict[str, Any]] = []
|
||||
for _, row in df.iterrows():
|
||||
fields: dict[str, Any] = {
|
||||
BATCH_FEISHU_PRIMARY_FIELD: format_feishu_value("库存日期", row.get("库存日期")),
|
||||
}
|
||||
for spec in BATCH_FEISHU_FIELDS:
|
||||
fields[spec.field_name] = format_feishu_value(spec.source_name, row.get(spec.source_name))
|
||||
records.append({"fields": fields})
|
||||
return records
|
||||
|
||||
|
||||
def ensure_batch_inventory_fields(
|
||||
client: FeishuClient,
|
||||
access_token: str,
|
||||
app_token: str,
|
||||
table_id: str,
|
||||
) -> list[str]:
|
||||
managed_names = {BATCH_FEISHU_PRIMARY_FIELD, *(field.field_name for field in BATCH_FEISHU_FIELDS)}
|
||||
fields = client.list_fields(access_token, app_token, table_id)
|
||||
primary_field = next((field for field in fields if field.get("is_primary")), None)
|
||||
if not primary_field:
|
||||
raise FeishuApiError("未找到批次库存表主字段")
|
||||
|
||||
duplicate_primary = next(
|
||||
(
|
||||
field for field in fields
|
||||
if not field.get("is_primary") and field.get("field_name") == BATCH_FEISHU_PRIMARY_FIELD
|
||||
),
|
||||
None,
|
||||
)
|
||||
if duplicate_primary:
|
||||
client.delete_field(access_token, app_token, table_id, duplicate_primary["field_id"])
|
||||
time.sleep(0.2)
|
||||
fields = client.list_fields(access_token, app_token, table_id)
|
||||
primary_field = next((field for field in fields if field.get("is_primary")), None)
|
||||
if not primary_field:
|
||||
raise FeishuApiError("删除重复主字段后未找到批次库存表主字段")
|
||||
|
||||
if primary_field.get("field_name") != BATCH_FEISHU_PRIMARY_FIELD:
|
||||
payload = {"field_name": BATCH_FEISHU_PRIMARY_FIELD, "type": primary_field.get("type")}
|
||||
if primary_field.get("property") is not None:
|
||||
payload["property"] = primary_field.get("property")
|
||||
client.update_field(access_token, app_token, table_id, primary_field["field_id"], payload)
|
||||
time.sleep(0.2)
|
||||
|
||||
fields = client.list_fields(access_token, app_token, table_id)
|
||||
for field in fields:
|
||||
field_name = field.get("field_name")
|
||||
field_id = field.get("field_id")
|
||||
if field.get("is_primary") or not field_name or not field_id:
|
||||
continue
|
||||
if field_name in managed_names:
|
||||
client.delete_field(access_token, app_token, table_id, field_id)
|
||||
time.sleep(0.2)
|
||||
|
||||
for spec in BATCH_FEISHU_FIELDS:
|
||||
payload: dict[str, Any] = {"field_name": spec.field_name, "type": spec.field_type}
|
||||
if spec.formatter is not None:
|
||||
payload["property"] = {"formatter": spec.formatter}
|
||||
client.create_field(access_token, app_token, table_id, payload)
|
||||
time.sleep(0.2)
|
||||
|
||||
return [
|
||||
field.get("field_name")
|
||||
for field in client.list_fields(access_token, app_token, table_id)
|
||||
if field.get("field_name")
|
||||
]
|
||||
|
||||
|
||||
def sync_report_to_feishu(df: pd.DataFrame, table_name: str = BATCH_FEISHU_TABLE_NAME) -> dict[str, Any]:
|
||||
app_config = build_app_config()
|
||||
client = FeishuClient(app_config)
|
||||
access_token = client.get_tenant_access_token()
|
||||
base_info = client.parse_base_info(client.build_base_url(), access_token)
|
||||
app_token = base_info["app_token"] or ""
|
||||
client.enable_advanced_permission(access_token, app_token)
|
||||
tables = client.list_tables(access_token, app_token)
|
||||
target_table = next((table for table in tables if table.get("name") == table_name), None)
|
||||
if not target_table:
|
||||
raise FeishuApiError(f"未找到目标飞书表: {table_name}")
|
||||
table_id = target_table.get("table_id")
|
||||
if not table_id:
|
||||
raise FeishuApiError(f"目标飞书表缺少 table_id: {table_name}")
|
||||
|
||||
log_event(logging.INFO, "开始同步飞书批次库存表", table_name=table_name, table_id=table_id, report_rows=len(df))
|
||||
available_fields = ensure_batch_inventory_fields(client, access_token, app_token, table_id)
|
||||
existing_records = client.list_records(access_token, app_token, table_id)
|
||||
existing_record_ids = [record.get("record_id") for record in existing_records if record.get("record_id")]
|
||||
deleted_count = (
|
||||
client.batch_delete_records(access_token, app_token, table_id, existing_record_ids)
|
||||
if existing_record_ids else 0
|
||||
)
|
||||
records = build_batch_feishu_records(df)
|
||||
created_count = (
|
||||
client.batch_create_records(access_token, app_token, table_id, records)
|
||||
if records else 0
|
||||
)
|
||||
log_event(
|
||||
logging.INFO,
|
||||
"飞书批次库存表同步完成",
|
||||
table_name=table_name,
|
||||
table_id=table_id,
|
||||
existing_records=len(existing_record_ids),
|
||||
deleted_records=deleted_count,
|
||||
created_records=created_count,
|
||||
available_fields=", ".join(available_fields),
|
||||
)
|
||||
return {
|
||||
"table_name": table_name,
|
||||
"table_id": table_id,
|
||||
"available_fields": available_fields,
|
||||
"existing_records": len(existing_record_ids),
|
||||
"deleted_records": deleted_count,
|
||||
"created_records": created_count,
|
||||
}
|
||||
|
||||
|
||||
def query_dataframe(db_config: DbConfig, sql: str, *, database_name: str) -> pd.DataFrame:
|
||||
started_at = time.monotonic()
|
||||
log_event(logging.INFO, "开始查询数据库", database=database_name)
|
||||
with psycopg2.connect(
|
||||
host=db_config.host,
|
||||
port=db_config.port,
|
||||
user=db_config.user,
|
||||
password=db_config.password,
|
||||
dbname=db_config.name,
|
||||
) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SET TIME ZONE 'Asia/Shanghai';")
|
||||
df = pd.read_sql_query(sql, conn)
|
||||
log_event(
|
||||
logging.INFO,
|
||||
"数据库查询完成",
|
||||
database=database_name,
|
||||
rows=len(df),
|
||||
elapsed_ms=int((time.monotonic() - started_at) * 1000),
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
def normalize_code(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def quote_sql_literal(value: str) -> str:
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
|
||||
|
||||
def extract_raw_codes(df: pd.DataFrame) -> list[str]:
|
||||
codes = (
|
||||
df["商家编码"]
|
||||
.map(normalize_code)
|
||||
.dropna()
|
||||
.drop_duplicates()
|
||||
.tolist()
|
||||
)
|
||||
log_event(logging.INFO, "已提取Raw商家编码集合", rows=len(codes), join_key="商家编码=统一规格编码")
|
||||
return codes
|
||||
|
||||
|
||||
def build_analytics_lock_sql(codes: list[str]) -> str:
|
||||
if not codes:
|
||||
raise ValueError("RAW_BATCH_SQL 未查到可用于查询 Analytics 的商家编码")
|
||||
code_values = ", ".join(f"({quote_sql_literal(code)})" for code in codes)
|
||||
return ANALYTICS_LOCK_SQL_TEMPLATE.format(code_values=code_values)
|
||||
|
||||
|
||||
def filter_batch_inventory(df: pd.DataFrame) -> pd.DataFrame:
|
||||
filtered = df.copy()
|
||||
filtered["货品名称"] = filtered["货品名称"].fillna("")
|
||||
filtered["仓库"] = filtered["仓库"].fillna("")
|
||||
filtered = filtered[
|
||||
filtered["货品名称"].str.contains("羽心堂", na=False)
|
||||
& filtered["仓库"].isin(WAREHOUSE_FILTER)
|
||||
].copy()
|
||||
filtered["关联编码"] = filtered["商家编码"].map(normalize_code)
|
||||
missing_join_key = filtered["关联编码"].isna().sum()
|
||||
if missing_join_key:
|
||||
filtered.loc[filtered["关联编码"].isna(), "关联编码"] = filtered.loc[
|
||||
filtered["关联编码"].isna(), "规格码"
|
||||
].map(normalize_code)
|
||||
log_event(
|
||||
logging.INFO,
|
||||
"Raw批次库存筛选完成",
|
||||
raw_rows=len(df),
|
||||
filtered_rows=len(filtered),
|
||||
warehouse_count=len(WAREHOUSE_FILTER),
|
||||
)
|
||||
return filtered
|
||||
|
||||
|
||||
def prepare_analytics_stock(df: pd.DataFrame) -> pd.DataFrame:
|
||||
prepared = df.copy()
|
||||
prepared["关联编码"] = prepared["统一规格编码"].map(normalize_code)
|
||||
log_event(logging.INFO, "店铺锁定库存整理完成", rows=len(prepared), join_key="统一规格编码")
|
||||
return prepared
|
||||
|
||||
|
||||
def build_report(batch_df: pd.DataFrame, analytics_df: pd.DataFrame) -> pd.DataFrame:
|
||||
merged = batch_df.merge(
|
||||
analytics_df,
|
||||
how="left",
|
||||
on="关联编码",
|
||||
suffixes=("", "_锁定库存"),
|
||||
)
|
||||
matched_rows = merged["统一规格编码"].notna().sum()
|
||||
unmatched_rows = merged["统一规格编码"].isna().sum()
|
||||
group_columns = [
|
||||
"库存日期",
|
||||
"商家编码",
|
||||
"货品名称",
|
||||
"一级分类",
|
||||
"批次号",
|
||||
"生产日期",
|
||||
"过期日期",
|
||||
"保质期",
|
||||
"距离到期天数",
|
||||
]
|
||||
inventory_columns = [
|
||||
"有赞_大眼睛买买买商店",
|
||||
"店铺-有赞-麦漾",
|
||||
"店铺-天猫-羽心堂旗舰店",
|
||||
"店铺-抖音-羽心堂旗舰店",
|
||||
]
|
||||
aggregation_map: dict[str, str] = {"正常库存": "sum"}
|
||||
for column in inventory_columns:
|
||||
if column in merged.columns:
|
||||
aggregation_map[column] = "max"
|
||||
if "近30天销售数量" in merged.columns:
|
||||
aggregation_map["近30天销售数量"] = "max"
|
||||
|
||||
grouped = (
|
||||
merged.groupby(group_columns, dropna=False, as_index=False)
|
||||
.agg(aggregation_map)
|
||||
)
|
||||
grouped["效期比例"] = grouped.apply(
|
||||
lambda row: (
|
||||
float(row["距离到期天数"]) / float(row["保质期"])
|
||||
if pd.notna(row["距离到期天数"]) and pd.notna(row["保质期"]) and float(row["保质期"]) != 0
|
||||
else None
|
||||
),
|
||||
axis=1,
|
||||
)
|
||||
report_columns = [
|
||||
"库存日期",
|
||||
"商家编码",
|
||||
"货品名称",
|
||||
"一级分类",
|
||||
"批次号",
|
||||
"生产日期",
|
||||
"过期日期",
|
||||
"正常库存",
|
||||
"有赞_大眼睛买买买商店",
|
||||
"店铺-有赞-麦漾",
|
||||
"店铺-天猫-羽心堂旗舰店",
|
||||
"店铺-抖音-羽心堂旗舰店",
|
||||
"近30天销售数量",
|
||||
"保质期",
|
||||
"距离到期天数",
|
||||
"效期比例",
|
||||
]
|
||||
available_columns = [column for column in report_columns if column in grouped.columns]
|
||||
report = grouped[available_columns].sort_values(
|
||||
by=["商家编码", "批次号", "生产日期"],
|
||||
na_position="last",
|
||||
).reset_index(drop=True)
|
||||
log_event(
|
||||
logging.INFO,
|
||||
"批次库存与店铺锁定库存汇总完成",
|
||||
report_rows=len(report),
|
||||
matched_rows=int(matched_rows),
|
||||
unmatched_rows=int(unmatched_rows),
|
||||
join_key="关联编码",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def save_report(df: pd.DataFrame, output_path: Path) -> None:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
started_at = time.monotonic()
|
||||
suffix = output_path.suffix.lower()
|
||||
if suffix == ".csv":
|
||||
df.to_csv(output_path, index=False, encoding="utf-8-sig")
|
||||
else:
|
||||
df.to_excel(output_path, index=False)
|
||||
log_event(
|
||||
logging.INFO,
|
||||
"报表输出完成",
|
||||
output=output_path,
|
||||
report_rows=len(df),
|
||||
elapsed_ms=int((time.monotonic() - started_at) * 1000),
|
||||
)
|
||||
|
||||
|
||||
def default_output_path() -> Path:
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
return REPORT_DIR / f"羽心堂批次库存与店铺锁定库存_{timestamp}.xlsx"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="生成羽心堂批次库存与店铺锁定库存合并报表")
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=str(default_output_path()),
|
||||
help="输出文件路径，支持 .xlsx 或 .csv，默认写入 reports/ 目录",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--feishu-table",
|
||||
default=BATCH_FEISHU_TABLE_NAME,
|
||||
help="飞书多维表中的目标表名，默认 批次库存表",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-feishu-sync",
|
||||
action="store_true",
|
||||
help="只生成本地报表，不同步到飞书",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
configure_logging()
|
||||
started_at = time.monotonic()
|
||||
args = parse_args()
|
||||
output_path = Path(args.output).expanduser().resolve()
|
||||
raw_config, env_path = load_db_config("RAW")
|
||||
analytics_config, _ = load_db_config("ANALYTICS")
|
||||
log_event(
|
||||
logging.INFO,
|
||||
"羽心堂批次库存报表任务启动",
|
||||
env_path=env_path,
|
||||
output=output_path,
|
||||
log_file=LOG_FILE,
|
||||
)
|
||||
raw_df = query_dataframe(raw_config, RAW_BATCH_SQL, database_name="Raw")
|
||||
raw_codes = extract_raw_codes(raw_df)
|
||||
analytics_sql = build_analytics_lock_sql(raw_codes)
|
||||
analytics_df = query_dataframe(analytics_config, analytics_sql, database_name="Analytics")
|
||||
filtered_batch_df = filter_batch_inventory(raw_df)
|
||||
prepared_analytics_df = prepare_analytics_stock(analytics_df)
|
||||
report_df = build_report(filtered_batch_df, prepared_analytics_df)
|
||||
save_report(report_df, output_path)
|
||||
if args.skip_feishu_sync:
|
||||
log_event(logging.INFO, "已跳过飞书批次库存表同步", table_name=args.feishu_table)
|
||||
else:
|
||||
sync_report_to_feishu(report_df, table_name=args.feishu_table)
|
||||
log_event(
|
||||
logging.INFO,
|
||||
"羽心堂批次库存报表任务完成",
|
||||
output=output_path,
|
||||
report_rows=len(report_df),
|
||||
elapsed_ms=int((time.monotonic() - started_at) * 1000),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user