BXY0414/logistics-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加prefect调度

Browse Source
This commit is contained in:
BXY0414
2026-03-26 10:52:35 +08:00
parent 2822c9b558
commit bef5c46f85
12 changed files with 753 additions and 3329 deletions
Download Patch File Download Diff File Expand all files Collapse all files
import_logistics_to_feishu_bitable.py
+220 -70
View File
@@ -1,8 +1,6 @@
#!/usr/bin/env python3
import csv
import json
import os
import sys
import time
from datetime import datetime
from pathlib import Path
@@ -23,9 +21,9 @@ except ImportError:
ENV_PATH = Path(".env")
DEFAULT_CSV_PATH = Path("物流.csv")
DEFAULT_BATCH_SIZE = 500
DEFAULT_SOURCE = "db"
DEFAULT_HTTP_RETRIES = 3
DEFAULT_HTTP_RETRY_DELAY = 2.0
DB_QUERY = """
SELECT
@@ -101,6 +99,11 @@ FIELD_TYPE_HINTS = {
}
def log(message: str, *, level: str = "INFO") -> None:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"[{timestamp}] [{level}] {message}")
def load_env(path: Path) -> None:
if not path.exists():
return
@@ -154,6 +157,8 @@ def feishu_request(
*,
token: Optional[str] = None,
payload: Optional[dict] = None,
retries: int = DEFAULT_HTTP_RETRIES,
retry_delay: float = DEFAULT_HTTP_RETRY_DELAY,
) -> dict:
headers = {"Content-Type": "application/json; charset=utf-8"}
if token:
@@ -163,18 +168,53 @@ def feishu_request(
if payload is not None:
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
req = request.Request(url=url, method=method, headers=headers, data=data)
try:
with request.urlopen(req, timeout=60) as resp:
body = resp.read().decode("utf-8")
except error.HTTPError as exc:
detail = exc.read().decode("utf-8", errors="replace")
raise RuntimeError(f"Feishu API 请求失败: {exc.code} {detail}") from exc
except error.URLError as exc:
raise RuntimeError(f"无法连接飞书 API: {exc}") from exc
for attempt in range(1, retries + 1):
req = request.Request(url=url, method=method, headers=headers, data=data)
try:
with request.urlopen(req, timeout=60) as resp:
body = resp.read().decode("utf-8")
break
except error.HTTPError as exc:
detail = exc.read().decode("utf-8", errors="replace")
should_retry = exc.code in {429, 500, 502, 503, 504}
log(
f"Feishu API HTTP 异常: method={method} status={exc.code} "
f"url={url} attempt={attempt}/{retries} detail={detail[:500]}",
level="ERROR",
)
if should_retry and attempt < retries:
sleep_seconds = retry_delay * attempt
log(
f"Feishu API 将重试: method={method} status={exc.code} "
f"next_retry_in={sleep_seconds:.1f}s",
level="WARN",
)
time.sleep(sleep_seconds)
continue
raise RuntimeError(f"Feishu API 请求失败: {exc.code} {detail}") from exc
except error.URLError as exc:
log(
f"Feishu API 网络异常: method={method} url={url} "
f"attempt={attempt}/{retries} error={exc}",
level="ERROR",
)
if attempt < retries:
sleep_seconds = retry_delay * attempt
log(
f"Feishu API 网络异常后重试: method={method} next_retry_in={sleep_seconds:.1f}s",
level="WARN",
)
time.sleep(sleep_seconds)
continue
raise RuntimeError(f"无法连接飞书 API: {exc}") from exc
result = json.loads(body)
if result.get("code") != 0:
log(
f"Feishu API 业务异常: method={method} url={url} "
f"code={result.get('code')} msg={result.get('msg')}",
level="ERROR",
)
raise RuntimeError(
f"Feishu API 返回错误: code={result.get('code')} msg={result.get('msg')} body={body}"
)
@@ -196,6 +236,44 @@ def get_tenant_access_token(app_id: str, app_secret: str) -> str:
return result["tenant_access_token"]
def send_feishu_group_message(text: str) -> None:
app_id = os.getenv("FEISHU_APP_ID")
app_secret = os.getenv("FEISHU_APP_SECRET")
chat_id = os.getenv("FEISHU_CHAT_ID")
if not app_id or not app_secret or not chat_id:
raise RuntimeError("缺少 FEISHU_APP_ID / FEISHU_APP_SECRET / FEISHU_CHAT_ID")
token = get_tenant_access_token(app_id, app_secret)
url = "https://open.feishu.cn/open-apis/im/v1/messages?receive_id_type=chat_id"
payload = {
"receive_id": chat_id,
"msg_type": "text",
"content": json.dumps({"text": text}, ensure_ascii=False),
}
feishu_request("POST", url, token=token, payload=payload)
def build_success_message(summary: Dict[str, int]) -> str:
base_url = os.getenv("FEISHU_BASE_URL", "")
started_at = summary.get("started_at", "")
finished_at = summary.get("finished_at", "")
message = (
"截止到中午12点，超48小时未发货的订单数据同步完成\n"
f"新增记录数: {summary['created']}\n"
)
if base_url:
message += f"\n多维表链接: {base_url}"
return message
def build_failure_message(error_text: str, finished_at: str) -> str:
return (
"物流多维表同步失败\n"
f"执行时间: {finished_at}\n"
f"错误信息: {error_text}"
)
def paginate_fields(app_token: str, table_id: str, token: str) -> List[dict]:
fields: List[dict] = []
page_token: Optional[str] = None
@@ -210,14 +288,45 @@ def paginate_fields(app_token: str, table_id: str, token: str) -> List[dict]:
)
result = feishu_request("GET", url, token=token)
data = get_response_data(result)
fields.extend(data.get("items", []))
items = data.get("items", [])
fields.extend(items)
if not data.get("has_more"):
break
page_token = data.get("page_token")
log(f"字段读取完成: total={len(fields)}")
return fields
def list_all_record_ids(app_token: str, table_id: str, token: str) -> List[str]:
record_ids: List[str] = []
page_token: Optional[str] = None
while True:
query = {"page_size": 100}
if page_token:
query["page_token"] = page_token
url = (
f"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}"
f"/tables/{table_id}/records?{parse.urlencode(query)}"
)
result = feishu_request("GET", url, token=token)
data = get_response_data(result)
items = data.get("items", [])
for item in items:
record_id = item.get("record_id") or item.get("id")
if record_id:
record_ids.append(record_id)
if not data.get("has_more"):
break
page_token = data.get("page_token")
if not page_token:
break
log(f"目标表现有记录统计: total={len(record_ids)}")
return record_ids
def create_field(app_token: str, table_id: str, token: str, field_name: str, field_type: int) -> dict:
url = f"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/fields"
payload = {
@@ -232,15 +341,6 @@ def create_field(app_token: str, table_id: str, token: str, field_name: str, fie
return field
def normalize_value(raw: Optional[str]) -> Optional[str]:
if raw is None:
return None
value = raw.strip()
if not value or value.upper() == "NULL":
return None
return value
def convert_for_field(value: Optional[str], field_type: int):
if value is None:
return None
@@ -264,31 +364,24 @@ def convert_for_field(value: Optional[str], field_type: int):
return value.lower() in {"1", "true", "yes", "y", "是"}
if field_type == FIELD_TYPE_DATETIME:
# CSV 中是 `2026-03-15 14:20:30+08` 这种格式，转换成毫秒时间戳。
try:
dt = datetime.strptime(value, "%Y-%m-%d %H:%M:%S+08")
return int(dt.timestamp() * 1000)
except ValueError:
return value
for pattern in ("%Y-%m-%d", "%Y-%m-%d %H:%M:%S+08", "%Y-%m-%d %H:%M:%S"):
try:
dt = datetime.strptime(value, pattern)
if pattern == "%Y-%m-%d":
dt = datetime(dt.year, dt.month, dt.day)
return int(dt.timestamp() * 1000)
except ValueError:
continue
return value
return value
def read_csv_rows(csv_path: Path) -> Tuple[List[str], List[Dict[str, Optional[str]]]]:
with csv_path.open("r", newline="", encoding="utf-8-sig") as handle:
reader = csv.DictReader(handle)
headers = reader.fieldnames or []
rows: List[Dict[str, Optional[str]]] = []
for row in reader:
rows.append({key: normalize_value(value) for key, value in row.items()})
return headers, rows
def normalize_db_value(value) -> Optional[str]:
if value is None:
return None
if isinstance(value, datetime):
return value.strftime("%Y-%m-%d %H:%M:%S+08")
return value.strftime("%Y-%m-%d")
return str(value).strip() or None
@@ -342,10 +435,10 @@ def ensure_fields(
FIELD_TYPE_HINTS.get(header, FIELD_TYPE_TEXT),
)
field_map[header] = created
print(f"已创建字段: {header}")
log(f"字段已创建: field_name={header}")
except RuntimeError as exc:
missing_headers.append(header)
print(f"跳过缺失字段: {header} ({exc})")
log(f"字段创建失败，已跳过: field_name={header} error={exc}", level="WARN")
return field_map, missing_headers
@@ -390,70 +483,127 @@ def batch_create_records(
total_batches = (len(records) + batch_size - 1) // batch_size
for batch_index, batch in enumerate(chunked(records, batch_size), start=1):
log(f"开始写入批次: batch={batch_index}/{total_batches} size={len(batch)}")
feishu_request("POST", url, token=token, payload={"records": batch})
created += len(batch)
print(f"已写入批次 {batch_index}/{total_batches}，累计 {created} 条")
log(f"写入批次完成: batch={batch_index}/{total_batches} created_total={created}")
time.sleep(0.15)
return created
def batch_delete_records(
app_token: str,
table_id: str,
token: str,
record_ids: List[str],
batch_size: int,
) -> int:
if not record_ids:
log("目标表当前没有历史记录，无需删除")
return 0
url = f"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/batch_delete"
deleted = 0
total_batches = (len(record_ids) + batch_size - 1) // batch_size
for batch_index, batch in enumerate(chunked(record_ids, batch_size), start=1):
log(f"开始删除批次: batch={batch_index}/{total_batches} size={len(batch)}")
result = feishu_request("POST", url, token=token, payload={"records": batch})
data = get_response_data(result)
response_records = data.get("records", [])
if response_records:
failed = [item.get("record_id") for item in response_records if not item.get("deleted", False)]
if failed:
log(f"删除批次存在失败记录: failed_count={len(failed)}", level="ERROR")
raise RuntimeError(f"删除记录失败，record_ids={failed}")
deleted += len(batch)
log(f"删除批次完成: batch={batch_index}/{total_batches} deleted_total={deleted}")
time.sleep(0.15)
return deleted
def print_permission_help(message: str) -> None:
print("飞书返回权限不足，当前应用还不能向这个多维表新增字段或记录。", file=sys.stderr)
print(f"原始错误: {message}", file=sys.stderr)
print(
log("飞书返回权限不足，当前应用还不能向这个多维表新增字段或记录。", level="ERROR")
log(f"原始错误: {message}", level="ERROR")
log(
"请在飞书开放平台或多维表权限设置中，确认这个应用已被添加到该 Base，且具备多维表读取/写入权限。",
file=sys.stderr,
level="ERROR",
)
print(
log(
"如果你手动补齐缺失字段并授权写入，重新执行 `python3 import_logistics_to_feishu_bitable.py` 即可继续导入。",
file=sys.stderr,
level="ERROR",
)
def main() -> int:
def execute_sync() -> Dict[str, int]:
load_env(ENV_PATH)
app_id = os.getenv("FEISHU_APP_ID")
app_secret = os.getenv("FEISHU_APP_SECRET")
base_url = os.getenv("FEISHU_BASE_URL")
source = os.getenv("DATA_SOURCE", DEFAULT_SOURCE).strip().lower()
csv_path = Path(os.getenv("CSV_PATH", str(DEFAULT_CSV_PATH)))
batch_size = int(os.getenv("BATCH_SIZE", str(DEFAULT_BATCH_SIZE)))
if not app_id or not app_secret or not base_url:
print("缺少 FEISHU_APP_ID / FEISHU_APP_SECRET / FEISHU_BASE_URL", file=sys.stderr)
return 1
if source not in {"db", "csv"}:
print(f"不支持的数据源: {source}，仅支持 db / csv", file=sys.stderr)
return 1
if source == "csv" and not csv_path.exists():
print(f"CSV 文件不存在: {csv_path}", file=sys.stderr)
return 1
raise RuntimeError("缺少 FEISHU_APP_ID / FEISHU_APP_SECRET / FEISHU_BASE_URL")
started_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
app_token, table_id, _view_id = parse_base_url(base_url)
if source == "db":
headers, rows = query_db_rows(DB_QUERY)
print(f"读取数据库完成，共 {len(rows)} 行")
else:
headers, rows = read_csv_rows(csv_path)
print(f"读取 CSV 完成: {csv_path}，共 {len(rows)} 行")
log(f"导入任务开始: source=db table_id={table_id}")
headers, rows = query_db_rows(DB_QUERY)
log(f"读取数据库完成: rows={len(rows)}")
try:
token = get_tenant_access_token(app_id, app_secret)
log("已获取 tenant_access_token")
field_map, missing_headers = ensure_fields(app_token, table_id, token, headers)
log(f"字段检查完成: field_count={len(field_map)} missing_count={len(missing_headers)}")
existing_record_ids = list_all_record_ids(app_token, table_id, token)
deleted = batch_delete_records(app_token, table_id, token, existing_record_ids, batch_size)
records = build_records(rows, field_map, missing_headers)
log(f"记录构建完成: source_rows={len(rows)} upload_records={len(records)}")
created = batch_create_records(app_token, table_id, token, records, batch_size)
except RuntimeError as exc:
if "91403" in str(exc) or "Forbidden" in str(exc):
print_permission_help(str(exc))
return 2
raise RuntimeError("飞书多维表权限不足") from exc
log(f"导入任务失败: error={exc}", level="ERROR")
raise
if missing_headers:
print(f"以下字段未写入，请在多维表中手动创建后重跑: {', '.join(missing_headers)}")
print(f"导入完成，共写入 {created} 条记录")
return 0
log(f"存在未写入字段: missing_count={len(missing_headers)}", level="WARN")
log(f"导入任务完成: deleted={deleted} created={created} source_rows={len(rows)}")
return {
"started_at": started_at,
"finished_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"deleted": deleted,
"created": created,
"source_rows": len(rows),
"missing_count": len(missing_headers),
}
def main() -> int:
try:
summary = execute_sync()
try:
send_feishu_group_message(build_success_message(summary))
log("飞书群通知发送成功")
except Exception as notify_exc:
log(f"飞书群通知发送失败: error={notify_exc}", level="ERROR")
return 0
except RuntimeError as exc:
try:
send_feishu_group_message(
build_failure_message(str(exc), datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
)
log("失败通知已发送到飞书群", level="WARN")
except Exception as notify_exc:
log(f"失败通知发送失败: error={notify_exc}", level="ERROR")
if str(exc) == "飞书多维表权限不足":
return 2
raise
if __name__ == "__main__":