添加 委外批次进数据库、委外批次入飞书

upload_inventory_page.py

@@ -10,22 +10,186 @@ Then open:
 
 from __future__ import annotations
 
-import cgi
+from dataclasses import dataclass
+from email import policy
+from email.parser import BytesParser
+import json
+import logging
+from logging.handlers import RotatingFileHandler
 from datetime import datetime
 from html import escape
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+import os
 from pathlib import Path
 import re
-import shutil
-from typing import Iterable
+import sys
+import time
+from typing import Any, Iterable
 from urllib.parse import parse_qs, urlparse
 
+import pandas as pd
+import psycopg2
+from dotenv import load_dotenv
+
 
 HOST = "127.0.0.1"
 PORT = 8080
 PROJECT_ROOT = Path(__file__).resolve().parent
 UPLOAD_DIR = PROJECT_ROOT / "batch_inventory_data"
+LOG_DIR = PROJECT_ROOT / "logs"
+LOG_FILE = LOG_DIR / "upload_inventory_page.log"
 MAX_UPLOAD_BYTES = 200 * 1024 * 1024
+DB_SCHEMA = "raw_data"
+DB_TABLE = "wangdian_batch_inventory"
+DATA_FIELD = "inventory_data"
+LOGGER_NAME = "inventory_upload"
+
+
+logger = logging.getLogger(LOGGER_NAME)
+SENSITIVE_PATTERN = re.compile(r"(?i)(password|passwd|secret|token|authorization)\s*[:=]\s*([^\s,;]+)")
+
+LOG_FIELD_LABELS = {
+    "advanced_permission_enabled": "高级权限已开启",
+    "app_token": "应用Token",
+    "bytes": "字节数",
+    "client": "客户端",
+    "columns": "列数",
+    "content_type": "内容类型",
+    "database": "数据库",
+    "db_schema": "数据库Schema",
+    "db_table": "数据库表",
+    "elapsed_ms": "耗时毫秒",
+    "encoding": "编码",
+    "env_path": "环境文件",
+    "error": "错误",
+    "field_name": "字段名",
+    "host": "主机",
+    "is_error": "是否错误",
+    "max_bytes": "最大字节数",
+    "max_upload_bytes": "最大上传字节数",
+    "message": "消息",
+    "log_file": "日志文件",
+    "original_filename": "原始文件名",
+    "path": "路径",
+    "port": "端口",
+    "rows": "行数",
+    "payload": "写入内容",
+    "schema": "Schema",
+    "sheet_name": "工作表",
+    "sheets": "工作表数",
+    "source_file": "源文件",
+    "stored_file": "归档文件",
+    "stored_filename": "归档文件名",
+    "suffix": "文件后缀",
+    "table": "表名",
+    "target_path": "目标路径",
+    "upload_dir": "上传目录",
+    "url": "地址",
+}
+
+
+def configure_logging() -> None:
+    LOG_DIR.mkdir(parents=True, exist_ok=True)
+    formatter = logging.Formatter(
+        fmt="%(asctime)s %(levelname)s %(name)s %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setFormatter(formatter)
+    file_handler = RotatingFileHandler(LOG_FILE, maxBytes=5 * 1024 * 1024, backupCount=5, encoding="utf-8")
+    file_handler.setFormatter(formatter)
+    root_logger = logging.getLogger()
+    root_logger.handlers.clear()
+    root_logger.setLevel(logging.INFO)
+    root_logger.addHandler(stream_handler)
+    root_logger.addHandler(file_handler)
+
+
+def sanitize_value(value: Any) -> Any:
+    if isinstance(value, str):
+        return SENSITIVE_PATTERN.sub(r"\1=***", value)
+    return value
+
+
+def log_event(level: int, event: str, **fields: Any) -> None:
+    safe_event = sanitize_value(event)
+    details = " ".join(
+        f"{LOG_FIELD_LABELS.get(key, key)}={sanitize_value(value)!r}" for key, value in fields.items()
+    )
+    logger.log(level, "事件=%s%s", safe_event, f" {details}" if details else "")
+
+
+@dataclass(frozen=True)
+class UploadedFile:
+    filename: str
+    content: bytes
+
+    @property
+    def size(self) -> int:
+        return len(self.content)
+
+
+@dataclass(frozen=True)
+class RawDbConfig:
+    host: str
+    port: str
+    user: str
+    password: str
+    name: str
+
+    @classmethod
+    def from_env(cls) -> "RawDbConfig":
+        env_path = PROJECT_ROOT / ".env"
+        load_dotenv(env_path)
+        values = {
+            "host": os.getenv("RAW_DB_HOST", ""),
+            "port": os.getenv("RAW_DB_PORT", ""),
+            "user": os.getenv("RAW_DB_USER", ""),
+            "password": os.getenv("RAW_DB_PASSWORD", ""),
+            "name": os.getenv("RAW_DB_NAME", ""),
+        }
+        missing = [f"RAW_DB_{key.upper()}" for key, value in values.items() if not value]
+        if missing:
+            raise RuntimeError(f"缺少 Raw 数据库配置: {', '.join(missing)}")
+        log_event(
+            logging.INFO,
+            "已加载Raw数据库配置",
+            env_path=env_path,
+            database=values["name"],
+        )
+        return cls(**values)
+
+    def connect(self):
+        log_event(
+            logging.INFO,
+            "开始连接Raw数据库",
+            database=self.name,
+        )
+        return psycopg2.connect(
+            host=self.host,
+            port=self.port,
+            user=self.user,
+            password=self.password,
+            dbname=self.name,
+        )
+
+
+def get_create_table_sql(schema_name: str, table_name: str) -> str:
+    full_table_name = f"{schema_name}.{table_name}"
+    return f"""
+    CREATE SCHEMA IF NOT EXISTS {schema_name};
+
+    CREATE TABLE IF NOT EXISTS {full_table_name} (
+        payload JSONB NOT NULL,
+        start_time TIMESTAMP WITH TIME ZONE,
+        end_time TIMESTAMP WITH TIME ZONE,
+        created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+        reserved_1 TEXT,
+        reserved_2 TEXT
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_{table_name}_created_at ON {full_table_name} (created_at);
+    """
 
 
 def safe_filename(filename: str) -> str:
@@ -44,9 +208,176 @@ def unique_upload_path(filename: str) -> Path:
     while candidate.exists():
         candidate = UPLOAD_DIR / f"{timestamp}_{counter}_{base_name}"
         counter += 1
+    log_event(
+        logging.INFO,
+        "已分配上传归档路径",
+        original_filename=filename,
+        stored_filename=candidate.name,
+        target_path=candidate,
+    )
     return candidate
 
 
+def parse_inventory_upload(headers: Any, body: bytes) -> UploadedFile | None:
+    content_type = headers.get("Content-Type", "")
+    if "multipart/form-data" not in content_type:
+        log_event(logging.WARNING, "上传请求内容类型不支持", content_type=content_type)
+        return None
+
+    message = BytesParser(policy=policy.default).parsebytes(
+        b"Content-Type: " + content_type.encode("utf-8") + b"\r\n"
+        b"MIME-Version: 1.0\r\n\r\n"
+        + body
+    )
+    for part in message.iter_parts():
+        if part.get_param("name", header="content-disposition") != "inventory_file":
+            continue
+        filename = part.get_filename()
+        content = part.get_payload(decode=True)
+        if filename and content is not None:
+            upload = UploadedFile(filename=filename, content=content)
+            log_event(
+                logging.INFO,
+                "已解析上传文件",
+                filename=upload.filename,
+                bytes=upload.size,
+                content_type=part.get_content_type(),
+            )
+            return upload
+    log_event(logging.WARNING, "上传表单缺少文件字段", field_name="inventory_file")
+    return None
+
+
+def normalize_cell(value: Any) -> Any:
+    if pd.isna(value):
+        return None
+    if hasattr(value, "isoformat"):
+        return value.isoformat()
+    return value
+
+
+def dataframe_to_rows(frame: pd.DataFrame, sheet_name: str | None = None) -> list[dict[str, Any]]:
+    frame = frame.dropna(how="all")
+    rows: list[dict[str, Any]] = []
+    for record in frame.to_dict(orient="records"):
+        row = {str(key): normalize_cell(value) for key, value in record.items()}
+        if sheet_name is not None:
+            row["_sheet_name"] = sheet_name
+        rows.append(row)
+    return rows
+
+
+def read_inventory_rows(file_path: Path) -> list[dict[str, Any]]:
+    suffix = file_path.suffix.lower()
+    log_event(logging.INFO, "开始解析库存文件", path=file_path, suffix=suffix)
+    if suffix in {".xlsx", ".xlsm", ".xls"}:
+        sheets = pd.read_excel(file_path, sheet_name=None, dtype=object)
+        include_sheet_name = len(sheets) > 1
+        rows: list[dict[str, Any]] = []
+        for sheet_name, frame in sheets.items():
+            sheet_rows = dataframe_to_rows(frame, sheet_name if include_sheet_name else None)
+            rows.extend(sheet_rows)
+            log_event(
+                logging.INFO,
+                "已解析Excel工作表",
+                path=file_path.name,
+                sheet_name=sheet_name,
+                rows=len(sheet_rows),
+                columns=len(frame.columns),
+            )
+        log_event(logging.INFO, "库存文件解析完成", path=file_path.name, rows=len(rows), sheets=len(sheets))
+        return rows
+
+    if suffix == ".csv":
+        last_error: Exception | None = None
+        for encoding in ("utf-8-sig", "gb18030"):
+            try:
+                frame = pd.read_csv(file_path, dtype=object, encoding=encoding)
+                rows = dataframe_to_rows(frame)
+                log_event(
+                    logging.INFO,
+                    "已解析CSV文件",
+                    path=file_path.name,
+                    rows=len(rows),
+                    columns=len(frame.columns),
+                    encoding=encoding,
+                )
+                return rows
+            except UnicodeDecodeError as exc:
+                last_error = exc
+                log_event(logging.WARNING, "CSV编码解析失败", path=file_path.name, encoding=encoding, error=exc)
+        if last_error is not None:
+            raise last_error
+
+    raise ValueError("仅支持 Excel 或 CSV 文件写入数据库。")
+
+
+def save_inventory_file_to_raw_db(file_path: Path, original_filename: str) -> int:
+    started_at = time.monotonic()
+    rows = read_inventory_rows(file_path)
+    if not rows:
+        log_event(logging.WARNING, "文件无数据跳过Raw写入", path=file_path.name)
+        return 0
+
+    config = RawDbConfig.from_env()
+    conn = config.connect()
+    try:
+        with conn:
+            with conn.cursor() as cur:
+                log_event(logging.INFO, "开始确认Raw数据表", schema=DB_SCHEMA, table=DB_TABLE)
+                cur.execute(get_create_table_sql(DB_SCHEMA, DB_TABLE))
+                log_event(logging.INFO, "Raw数据表确认完成", schema=DB_SCHEMA, table=DB_TABLE)
+                upload_time = datetime.now().isoformat(timespec="seconds")
+                total_count = len(rows)
+                log_event(
+                    logging.INFO,
+                    "开始写入Raw数据",
+                    schema=DB_SCHEMA,
+                    table=DB_TABLE,
+                    source_file=original_filename,
+                    stored_file=file_path.name,
+                    rows=total_count,
+                )
+                for item in rows:
+                    payload = {
+                        "source_file": original_filename,
+                        "stored_file": file_path.name,
+                        "uploaded_at": upload_time,
+                        "total_count": total_count,
+                        DATA_FIELD: item,
+                    }
+                    log_event(
+                        logging.INFO,
+                        "写入Raw数据库单行数据",
+                        stored_file=file_path.name,
+                        payload=json.dumps(payload, ensure_ascii=False),
+                    )
+                    cur.execute(
+                        f"""
+                        INSERT INTO {DB_SCHEMA}.{DB_TABLE} (
+                            payload, start_time, end_time
+                        ) VALUES (
+                            %s::jsonb, %s, %s
+                        );
+                        """,
+                        (json.dumps(payload, ensure_ascii=False), None, None),
+                    )
+        elapsed_ms = int((time.monotonic() - started_at) * 1000)
+        log_event(
+            logging.INFO,
+            "Raw数据写入完成",
+            schema=DB_SCHEMA,
+            table=DB_TABLE,
+            stored_file=file_path.name,
+            rows=len(rows),
+            elapsed_ms=elapsed_ms,
+        )
+        return len(rows)
+    finally:
+        conn.close()
+        log_event(logging.INFO, "Raw数据库连接已关闭", schema=DB_SCHEMA, table=DB_TABLE)
+
+
 def list_uploaded_files(limit: int = 20) -> list[Path]:
     UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
     files = [path for path in UPLOAD_DIR.iterdir() if path.is_file() and not path.name.startswith(".")]
@@ -597,11 +928,13 @@ class UploadHandler(BaseHTTPRequestHandler):
 
     def do_GET(self) -> None:
         parsed = urlparse(self.path)
+        log_event(logging.INFO, "收到GET请求", client=self.client_address[0], path=parsed.path)
         if parsed.path == "/favicon.ico":
             self.send_response(204)
             self.end_headers()
             return
         if parsed.path != "/":
+            log_event(logging.WARNING, "请求路径不存在", client=self.client_address[0], path=parsed.path)
             self.send_error(404, "Not Found")
             return
 
@@ -612,33 +945,69 @@ class UploadHandler(BaseHTTPRequestHandler):
 
     def do_POST(self) -> None:
         path = urlparse(self.path).path
+        request_started_at = time.monotonic()
+        log_event(logging.INFO, "开始处理POST请求", client=self.client_address[0], path=path)
         if path != "/upload":
+            log_event(logging.WARNING, "请求路径不存在", client=self.client_address[0], path=path)
             self.send_error(404, "Not Found")
             return
 
         content_length = int(self.headers.get("Content-Length", "0") or "0")
+        log_event(logging.INFO, "收到上传请求", client=self.client_address[0], bytes=content_length)
         if content_length <= 0:
+            log_event(logging.WARNING, "上传请求为空", client=self.client_address[0])
             self.redirect("/", "没有收到上传文件。", is_error=True)
             return
         if content_length > MAX_UPLOAD_BYTES:
+            log_event(
+                logging.WARNING,
+                "上传文件超过大小限制",
+                client=self.client_address[0],
+                bytes=content_length,
+                max_bytes=MAX_UPLOAD_BYTES,
+            )
             self.redirect("/", f"文件太大，最大支持 {human_size(MAX_UPLOAD_BYTES)}。", is_error=True)
             return
 
-        form = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={
-            "REQUEST_METHOD": "POST",
-            "CONTENT_TYPE": self.headers.get("Content-Type", ""),
-            "CONTENT_LENGTH": str(content_length),
-        })
-        file_item = form["inventory_file"] if "inventory_file" in form else None
-        if file_item is None or not getattr(file_item, "filename", ""):
+        upload = parse_inventory_upload(self.headers, self.rfile.read(content_length))
+        if upload is None:
             self.redirect("/", "请选择需要上传的文件。", is_error=True)
             return
 
-        target_path = unique_upload_path(file_item.filename)
+        target_path = unique_upload_path(upload.filename)
         with target_path.open("wb") as output:
-            shutil.copyfileobj(file_item.file, output)
+            output.write(upload.content)
+        log_event(
+            logging.INFO,
+            "上传文件已保存",
+            original_filename=upload.filename,
+            stored_file=target_path.name,
+            path=target_path,
+            bytes=upload.size,
+        )
 
-        self.redirect("/", f"上传成功：{target_path.name}")
+        try:
+            saved_rows = save_inventory_file_to_raw_db(target_path, upload.filename)
+        except Exception as exc:
+            log_event(
+                logging.ERROR,
+                "上传文件写入Raw数据库失败",
+                stored_file=target_path.name,
+                error=exc,
+                elapsed_ms=int((time.monotonic() - request_started_at) * 1000),
+            )
+            self.redirect("/", f"文件已保存，但写入数据库失败：{exc}", is_error=True)
+            return
+
+        log_event(
+            logging.INFO,
+            "上传请求处理完成",
+            original_filename=upload.filename,
+            stored_file=target_path.name,
+            rows=saved_rows,
+            elapsed_ms=int((time.monotonic() - request_started_at) * 1000),
+        )
+        self.redirect("/", f"上传成功：{target_path.name}，已写入 {saved_rows} 行到 {DB_SCHEMA}.{DB_TABLE}")
 
     def respond_html(self, body: bytes) -> None:
         self.send_response(200)
@@ -650,27 +1019,39 @@ class UploadHandler(BaseHTTPRequestHandler):
     def redirect(self, path: str, message: str, is_error: bool = False) -> None:
         from urllib.parse import urlencode
 
+        log_event(logging.INFO, "页面重定向", path=path, is_error=is_error, message=message)
         query = urlencode({"message": message, "error": "1" if is_error else "0"})
         self.send_response(303)
         self.send_header("Location", f"{path}?{query}")
         self.end_headers()
 
     def log_message(self, format: str, *args: object) -> None:
-        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f"[{timestamp}] {self.address_string()} - {format % args}")
+        log_event(logging.INFO, "HTTP服务日志", client=self.address_string(), message=format % args)
 
 
 def main() -> int:
+    configure_logging()
     UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
+    log_event(
+        logging.INFO,
+        "上传服务启动中",
+        host=HOST,
+        port=PORT,
+        upload_dir=UPLOAD_DIR,
+        max_upload_bytes=MAX_UPLOAD_BYTES,
+        db_schema=DB_SCHEMA,
+        db_table=DB_TABLE,
+        log_file=LOG_FILE,
+    )
     server = ThreadingHTTPServer((HOST, PORT), UploadHandler)
-    print(f"上传页面已启动: http://{HOST}:{PORT}")
-    print(f"文件保存目录: {UPLOAD_DIR}")
+    log_event(logging.INFO, "上传服务已启动", url=f"http://{HOST}:{PORT}", upload_dir=UPLOAD_DIR, log_file=LOG_FILE)
     try:
         server.serve_forever()
     except KeyboardInterrupt:
-        print("\n上传页面已停止。")
+        log_event(logging.INFO, "收到键盘中断准备停止服务")
     finally:
         server.server_close()
+        log_event(logging.INFO, "上传服务已停止", host=HOST, port=PORT)
     return 0