#!/usr/bin/env python3 """Local inventory file upload page. Run: python3 upload_inventory_page.py Then open: http://127.0.0.1:8080 """ from __future__ import annotations from dataclasses import dataclass from email import policy from email.parser import BytesParser import json import logging from logging.handlers import RotatingFileHandler from datetime import datetime from html import escape from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import os from pathlib import Path import re import sys import time from typing import Any, Iterable from urllib.parse import parse_qs, urlparse import pandas as pd import psycopg2 from dotenv import load_dotenv HOST = "127.0.0.1" PORT = 8080 PROJECT_ROOT = Path(__file__).resolve().parent UPLOAD_DIR = PROJECT_ROOT / "batch_inventory_data" LOG_DIR = PROJECT_ROOT / "logs" LOG_FILE = LOG_DIR / "upload_inventory_page.log" MAX_UPLOAD_BYTES = 200 * 1024 * 1024 DB_SCHEMA = "raw_data" DB_TABLE = "wangdian_batch_inventory" DATA_FIELD = "inventory_data" LOGGER_NAME = "inventory_upload" logger = logging.getLogger(LOGGER_NAME) SENSITIVE_PATTERN = re.compile(r"(?i)(password|passwd|secret|token|authorization)\s*[:=]\s*([^\s,;]+)") LOG_FIELD_LABELS = { "advanced_permission_enabled": "高级权限已开启", "app_token": "应用Token", "bytes": "字节数", "client": "客户端", "columns": "列数", "content_type": "内容类型", "database": "数据库", "db_schema": "数据库Schema", "db_table": "数据库表", "elapsed_ms": "耗时毫秒", "encoding": "编码", "env_path": "环境文件", "error": "错误", "field_name": "字段名", "host": "主机", "is_error": "是否错误", "max_bytes": "最大字节数", "max_upload_bytes": "最大上传字节数", "message": "消息", "log_file": "日志文件", "original_filename": "原始文件名", "path": "路径", "port": "端口", "rows": "行数", "payload": "写入内容", "schema": "Schema", "sheet_name": "工作表", "sheets": "工作表数", "source_file": "源文件", "stored_file": "归档文件", "stored_filename": "归档文件名", "suffix": "文件后缀", "table": "表名", "target_path": "目标路径", "upload_dir": "上传目录", "url": "地址", } def configure_logging() -> None: LOG_DIR.mkdir(parents=True, exist_ok=True) formatter = logging.Formatter( fmt="%(asctime)s %(levelname)s %(name)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", ) stream_handler = logging.StreamHandler(sys.stdout) stream_handler.setFormatter(formatter) file_handler = RotatingFileHandler(LOG_FILE, maxBytes=5 * 1024 * 1024, backupCount=5, encoding="utf-8") file_handler.setFormatter(formatter) root_logger = logging.getLogger() root_logger.handlers.clear() root_logger.setLevel(logging.INFO) root_logger.addHandler(stream_handler) root_logger.addHandler(file_handler) def sanitize_value(value: Any) -> Any: if isinstance(value, str): return SENSITIVE_PATTERN.sub(r"\1=***", value) return value def log_event(level: int, event: str, **fields: Any) -> None: safe_event = sanitize_value(event) details = " ".join( f"{LOG_FIELD_LABELS.get(key, key)}={sanitize_value(value)!r}" for key, value in fields.items() ) logger.log(level, "事件=%s%s", safe_event, f" {details}" if details else "") @dataclass(frozen=True) class UploadedFile: filename: str content: bytes @property def size(self) -> int: return len(self.content) @dataclass(frozen=True) class RawDbConfig: host: str port: str user: str password: str name: str @classmethod def from_env(cls) -> "RawDbConfig": env_path = PROJECT_ROOT / ".env" load_dotenv(env_path) values = { "host": os.getenv("RAW_DB_HOST", ""), "port": os.getenv("RAW_DB_PORT", ""), "user": os.getenv("RAW_DB_USER", ""), "password": os.getenv("RAW_DB_PASSWORD", ""), "name": os.getenv("RAW_DB_NAME", ""), } missing = [f"RAW_DB_{key.upper()}" for key, value in values.items() if not value] if missing: raise RuntimeError(f"缺少 Raw 数据库配置: {', '.join(missing)}") log_event( logging.INFO, "已加载Raw数据库配置", env_path=env_path, database=values["name"], ) return cls(**values) def connect(self): log_event( logging.INFO, "开始连接Raw数据库", database=self.name, ) return psycopg2.connect( host=self.host, port=self.port, user=self.user, password=self.password, dbname=self.name, ) def get_create_table_sql(schema_name: str, table_name: str) -> str: full_table_name = f"{schema_name}.{table_name}" return f""" CREATE SCHEMA IF NOT EXISTS {schema_name}; CREATE TABLE IF NOT EXISTS {full_table_name} ( payload JSONB NOT NULL, start_time TIMESTAMP WITH TIME ZONE, end_time TIMESTAMP WITH TIME ZONE, created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), reserved_1 TEXT, reserved_2 TEXT ); CREATE INDEX IF NOT EXISTS idx_{table_name}_created_at ON {full_table_name} (created_at); """ def safe_filename(filename: str) -> str: name = Path(filename or "inventory_file").name.strip() name = re.sub(r"[^\w.\-\u4e00-\u9fff（）() ]+", "_", name) name = re.sub(r"\s+", "_", name) return name or "inventory_file" def unique_upload_path(filename: str) -> Path: UPLOAD_DIR.mkdir(parents=True, exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") base_name = safe_filename(filename) candidate = UPLOAD_DIR / f"{timestamp}_{base_name}" counter = 1 while candidate.exists(): candidate = UPLOAD_DIR / f"{timestamp}_{counter}_{base_name}" counter += 1 log_event( logging.INFO, "已分配上传归档路径", original_filename=filename, stored_filename=candidate.name, target_path=candidate, ) return candidate def parse_inventory_upload(headers: Any, body: bytes) -> UploadedFile | None: content_type = headers.get("Content-Type", "") if "multipart/form-data" not in content_type: log_event(logging.WARNING, "上传请求内容类型不支持", content_type=content_type) return None message = BytesParser(policy=policy.default).parsebytes( b"Content-Type: " + content_type.encode("utf-8") + b"\r\n" b"MIME-Version: 1.0\r\n\r\n" + body ) for part in message.iter_parts(): if part.get_param("name", header="content-disposition") != "inventory_file": continue filename = part.get_filename() content = part.get_payload(decode=True) if filename and content is not None: upload = UploadedFile(filename=filename, content=content) log_event( logging.INFO, "已解析上传文件", filename=upload.filename, bytes=upload.size, content_type=part.get_content_type(), ) return upload log_event(logging.WARNING, "上传表单缺少文件字段", field_name="inventory_file") return None def normalize_cell(value: Any) -> Any: if pd.isna(value): return None if hasattr(value, "isoformat"): return value.isoformat() return value def dataframe_to_rows(frame: pd.DataFrame, sheet_name: str | None = None) -> list[dict[str, Any]]: frame = frame.dropna(how="all") rows: list[dict[str, Any]] = [] for record in frame.to_dict(orient="records"): row = {str(key): normalize_cell(value) for key, value in record.items()} if sheet_name is not None: row["_sheet_name"] = sheet_name rows.append(row) return rows def read_inventory_rows(file_path: Path) -> list[dict[str, Any]]: suffix = file_path.suffix.lower() log_event(logging.INFO, "开始解析库存文件", path=file_path, suffix=suffix) if suffix in {".xlsx", ".xlsm", ".xls"}: sheets = pd.read_excel(file_path, sheet_name=None, dtype=object) include_sheet_name = len(sheets) > 1 rows: list[dict[str, Any]] = [] for sheet_name, frame in sheets.items(): sheet_rows = dataframe_to_rows(frame, sheet_name if include_sheet_name else None) rows.extend(sheet_rows) log_event( logging.INFO, "已解析Excel工作表", path=file_path.name, sheet_name=sheet_name, rows=len(sheet_rows), columns=len(frame.columns), ) log_event(logging.INFO, "库存文件解析完成", path=file_path.name, rows=len(rows), sheets=len(sheets)) return rows if suffix == ".csv": last_error: Exception | None = None for encoding in ("utf-8-sig", "gb18030"): try: frame = pd.read_csv(file_path, dtype=object, encoding=encoding) rows = dataframe_to_rows(frame) log_event( logging.INFO, "已解析CSV文件", path=file_path.name, rows=len(rows), columns=len(frame.columns), encoding=encoding, ) return rows except UnicodeDecodeError as exc: last_error = exc log_event(logging.WARNING, "CSV编码解析失败", path=file_path.name, encoding=encoding, error=exc) if last_error is not None: raise last_error raise ValueError("仅支持 Excel 或 CSV 文件写入数据库。") def save_inventory_file_to_raw_db(file_path: Path, original_filename: str) -> int: started_at = time.monotonic() rows = read_inventory_rows(file_path) if not rows: log_event(logging.WARNING, "文件无数据跳过Raw写入", path=file_path.name) return 0 config = RawDbConfig.from_env() conn = config.connect() try: with conn: with conn.cursor() as cur: log_event(logging.INFO, "开始确认Raw数据表", schema=DB_SCHEMA, table=DB_TABLE) cur.execute(get_create_table_sql(DB_SCHEMA, DB_TABLE)) log_event(logging.INFO, "Raw数据表确认完成", schema=DB_SCHEMA, table=DB_TABLE) upload_time = datetime.now().isoformat(timespec="seconds") total_count = len(rows) log_event( logging.INFO, "开始写入Raw数据", schema=DB_SCHEMA, table=DB_TABLE, source_file=original_filename, stored_file=file_path.name, rows=total_count, ) for item in rows: payload = { "source_file": original_filename, "stored_file": file_path.name, "uploaded_at": upload_time, "total_count": total_count, DATA_FIELD: item, } log_event( logging.INFO, "写入Raw数据库单行数据", stored_file=file_path.name, payload=json.dumps(payload, ensure_ascii=False), ) cur.execute( f""" INSERT INTO {DB_SCHEMA}.{DB_TABLE} ( payload, start_time, end_time ) VALUES ( %s::jsonb, %s, %s ); """, (json.dumps(payload, ensure_ascii=False), None, None), ) elapsed_ms = int((time.monotonic() - started_at) * 1000) log_event( logging.INFO, "Raw数据写入完成", schema=DB_SCHEMA, table=DB_TABLE, stored_file=file_path.name, rows=len(rows), elapsed_ms=elapsed_ms, ) return len(rows) finally: conn.close() log_event(logging.INFO, "Raw数据库连接已关闭", schema=DB_SCHEMA, table=DB_TABLE) def list_uploaded_files(limit: int = 20) -> list[Path]: UPLOAD_DIR.mkdir(parents=True, exist_ok=True) files = [path for path in UPLOAD_DIR.iterdir() if path.is_file() and not path.name.startswith(".")] return sorted(files, key=lambda item: item.stat().st_mtime, reverse=True)[:limit] def human_size(size: int) -> str: units = ["B", "KB", "MB", "GB"] value = float(size) for unit in units: if value < 1024 or unit == units[-1]: return f"{value:.1f} {unit}" if unit != "B" else f"{int(value)} {unit}" value /= 1024 return f"{size} B" def render_recent_files(files: Iterable[Path]) -> str: rows = [] for path in files: stat = path.stat() uploaded_at = datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M:%S") rows.append( "" f"{escape(path.name)}" f"{escape(human_size(stat.st_size))}" f"{escape(uploaded_at)}" "" ) if not rows: return '

还没有上传文件。上传成功后，最近文件会显示在这里。

' return ( '' "" f"{''.join(rows)}" "
文件名大小上传时间
" ) def render_page(message: str = "", is_error: bool = False) -> bytes: message_html = "" if message: message_class = "message error" if is_error else "message success" message_html = f'
{escape(message)}
' html = f""" 委外批次库存文件上传
委委外批次库存上传台
每周约 2 次使用

投递委外批次库存文件

拖入委外仓、加工方或批次盘点文件，系统会按上传时间自动归档，便于后续同步、核对和追溯。

ARCHIVE PATH {escape(str(UPLOAD_DIR))}
{message_html}
⇣
DROP FILE HERE

拖入文件，立即上传

支持 Excel、CSV 等委外批次库存文件。松开鼠标后自动保存到 batch_inventory_data，无需点击选择文件。

等待拖入文件

最近上传的委外批次文件

{render_recent_files(list_uploaded_files())}
""" return html.encode("utf-8") class UploadHandler(BaseHTTPRequestHandler): server_version = "InventoryUpload/1.0" def do_GET(self) -> None: parsed = urlparse(self.path) log_event(logging.INFO, "收到GET请求", client=self.client_address[0], path=parsed.path) if parsed.path == "/favicon.ico": self.send_response(204) self.end_headers() return if parsed.path != "/": log_event(logging.WARNING, "请求路径不存在", client=self.client_address[0], path=parsed.path) self.send_error(404, "Not Found") return params = parse_qs(parsed.query) message = params.get("message", [""])[0] is_error = params.get("error", ["0"])[0] == "1" self.respond_html(render_page(message, is_error)) def do_POST(self) -> None: path = urlparse(self.path).path request_started_at = time.monotonic() log_event(logging.INFO, "开始处理POST请求", client=self.client_address[0], path=path) if path != "/upload": log_event(logging.WARNING, "请求路径不存在", client=self.client_address[0], path=path) self.send_error(404, "Not Found") return content_length = int(self.headers.get("Content-Length", "0") or "0") log_event(logging.INFO, "收到上传请求", client=self.client_address[0], bytes=content_length) if content_length <= 0: log_event(logging.WARNING, "上传请求为空", client=self.client_address[0]) self.redirect("/", "没有收到上传文件。", is_error=True) return if content_length > MAX_UPLOAD_BYTES: log_event( logging.WARNING, "上传文件超过大小限制", client=self.client_address[0], bytes=content_length, max_bytes=MAX_UPLOAD_BYTES, ) self.redirect("/", f"文件太大，最大支持 {human_size(MAX_UPLOAD_BYTES)}。", is_error=True) return upload = parse_inventory_upload(self.headers, self.rfile.read(content_length)) if upload is None: self.redirect("/", "请选择需要上传的文件。", is_error=True) return target_path = unique_upload_path(upload.filename) with target_path.open("wb") as output: output.write(upload.content) log_event( logging.INFO, "上传文件已保存", original_filename=upload.filename, stored_file=target_path.name, path=target_path, bytes=upload.size, ) try: saved_rows = save_inventory_file_to_raw_db(target_path, upload.filename) except Exception as exc: log_event( logging.ERROR, "上传文件写入Raw数据库失败", stored_file=target_path.name, error=exc, elapsed_ms=int((time.monotonic() - request_started_at) * 1000), ) self.redirect("/", f"文件已保存，但写入数据库失败：{exc}", is_error=True) return log_event( logging.INFO, "上传请求处理完成", original_filename=upload.filename, stored_file=target_path.name, rows=saved_rows, elapsed_ms=int((time.monotonic() - request_started_at) * 1000), ) self.redirect("/", f"上传成功：{target_path.name}，已写入 {saved_rows} 行到 {DB_SCHEMA}.{DB_TABLE}") def respond_html(self, body: bytes) -> None: self.send_response(200) self.send_header("Content-Type", "text/html; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def redirect(self, path: str, message: str, is_error: bool = False) -> None: from urllib.parse import urlencode log_event(logging.INFO, "页面重定向", path=path, is_error=is_error, message=message) query = urlencode({"message": message, "error": "1" if is_error else "0"}) self.send_response(303) self.send_header("Location", f"{path}?{query}") self.end_headers() def log_message(self, format: str, *args: object) -> None: log_event(logging.INFO, "HTTP服务日志", client=self.address_string(), message=format % args) def main() -> int: configure_logging() UPLOAD_DIR.mkdir(parents=True, exist_ok=True) log_event( logging.INFO, "上传服务启动中", host=HOST, port=PORT, upload_dir=UPLOAD_DIR, max_upload_bytes=MAX_UPLOAD_BYTES, db_schema=DB_SCHEMA, db_table=DB_TABLE, log_file=LOG_FILE, ) server = ThreadingHTTPServer((HOST, PORT), UploadHandler) log_event(logging.INFO, "上传服务已启动", url=f"http://{HOST}:{PORT}", upload_dir=UPLOAD_DIR, log_file=LOG_FILE) try: server.serve_forever() except KeyboardInterrupt: log_event(logging.INFO, "收到键盘中断准备停止服务") finally: server.server_close() log_event(logging.INFO, "上传服务已停止", host=HOST, port=PORT) return 0 if __name__ == "__main__": raise SystemExit(main())