ODS 完成

etl_billiards/utils/json_store.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""JSON 归档/读取的通用工具。"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+ENDPOINT_FILENAME_MAP: dict[str, str] = {
+    "/memberprofile/gettenantmemberlist": "member_profiles.json",
+    "/memberprofile/getmembercardbalancechange": "member_balance_changes.json",
+    "/memberprofile/gettenantmembercardlist": "member_stored_value_cards.json",
+    "/site/getrechargesettlelist": "recharge_settlements.json",
+    "/assistantperformance/getabolitionassistant": "assistant_cancellation_records.json",
+    "/assistantperformance/getorderassistantdetails": "assistant_service_records.json",
+    "/personnelmanagement/searchassistantinfo": "assistant_accounts_master.json",
+    "/table/getsitetables": "site_tables_master.json",
+    "/site/gettaifeeadjustlist": "table_fee_discount_records.json",
+    "/site/getsitetableorderdetails": "table_fee_transactions.json",
+    "/tenantgoods/querytenantgoods": "tenant_goods_master.json",
+    "/packagecoupon/querypackagecouponlist": "group_buy_packages.json",
+    "/site/getsitetableusedetails": "group_buy_redemption_records.json",
+    "/order/getordersettleticketnew": "settlement_ticket_details.json",
+    "/promotion/getofflinecouponconsumepagelist": "platform_coupon_redemption_records.json",
+    "/goodsstockmanage/querygoodsoutboundreceipt": "goods_stock_movements.json",
+    "/tenantgoodscategory/queryprimarysecondarycategory": "stock_goods_category_tree.json",
+    "/tenantgoods/getgoodsstockreport": "goods_stock_summary.json",
+    "/paylog/getpayloglistpage": "payment_transactions.json",
+    "/site/getallordersettlelist": "settlement_records.json",
+    "/order/getrefundpayloglist": "refund_transactions.json",
+    "/tenantgoods/getgoodsinventorylist": "store_goods_master.json",
+    "/tenantgoods/getgoodssaleslist": "store_goods_sales_records.json",
+}
+
+def endpoint_to_filename(endpoint: str) -> str:
+    """
+    将 API endpoint 转换为规范化的文件名，优先使用 非球接口API.md 中约定的名称。
+    未覆盖的路径会回退到“去掉开头斜杠 -> 用双下划线替换斜杠 -> 小写”的规则。
+    """
+    normalized = _normalize_endpoint(endpoint)
+    if normalized in ENDPOINT_FILENAME_MAP:
+        return ENDPOINT_FILENAME_MAP[normalized]
+
+    fallback = normalized.strip("/").replace("/", "__").replace(" ", "_")
+    return f"{fallback or 'root'}.json"
+
+
+def dump_json(path: Path, payload: Any, pretty: bool = False):
+    """将 JSON 对象写入文件，默认紧凑，可选美化。"""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as fp:
+        json.dump(payload, fp, ensure_ascii=False, indent=2 if pretty else None)
+
+
+def _normalize_endpoint(endpoint: str) -> str:
+    """标准化 endpoint，提取路径部分并统一小写、去除 base 前缀。"""
+    raw = str(endpoint or "").strip()
+    if not raw:
+        return ""
+
+    parsed = urlparse(raw)
+    path = parsed.path or raw
+    if not path.startswith("/"):
+        path = f"/{path}"
+
+    path = path.rstrip("/") or "/"
+    lowered = path.lower()
+    for prefix in ("/apiprod/admin/v1", "apiprod/admin/v1"):
+        if lowered.startswith(prefix):
+            path = path[len(prefix) :]
+            if not path.startswith("/"):
+                path = f"/{path}"
+            path = path.rstrip("/") or "/"
+            lowered = path.lower()
+            break
+
+    return lowered

etl_billiards/utils/reporting.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+"""简单的任务结果汇总与格式化工具。"""
+from __future__ import annotations
+
+from typing import Iterable
+
+
+def summarize_counts(task_results: Iterable[dict]) -> dict:
+    """
+    汇总多个任务的 counts，返回总计与逐任务明细。
+    task_results: 形如 {"task_code": str, "counts": {...}} 的字典序列。
+    """
+    totals = {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}
+    details = []
+
+    for res in task_results:
+        code = res.get("task_code") or res.get("code") or "UNKNOWN"
+        counts = res.get("counts") or {}
+        row = {"task_code": code}
+        for key in totals.keys():
+            val = int(counts.get(key, 0) or 0)
+            row[key] = val
+            totals[key] += val
+        details.append(row)
+
+    return {"total": totals, "details": details}
+
+
+def format_report(summary: dict) -> str:
+    """将 summarize_counts 的输出格式化为可读文案。"""
+    lines = []
+    totals = summary.get("total", {})
+    lines.append(
+        "TOTAL fetched={fetched} inserted={inserted} updated={updated} skipped={skipped} errors={errors}".format(
+            fetched=totals.get("fetched", 0),
+            inserted=totals.get("inserted", 0),
+            updated=totals.get("updated", 0),
+            skipped=totals.get("skipped", 0),
+            errors=totals.get("errors", 0),
+        )
+    )
+    for row in summary.get("details", []):
+        lines.append(
+            "{task_code}: fetched={fetched} inserted={inserted} updated={updated} skipped={skipped} errors={errors}".format(
+                task_code=row.get("task_code", "UNKNOWN"),
+                fetched=row.get("fetched", 0),
+                inserted=row.get("inserted", 0),
+                updated=row.get("updated", 0),
+                skipped=row.get("skipped", 0),
+                errors=row.get("errors", 0),
+            )
+        )
+    return "\n".join(lines)