阶段性更新

2025-11-20 01:27:33 +08:00
parent 92f219b575
commit cbd16a39ba
25 changed files with 7825 additions and 721 deletions
--- a/etl_billiards/tests/testdata_json/summary.txt
+++ b/etl_billiards/tests/testdata_json/summary.txt
@@ -0,0 +1,144 @@
+==========================================================================================
+20251110_034959_助教流水.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'orderAssistantDetails']
+  list orderAssistantDetails len 100, elem type dict, keys ['assistantNo', 'nickname', 'levelName', 'assistantName', 'tableName', 'siteProfile', 'skillName', 'id', 'order_trade_no', 'site_id']
+==========================================================================================
+20251110_035004_助教废除.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'abolitionAssistants']
+  list abolitionAssistants len 15, elem type dict, keys ['siteProfile', 'createTime', 'id', 'siteId', 'tableAreaId', 'tableId', 'tableArea', 'tableName', 'assistantOn', 'assistantName']
+==========================================================================================
+20251110_035011_台费流水.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'siteTableUseDetailsList']
+  list siteTableUseDetailsList len 100, elem type dict, keys ['siteProfile', 'id', 'order_trade_no', 'site_id', 'tenant_id', 'member_id', 'operator_id', 'operator_name', 'order_settle_id', 'ledger_unit_price']
+==========================================================================================
+20251110_035904_小票详情.json
+root list len 193
+sample keys ['orderSettleId', 'data']
+data keys ['data', 'code']
+  dict data keys ['tenantId', 'siteId', 'orderSettleId', 'orderSettleNumber', 'assistantManualDiscount', 'siteName', 'tenantName', 'siteAddress', 'siteBusinessTel', 'ticketRemark']
+==========================================================================================
+20251110_035908_台费打折.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'taiFeeAdjustInfos']
+  list taiFeeAdjustInfos len 100, elem type dict, keys ['tableProfile', 'siteProfile', 'id', 'adjust_type', 'applicant_id', 'applicant_name', 'create_time', 'is_delete', 'ledger_amount', 'ledger_count']
+==========================================================================================
+20251110_035916_结账记录.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'settleList']
+  list settleList len 100, elem type dict, keys ['siteProfile', 'settleList']
+==========================================================================================
+20251110_035923_支付记录.json
+root list len 200
+sample keys ['siteProfile', 'create_time', 'pay_amount', 'pay_status', 'pay_time', 'online_pay_channel', 'relate_type', 'relate_id', 'site_id', 'id', 'payment_method']
+  dict siteProfile keys ['id', 'org_id', 'shop_name', 'avatar', 'business_tel', 'full_address', 'address', 'longitude', 'latitude', 'tenant_site_region_id']
+==========================================================================================
+20251110_035929_退款记录.json
+root list len 11
+sample keys ['tenantName', 'siteProfile', 'id', 'site_id', 'tenant_id', 'pay_sn', 'pay_amount', 'pay_status', 'pay_time', 'create_time', 'relate_type', 'relate_id', 'is_revoke', 'is_delete', 'online_pay_channel', 'payment_method', 'balance_frozen_amount', 'card_frozen_amount', 'member_id', 'member_card_id']
+  dict siteProfile keys ['id', 'org_id', 'shop_name', 'avatar', 'business_tel', 'full_address', 'address', 'longitude', 'latitude', 'tenant_site_region_id']
+==========================================================================================
+20251110_035934_平台验券记录.json
+root list len 200
+sample keys ['siteProfile', 'id', 'tenant_id', 'site_id', 'sale_price', 'coupon_code', 'coupon_channel', 'site_order_id', 'coupon_free_time', 'use_status', 'create_time', 'is_delete', 'coupon_name', 'coupon_cover', 'coupon_remark', 'channel_deal_id', 'group_package_id', 'consume_time', 'groupon_type', 'coupon_money']
+  dict siteProfile keys ['id', 'org_id', 'shop_name', 'avatar', 'business_tel', 'full_address', 'address', 'longitude', 'latitude', 'tenant_site_region_id']
+==========================================================================================
+20251110_035941_商品档案.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'tenantGoodsList']
+  list tenantGoodsList len 100, elem type dict, keys ['categoryName', 'isInSite', 'commodityCode', 'id', 'tenant_id', 'goods_name', 'goods_cover', 'goods_state', 'goods_category_id', 'unit']
+==========================================================================================
+20251110_035948_门店销售记录.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'orderGoodsLedgers']
+  list orderGoodsLedgers len 100, elem type dict, keys ['siteId', 'siteName', 'orderGoodsId', 'openSalesman', 'id', 'order_trade_no', 'site_id', 'tenant_id', 'operator_id', 'operator_name']
+==========================================================================================
+20251110_043159_库存变化记录1.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'queryDeliveryRecordsList']
+  list queryDeliveryRecordsList len 100, elem type dict, keys ['siteGoodsStockId', 'siteGoodsId', 'siteId', 'tenantId', 'stockType', 'goodsName', 'createTime', 'startNum', 'endNum', 'changeNum']
+==========================================================================================
+20251110_043204_库存变化记录2.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'goodsCategoryList']
+  list goodsCategoryList len 9, elem type dict, keys ['id', 'tenant_id', 'category_name', 'alias_name', 'pid', 'business_name', 'tenant_goods_business_id', 'open_salesman', 'categoryBoxes', 'sort']
+==========================================================================================
+20251110_043209_会员档案.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'tenantMemberInfos']
+  list tenantMemberInfos len 100, elem type dict, keys ['id', 'create_time', 'member_card_grade_code', 'mobile', 'nickname', 'register_site_id', 'site_name', 'member_card_grade_name', 'system_member_id', 'tenant_id']
+==========================================================================================
+20251110_043217_余额变更记录.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'tenantMemberCardLogs']
+  list tenantMemberCardLogs len 100, elem type dict, keys ['memberCardTypeName', 'paySiteName', 'registerSiteName', 'memberName', 'memberMobile', 'id', 'account_data', 'after', 'before', 'card_type_id']
+==========================================================================================
+20251110_043223_储值卡列表.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'totalOther', 'tenantMemberCards']
+  list tenantMemberCards len 100, elem type dict, keys ['site_name', 'member_name', 'member_mobile', 'member_card_type_name', 'table_service_discount', 'assistant_service_discount', 'coupon_discount', 'goods_service_discount', 'is_allow_give', 'able_cross_site']
+==========================================================================================
+20251110_043231_充值记录.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'settleList']
+  list settleList len 74, elem type dict, keys ['siteProfile', 'settleList']
+==========================================================================================
+20251110_043237_助教账号1.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'assistantInfos']
+  list assistantInfos len 50, elem type dict, keys ['job_num', 'shop_name', 'group_id', 'group_name', 'staff_profile_id', 'ding_talk_synced', 'entry_type', 'team_name', 'entry_sign_status', 'resign_sign_status']
+==========================================================================================
+20251110_043243_助教账号2.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'assistantInfos']
+  list assistantInfos len 50, elem type dict, keys ['job_num', 'shop_name', 'group_id', 'group_name', 'staff_profile_id', 'ding_talk_synced', 'entry_type', 'team_name', 'entry_sign_status', 'resign_sign_status']
+==========================================================================================
+20251110_043250_台桌列表.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'siteTables']
+  list siteTables len 71, elem type dict, keys ['id', 'audit_status', 'charge_free', 'self_table', 'create_time', 'is_rest_area', 'light_status', 'show_status', 'site_id', 'site_table_area_id']
+==========================================================================================
+20251110_043255_团购套餐.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'packageCouponList']
+  list packageCouponList len 17, elem type dict, keys ['site_name', 'effective_status', 'id', 'site_id', 'tenant_id', 'package_name', 'table_area_id', 'table_area_name', 'selling_price', 'duration']
+==========================================================================================
+20251110_043302_团购套餐流水.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'couponAmountSum', 'siteTableUseDetailsList']
+  list siteTableUseDetailsList len 100, elem type dict, keys ['tableName', 'tableAreaName', 'siteName', 'goodsOptionPrice', 'id', 'order_trade_no', 'table_id', 'site_id', 'tenant_id', 'operator_id']
+==========================================================================================
+20251110_043308_库存汇总.json
+root list len 161
+sample keys ['siteGoodsId', 'goodsName', 'goodsUnit', 'goodsCategoryId', 'goodsCategorySecondId', 'rangeStartStock', 'rangeEndStock', 'rangeIn', 'rangeOut', 'rangeInventory', 'rangeSale', 'rangeSaleMoney', 'currentStock', 'categoryName']
+==========================================================================================
+20251110_051132_门店商品档案1.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['total', 'orderGoodsList']
+  list orderGoodsList len 100, elem type dict, keys ['siteName', 'oneCategoryName', 'twoCategoryName', 'id', 'tenant_goods_id', 'site_id', 'tenant_id', 'goods_name', 'goods_cover', 'goods_state']
+==========================================================================================
+20251110_051138_门店商品档案2.json
+root list len 2
+sample keys ['data', 'code']
+data keys ['goodsStockA', 'goodsStockB', 'goodsSaleNum', 'stockSumMoney']
--- a/etl_billiards/tests/unit/task_test_utils.py
+++ b/etl_billiards/tests/unit/task_test_utils.py
@@ -135,16 +135,41 @@ class FakeDBOperations:

    def __init__(self):
        self.upserts: List[Dict] = []
+        self.executes: List[Dict] = []
        self.commits = 0
        self.rollbacks = 0
        self.conn = FakeConnection()

    def batch_upsert_with_returning(self, sql: str, rows: List[Dict], page_size: int = 1000):
-        self.upserts.append({"sql": sql.strip(), "count": len(rows), "page_size": page_size})
+        self.upserts.append(
+            {
+                "sql": sql.strip(),
+                "count": len(rows),
+                "page_size": page_size,
+                "rows": [dict(row) for row in rows],
+            }
+        )
        return len(rows), 0

    def batch_execute(self, sql: str, rows: List[Dict], page_size: int = 1000):
-        self.upserts.append({"sql": sql.strip(), "count": len(rows), "page_size": page_size})
+        self.executes.append(
+            {
+                "sql": sql.strip(),
+                "count": len(rows),
+                "page_size": page_size,
+                "rows": [dict(row) for row in rows],
+            }
+        )
+
+    def execute(self, sql: str, params=None):
+        self.executes.append({"sql": sql.strip(), "params": params})
+
+    def query(self, sql: str, params=None):
+        self.executes.append({"sql": sql.strip(), "params": params, "type": "query"})
+        return []
+
+    def cursor(self):
+        return self.conn.cursor()

    def commit(self):
        self.commits += 1
@@ -161,22 +186,53 @@ class FakeAPIClient:
        self.calls: List[Dict] = []

    # pylint: disable=unused-argument
-    def get_paginated(self, endpoint: str, params=None, **kwargs):
+    def iter_paginated(
+        self,
+        endpoint: str,
+        params=None,
+        page_size: int = 200,
+        page_field: str = "pageIndex",
+        size_field: str = "pageSize",
+        data_path: Tuple[str, ...] = (),
+        list_key: str | None = None,
+    ):
        self.calls.append({"endpoint": endpoint, "params": params})
        if endpoint not in self.data_map:
            raise AssertionError(f"Missing fixture for endpoint {endpoint}")
-        return list(self.data_map[endpoint]), [{"page": 1, "size": len(self.data_map[endpoint])}]
+
+        records = list(self.data_map[endpoint])
+        yield 1, records, dict(params or {}), {"data": records}
+
+    def get_paginated(self, endpoint: str, params=None, **kwargs):
+        records = []
+        pages = []
+        for page_no, page_records, req, resp in self.iter_paginated(endpoint, params, **kwargs):
+            records.extend(page_records)
+            pages.append({"page": page_no, "request": req, "response": resp})
+        return records, pages
+
+    def get_source_hint(self, endpoint: str) -> str | None:
+        return None


 class OfflineAPIClient:
-    """离线模式专用 API Client，根据 endpoint 读取归档 JSON、套用 data_path 并回放列表数据。"""
+    """离线模式专用 API Client，根据 endpoint 读取归档 JSON、套入 data_path 并回放列表数据。"""

    def __init__(self, file_map: Dict[str, Path]):
        self.file_map = {k: Path(v) for k, v in file_map.items()}
        self.calls: List[Dict] = []

    # pylint: disable=unused-argument
-    def get_paginated(self, endpoint: str, params=None, page_size: int = 200, data_path: Tuple[str, ...] = (), **kwargs):
+    def iter_paginated(
+        self,
+        endpoint: str,
+        params=None,
+        page_size: int = 200,
+        page_field: str = "pageIndex",
+        size_field: str = "pageSize",
+        data_path: Tuple[str, ...] = (),
+        list_key: str | None = None,
+    ):
        self.calls.append({"endpoint": endpoint, "params": params})
        if endpoint not in self.file_map:
            raise AssertionError(f"Missing archive for endpoint {endpoint}")
@@ -188,17 +244,42 @@ class OfflineAPIClient:
        for key in data_path:
            if isinstance(data, dict):
                data = data.get(key, [])
-            else:
-                data = []
-                break
+
+        if list_key and isinstance(data, dict):
+            data = data.get(list_key, [])

        if not isinstance(data, list):
            data = []

-        return data, [{"page": 1, "mode": "offline"}]
+        total = len(data)
+        start = 0
+        page = 1
+        while start < total or (start == 0 and total == 0):
+            chunk = data[start : start + page_size]
+            if not chunk and total != 0:
+                break
+            yield page, list(chunk), dict(params or {}), payload
+            if len(chunk) < page_size:
+                break
+            start += page_size
+            page += 1
+
+    def get_paginated(self, endpoint: str, params=None, **kwargs):
+        records = []
+        pages = []
+        for page_no, page_records, req, resp in self.iter_paginated(endpoint, params, **kwargs):
+            records.extend(page_records)
+            pages.append({"page": page_no, "request": req, "response": resp})
+        return records, pages
+
+    def get_source_hint(self, endpoint: str) -> str | None:
+        if endpoint not in self.file_map:
+            return None
+        return str(self.file_map[endpoint])


 class RealDBOperationsAdapter:
+
    """连接真实 PostgreSQL 的适配器，为任务提供 batch_upsert + 事务能力。"""

    def __init__(self, dsn: str):
--- a/etl_billiards/tests/unit/test_ods_tasks.py
+++ b/etl_billiards/tests/unit/test_ods_tasks.py
@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+"""Unit tests for the new ODS ingestion tasks."""
+import logging
+import sys
+from pathlib import Path
+
+# Ensure project root is resolvable when running tests in isolation
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from tasks.ods_tasks import ODS_TASK_CLASSES
+from .task_test_utils import create_test_config, get_db_operations, FakeAPIClient
+
+
+def _build_config(tmp_path):
+    archive_dir = tmp_path / "archive"
+    temp_dir = tmp_path / "temp"
+    return create_test_config("ONLINE", archive_dir, temp_dir)
+
+
+def test_ods_order_settle_ingest(tmp_path):
+    """Ensure ODS_ORDER_SETTLE task writes raw payload + metadata."""
+    config = _build_config(tmp_path)
+    sample = [
+        {
+            "orderSettleId": 701,
+            "orderTradeNo": 8001,
+            "anyField": "value",
+        }
+    ]
+    api = FakeAPIClient({"/order/list": sample})
+    task_cls = ODS_TASK_CLASSES["ODS_ORDER_SETTLE"]
+
+    with get_db_operations() as db_ops:
+        task = task_cls(config, db_ops, api, logging.getLogger("test_ods_order"))
+        result = task.execute()
+
+    assert result["status"] == "SUCCESS"
+    assert result["counts"]["fetched"] == 1
+    assert db_ops.commits == 1
+    row = db_ops.upserts[0]["rows"][0]
+    assert row["order_settle_id"] == 701
+    assert row["order_trade_no"] == 8001
+    assert row["source_endpoint"] == "/order/list"
+    assert '"orderSettleId": 701' in row["payload"]
+
+
+def test_ods_payment_ingest(tmp_path):
+    """Ensure ODS_PAYMENT task stores relate fields and payload."""
+    config = _build_config(tmp_path)
+    sample = [
+        {
+            "payId": 901,
+            "relateType": "ORDER",
+            "relateId": 123,
+            "payAmount": "100.00",
+        }
+    ]
+    api = FakeAPIClient({"/pay/records": sample})
+    task_cls = ODS_TASK_CLASSES["ODS_PAYMENT"]
+
+    with get_db_operations() as db_ops:
+        task = task_cls(config, db_ops, api, logging.getLogger("test_ods_payment"))
+        result = task.execute()
+
+    assert result["status"] == "SUCCESS"
+    assert result["counts"]["fetched"] == 1
+    assert db_ops.commits == 1
+    row = db_ops.upserts[0]["rows"][0]
+    assert row["pay_id"] == 901
+    assert row["relate_type"] == "ORDER"
+    assert row["relate_id"] == 123
+    assert '"payId": 901' in row["payload"]