ODS 完成

etl_billiards/tasks/assistant_abolish_task.py

@@ -3,7 +3,7 @@
 
 import json
 
-from .base_task import BaseTask
+from .base_task import BaseTask, TaskContext
 from loaders.facts.assistant_abolish import AssistantAbolishLoader
 from models.parsers import TypeParser
 
@@ -14,54 +14,54 @@ class AssistantAbolishTask(BaseTask):
     def get_task_code(self) -> str:
         return "ASSISTANT_ABOLISH"
 
-    def execute(self) -> dict:
-        self.logger.info("开始执行 ASSISTANT_ABOLISH 任务")
-        window_start, window_end, _ = self._get_time_window()
-        params = {
-            "storeId": self.config.get("app.store_id"),
-            "startTime": TypeParser.format_timestamp(window_start, self.tz),
-            "endTime": TypeParser.format_timestamp(window_end, self.tz),
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params(
+            {
+                "siteId": context.store_id,
+                "startTime": TypeParser.format_timestamp(context.window_start, self.tz),
+                "endTime": TypeParser.format_timestamp(context.window_end, self.tz),
+            }
+        )
+        records, _ = self.api.get_paginated(
+            endpoint="/AssistantPerformance/GetAbolitionAssistant",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+            list_key="abolitionAssistants",
+        )
+        return {"records": records}
+
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            mapped = self._parse_record(raw, context.store_id)
+            if mapped:
+                parsed.append(mapped)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
         }
 
-        try:
-            records, _ = self.api.get_paginated(
-                endpoint="/Assistant/AbolishList",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data", "abolitionAssistants"),
-            )
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = AssistantAbolishLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_records(transformed["records"])
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
 
-            parsed = []
-            for raw in records:
-                mapped = self._parse_record(raw)
-                if mapped:
-                    parsed.append(mapped)
-
-            loader = AssistantAbolishLoader(self.db)
-            inserted, updated, skipped = loader.upsert_records(parsed)
-
-            self.db.commit()
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0,
-            }
-            self.logger.info(f"ASSISTANT_ABOLISH 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-        except Exception:
-            self.db.rollback()
-            self.logger.error("ASSISTANT_ABOLISH 失败", exc_info=True)
-            raise
-
-    def _parse_record(self, raw: dict) -> dict | None:
+    def _parse_record(self, raw: dict, store_id: int) -> dict | None:
         abolish_id = TypeParser.parse_int(raw.get("id"))
         if not abolish_id:
-            self.logger.warning("跳过缺少 id 的助教作废记录: %s", raw)
+            self.logger.warning("跳过缺少作废ID的记录: %s", raw)
             return None
 
-        store_id = self.config.get("app.store_id")
         return {
             "store_id": store_id,
             "abolish_id": abolish_id,
@@ -72,9 +72,7 @@ class AssistantAbolishTask(BaseTask):
             "assistant_no": raw.get("assistantOn"),
             "assistant_name": raw.get("assistantName"),
             "charge_minutes": TypeParser.parse_int(raw.get("pdChargeMinutes")),
-            "abolish_amount": TypeParser.parse_decimal(
-                raw.get("assistantAbolishAmount")
-            ),
+            "abolish_amount": TypeParser.parse_decimal(raw.get("assistantAbolishAmount")),
             "create_time": TypeParser.parse_timestamp(
                 raw.get("createTime") or raw.get("create_time"), self.tz
             ),

etl_billiards/tasks/assistants_task.py

@@ -3,7 +3,7 @@
 
 import json
 
-from .base_task import BaseTask
+from .base_task import BaseTask, TaskContext
 from loaders.dimensions.assistant import AssistantLoader
 from models.parsers import TypeParser
 
@@ -14,49 +14,48 @@ class AssistantsTask(BaseTask):
     def get_task_code(self) -> str:
         return "ASSISTANTS"
 
-    def execute(self) -> dict:
-        self.logger.info("开始执行 ASSISTANTS 任务")
-        params = {"storeId": self.config.get("app.store_id")}
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params({"siteId": context.store_id})
+        records, _ = self.api.get_paginated(
+            endpoint="/PersonnelManagement/SearchAssistantInfo",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+            list_key="assistantInfos",
+        )
+        return {"records": records}
 
-        try:
-            records, _ = self.api.get_paginated(
-                endpoint="/Assistant/List",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data", "assistantInfos"),
-            )
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            mapped = self._parse_assistant(raw, context.store_id)
+            if mapped:
+                parsed.append(mapped)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
+        }
 
-            parsed = []
-            for raw in records:
-                mapped = self._parse_assistant(raw)
-                if mapped:
-                    parsed.append(mapped)
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = AssistantLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_assistants(transformed["records"])
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
 
-            loader = AssistantLoader(self.db)
-            inserted, updated, skipped = loader.upsert_assistants(parsed)
-
-            self.db.commit()
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0,
-            }
-            self.logger.info(f"ASSISTANTS 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-        except Exception:
-            self.db.rollback()
-            self.logger.error("ASSISTANTS 失败", exc_info=True)
-            raise
-
-    def _parse_assistant(self, raw: dict) -> dict | None:
+    def _parse_assistant(self, raw: dict, store_id: int) -> dict | None:
         assistant_id = TypeParser.parse_int(raw.get("id"))
         if not assistant_id:
-            self.logger.warning("跳过缺少 id 的助教数据: %s", raw)
+            self.logger.warning("跳过缺少助教ID的数据: %s", raw)
             return None
 
-        store_id = self.config.get("app.store_id")
         return {
             "store_id": store_id,
             "assistant_id": assistant_id,

etl_billiards/tasks/base_task.py

@@ -1,62 +1,141 @@
 # -*- coding: utf-8 -*-
-"""ETL任务基类"""
+"""ETL任务基类（引入 Extract/Transform/Load 模板方法）"""
+from __future__ import annotations
+
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 from zoneinfo import ZoneInfo
 
+
+@dataclass(frozen=True)
+class TaskContext:
+    """统一透传给 Extract/Transform/Load 的运行期信息。"""
+
+    store_id: int
+    window_start: datetime
+    window_end: datetime
+    window_minutes: int
+    cursor: dict | None = None
+
+
 class BaseTask:
-    """ETL任务基类"""
-    
+    """提供 E/T/L 模板的任务基类。"""
+
     def __init__(self, config, db_connection, api_client, logger):
         self.config = config
         self.db = db_connection
         self.api = api_client
         self.logger = logger
         self.tz = ZoneInfo(config.get("app.timezone", "Asia/Taipei"))
-    
+
+    # ------------------------------------------------------------------ 基本信息
     def get_task_code(self) -> str:
         """获取任务代码"""
         raise NotImplementedError("子类需实现 get_task_code 方法")
-    
-    def execute(self) -> dict:
-        """执行任务"""
-        raise NotImplementedError("子类需实现 execute 方法")
-    
+
+    # ------------------------------------------------------------------ E/T/L 钩子
+    def extract(self, context: TaskContext):
+        """提取数据"""
+        raise NotImplementedError("子类需实现 extract 方法")
+
+    def transform(self, extracted, context: TaskContext):
+        """转换数据"""
+        return extracted
+
+    def load(self, transformed, context: TaskContext) -> dict:
+        """加载数据并返回统计信息"""
+        raise NotImplementedError("子类需实现 load 方法")
+
+    # ------------------------------------------------------------------ 主流程
+    def execute(self, cursor_data: dict | None = None) -> dict:
+        """统一 orchestrate Extract → Transform → Load"""
+        context = self._build_context(cursor_data)
+        task_code = self.get_task_code()
+        self.logger.info(
+            "%s: 开始执行，窗口[%s ~ %s]",
+            task_code,
+            context.window_start,
+            context.window_end,
+        )
+
+        try:
+            extracted = self.extract(context)
+            transformed = self.transform(extracted, context)
+            counts = self.load(transformed, context) or {}
+            self.db.commit()
+        except Exception:
+            self.db.rollback()
+            self.logger.error("%s: 执行失败", task_code, exc_info=True)
+            raise
+
+        result = self._build_result("SUCCESS", counts)
+        result["window"] = {
+            "start": context.window_start,
+            "end": context.window_end,
+            "minutes": context.window_minutes,
+        }
+        self.logger.info("%s: 完成，统计=%s", task_code, result["counts"])
+        return result
+
+    # ------------------------------------------------------------------ 辅助方法
+    def _build_context(self, cursor_data: dict | None) -> TaskContext:
+        window_start, window_end, window_minutes = self._get_time_window(cursor_data)
+        return TaskContext(
+            store_id=self.config.get("app.store_id"),
+            window_start=window_start,
+            window_end=window_end,
+            window_minutes=window_minutes,
+            cursor=cursor_data,
+        )
+
     def _get_time_window(self, cursor_data: dict = None) -> tuple:
         """计算时间窗口"""
         now = datetime.now(self.tz)
-        
-        # 判断是否在闲时窗口
+
         idle_start = self.config.get("run.idle_window.start", "04:00")
         idle_end = self.config.get("run.idle_window.end", "16:00")
-        
         is_idle = self._is_in_idle_window(now, idle_start, idle_end)
-        
-        # 获取窗口大小
+
         if is_idle:
             window_minutes = self.config.get("run.window_minutes.default_idle", 180)
         else:
             window_minutes = self.config.get("run.window_minutes.default_busy", 30)
-        
-        # 计算窗口
+
         overlap_seconds = self.config.get("run.overlap_seconds", 120)
-        
+
         if cursor_data and cursor_data.get("last_end"):
             window_start = cursor_data["last_end"] - timedelta(seconds=overlap_seconds)
         else:
             window_start = now - timedelta(minutes=window_minutes)
-        
+
         window_end = now
-        
         return window_start, window_end, window_minutes
-    
+
     def _is_in_idle_window(self, dt: datetime, start_time: str, end_time: str) -> bool:
         """判断是否在闲时窗口"""
         current_time = dt.strftime("%H:%M")
         return start_time <= current_time <= end_time
-    
+
+    def _merge_common_params(self, base: dict) -> dict:
+        """
+        合并全局/任务级参数池，便于在配置中统一覆<E4B880>?/追加过滤条件。
+        支持：
+        - api.params 下的通用键<E794A8>?
+        - api.params.<task_code_lower> 下的任务级键<E7BAA7>?
+        """
+        merged: dict = {}
+        common = self.config.get("api.params", {}) or {}
+        if isinstance(common, dict):
+            merged.update(common)
+
+        task_key = f"api.params.{self.get_task_code().lower()}"
+        scoped = self.config.get(task_key, {}) or {}
+        if isinstance(scoped, dict):
+            merged.update(scoped)
+
+        merged.update(base)
+        return merged
+
     def _build_result(self, status: str, counts: dict) -> dict:
         """构建结果字典"""
-        return {
-            "status": status,
-            "counts": counts
-        }
+        return {"status": status, "counts": counts}

etl_billiards/tasks/coupon_usage_task.py

@@ -3,65 +3,66 @@
 
 import json
 
-from .base_task import BaseTask
+from .base_task import BaseTask, TaskContext
 from loaders.facts.coupon_usage import CouponUsageLoader
 from models.parsers import TypeParser
 
 
 class CouponUsageTask(BaseTask):
-    """同步平台券验证/核销记录"""
+    """同步平台券验券/核销记录"""
 
     def get_task_code(self) -> str:
         return "COUPON_USAGE"
 
-    def execute(self) -> dict:
-        self.logger.info("开始执行 COUPON_USAGE 任务")
-        window_start, window_end, _ = self._get_time_window()
-        params = {
-            "storeId": self.config.get("app.store_id"),
-            "startTime": TypeParser.format_timestamp(window_start, self.tz),
-            "endTime": TypeParser.format_timestamp(window_end, self.tz),
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params(
+            {
+                "siteId": context.store_id,
+                "startTime": TypeParser.format_timestamp(context.window_start, self.tz),
+                "endTime": TypeParser.format_timestamp(context.window_end, self.tz),
+            }
+        )
+        records, _ = self.api.get_paginated(
+            endpoint="/Promotion/GetOfflineCouponConsumePageList",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+        )
+        return {"records": records}
+
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            mapped = self._parse_usage(raw, context.store_id)
+            if mapped:
+                parsed.append(mapped)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
         }
 
-        try:
-            records, _ = self.api.get_paginated(
-                endpoint="/Coupon/UsageList",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=(),
-            )
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = CouponUsageLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_coupon_usage(
+            transformed["records"]
+        )
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
 
-            parsed = []
-            for raw in records:
-                mapped = self._parse_usage(raw)
-                if mapped:
-                    parsed.append(mapped)
-
-            loader = CouponUsageLoader(self.db)
-            inserted, updated, skipped = loader.upsert_coupon_usage(parsed)
-
-            self.db.commit()
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0,
-            }
-            self.logger.info(f"COUPON_USAGE 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-        except Exception:
-            self.db.rollback()
-            self.logger.error("COUPON_USAGE 失败", exc_info=True)
-            raise
-
-    def _parse_usage(self, raw: dict) -> dict | None:
+    def _parse_usage(self, raw: dict, store_id: int) -> dict | None:
         usage_id = TypeParser.parse_int(raw.get("id"))
         if not usage_id:
-            self.logger.warning("跳过缺少 id 的券核销记录: %s", raw)
+            self.logger.warning("跳过缺少券核销ID的记录: %s", raw)
             return None
 
-        store_id = self.config.get("app.store_id")
         return {
             "store_id": store_id,
             "usage_id": usage_id,

etl_billiards/tasks/inventory_change_task.py

@@ -3,7 +3,7 @@
 
 import json
 
-from .base_task import BaseTask
+from .base_task import BaseTask, TaskContext
 from loaders.facts.inventory_change import InventoryChangeLoader
 from models.parsers import TypeParser
 
@@ -14,56 +14,56 @@ class InventoryChangeTask(BaseTask):
     def get_task_code(self) -> str:
         return "INVENTORY_CHANGE"
 
-    def execute(self) -> dict:
-        self.logger.info("开始执行 INVENTORY_CHANGE 任务")
-        window_start, window_end, _ = self._get_time_window()
-        params = {
-            "storeId": self.config.get("app.store_id"),
-            "startTime": TypeParser.format_timestamp(window_start, self.tz),
-            "endTime": TypeParser.format_timestamp(window_end, self.tz),
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params(
+            {
+                "siteId": context.store_id,
+                "startTime": TypeParser.format_timestamp(context.window_start, self.tz),
+                "endTime": TypeParser.format_timestamp(context.window_end, self.tz),
+            }
+        )
+        records, _ = self.api.get_paginated(
+            endpoint="/GoodsStockManage/QueryGoodsOutboundReceipt",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+            list_key="queryDeliveryRecordsList",
+        )
+        return {"records": records}
+
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            mapped = self._parse_change(raw, context.store_id)
+            if mapped:
+                parsed.append(mapped)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
         }
 
-        try:
-            records, _ = self.api.get_paginated(
-                endpoint="/Inventory/ChangeList",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data", "queryDeliveryRecordsList"),
-            )
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = InventoryChangeLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_changes(transformed["records"])
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
 
-            parsed = []
-            for raw in records:
-                mapped = self._parse_change(raw)
-                if mapped:
-                    parsed.append(mapped)
-
-            loader = InventoryChangeLoader(self.db)
-            inserted, updated, skipped = loader.upsert_changes(parsed)
-
-            self.db.commit()
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0,
-            }
-            self.logger.info(f"INVENTORY_CHANGE 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-        except Exception:
-            self.db.rollback()
-            self.logger.error("INVENTORY_CHANGE 失败", exc_info=True)
-            raise
-
-    def _parse_change(self, raw: dict) -> dict | None:
+    def _parse_change(self, raw: dict, store_id: int) -> dict | None:
         change_id = TypeParser.parse_int(
             raw.get("siteGoodsStockId") or raw.get("site_goods_stock_id")
         )
         if not change_id:
-            self.logger.warning("跳过缺少变动 id 的库存记录: %s", raw)
+            self.logger.warning("跳过缺少库存变动ID的记录: %s", raw)
             return None
 
-        store_id = self.config.get("app.store_id")
         return {
             "store_id": store_id,
             "change_id": change_id,

etl_billiards/tasks/ledger_task.py

@@ -3,7 +3,7 @@
 
 import json
 
-from .base_task import BaseTask
+from .base_task import BaseTask, TaskContext
 from loaders.facts.assistant_ledger import AssistantLedgerLoader
 from models.parsers import TypeParser
 
@@ -14,54 +14,54 @@ class LedgerTask(BaseTask):
     def get_task_code(self) -> str:
         return "LEDGER"
 
-    def execute(self) -> dict:
-        self.logger.info("开始执行 LEDGER 任务")
-        window_start, window_end, _ = self._get_time_window()
-        params = {
-            "storeId": self.config.get("app.store_id"),
-            "startTime": TypeParser.format_timestamp(window_start, self.tz),
-            "endTime": TypeParser.format_timestamp(window_end, self.tz),
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params(
+            {
+                "siteId": context.store_id,
+                "startTime": TypeParser.format_timestamp(context.window_start, self.tz),
+                "endTime": TypeParser.format_timestamp(context.window_end, self.tz),
+            }
+        )
+        records, _ = self.api.get_paginated(
+            endpoint="/AssistantPerformance/GetOrderAssistantDetails",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+            list_key="orderAssistantDetails",
+        )
+        return {"records": records}
+
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            mapped = self._parse_ledger(raw, context.store_id)
+            if mapped:
+                parsed.append(mapped)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
         }
 
-        try:
-            records, _ = self.api.get_paginated(
-                endpoint="/Assistant/LedgerList",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data", "orderAssistantDetails"),
-            )
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = AssistantLedgerLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_ledgers(transformed["records"])
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
 
-            parsed = []
-            for raw in records:
-                mapped = self._parse_ledger(raw)
-                if mapped:
-                    parsed.append(mapped)
-
-            loader = AssistantLedgerLoader(self.db)
-            inserted, updated, skipped = loader.upsert_ledgers(parsed)
-
-            self.db.commit()
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0,
-            }
-            self.logger.info(f"LEDGER 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-        except Exception:
-            self.db.rollback()
-            self.logger.error("LEDGER 失败", exc_info=True)
-            raise
-
-    def _parse_ledger(self, raw: dict) -> dict | None:
+    def _parse_ledger(self, raw: dict, store_id: int) -> dict | None:
         ledger_id = TypeParser.parse_int(raw.get("id"))
         if not ledger_id:
-            self.logger.warning("跳过缺少 id 的助教流水: %s", raw)
+            self.logger.warning("跳过缺少助教流水ID的记录: %s", raw)
             return None
 
-        store_id = self.config.get("app.store_id")
         return {
             "store_id": store_id,
             "ledger_id": ledger_id,
@@ -100,12 +100,8 @@ class LedgerTask(BaseTask):
             "ledger_end_time": TypeParser.parse_timestamp(
                 raw.get("ledger_end_time"), self.tz
             ),
-            "start_use_time": TypeParser.parse_timestamp(
-                raw.get("start_use_time"), self.tz
-            ),
-            "last_use_time": TypeParser.parse_timestamp(
-                raw.get("last_use_time"), self.tz
-            ),
+            "start_use_time": TypeParser.parse_timestamp(raw.get("start_use_time"), self.tz),
+            "last_use_time": TypeParser.parse_timestamp(raw.get("last_use_time"), self.tz),
             "income_seconds": TypeParser.parse_int(raw.get("income_seconds")),
             "real_use_seconds": TypeParser.parse_int(raw.get("real_use_seconds")),
             "is_trash": raw.get("is_trash"),

etl_billiards/tasks/manual_ingest_task.py

@@ -1,176 +1,719 @@
 # -*- coding: utf-8 -*-
-import os
+"""Manual ingestion task that replays archived JSON into ODS tables."""
+from __future__ import annotations
+
 import json
+import os
 from datetime import datetime
+from typing import Iterable, Iterator
+
 from .base_task import BaseTask
-from loaders.ods.generic import GenericODSLoader
+
 
 class ManualIngestTask(BaseTask):
     """
-    Task to ingest manually fetched JSON files from a directory into ODS tables.
+    Load archived API responses (tests/source-data-doc) into billiards_ods.* tables.
+    Used when upstream API is unavailable and we need to replay captured payloads.
     """
 
-    FILE_MAPPING = {
-        "小票详情": "billiards_ods.ods_ticket_detail",
-        "结账记录": "billiards_ods.ods_order_settle",
-        "支付记录": "billiards_ods.ods_payment",
-        "助教流水": "billiards_ods.ods_assistant_ledger",
-        "助教废除": "billiards_ods.ods_assistant_abolish",
-        "商品档案": "billiards_ods.ods_goods_ledger", # Note: This might be dim_product source, but mapping to ledger for now if it's sales
-        "库存变化": "billiards_ods.ods_inventory_change",
-        "会员档案": "billiards_ods.ods_member",
-        "充值记录": "billiards_ods.ods_member_card", # Approx
-        "团购套餐": "billiards_ods.ods_package_coupon",
-        "库存汇总": "billiards_ods.ods_inventory_stock"
-    }
+    FILE_MAPPING: list[tuple[tuple[str, ...], str]] = [
+        (("会员档案",), "billiards_ods.ods_member_profile"),
+        (("储值卡列表", "储值卡"), "billiards_ods.ods_member_card"),
+        (("充值记录",), "billiards_ods.ods_recharge_record"),
+        (("余额变动",), "billiards_ods.ods_balance_change"),
+        (("助教账号",), "billiards_ods.ods_assistant_account"),
+        (("助教流水",), "billiards_ods.ods_assistant_service_log"),
+        (("助教废除", "助教作废"), "billiards_ods.ods_assistant_cancel_log"),
+        (("台桌列表",), "billiards_ods.ods_table_info"),
+        (("台费流水",), "billiards_ods.ods_table_use_log"),
+        (("台费打折",), "billiards_ods.ods_table_fee_adjust"),
+        (("商品档案",), "billiards_ods.ods_store_product"),
+        (("门店商品销售", "销售记录"), "billiards_ods.ods_store_sale_item"),
+        (("团购套餐定义", "套餐定义"), "billiards_ods.ods_group_package"),
+        (("团购套餐使用", "套餐使用"), "billiards_ods.ods_group_package_log"),
+        (("平台验券", "验券记录"), "billiards_ods.ods_platform_coupon_log"),
+        (("库存汇总",), "billiards_ods.ods_inventory_stock"),
+        (("库存变化记录1",), "billiards_ods.ods_inventory_change"),
+        (("库存变化记录2", "分类配置"), "billiards_ods.ods_goods_category"),
+        (("结账记录",), "billiards_ods.ods_order_settle"),
+        (("小票详情", "小票明细", "票详"), "billiards_ods.ods_order_receipt_detail"),
+        (("支付记录",), "billiards_ods.ods_payment_record"),
+        (("退款记录",), "billiards_ods.ods_refund_record"),
+    ]
+    WRAPPER_META_KEYS = {"code", "message", "msg", "success", "error", "status"}
 
     def get_task_code(self) -> str:
         return "MANUAL_INGEST"
 
     def execute(self) -> dict:
         self.logger.info("Starting Manual Ingest Task")
-        
-        # Configurable directory, default to tests/testdata_json for now
-        data_dir = self.config.get("manual.data_dir", r"c:\dev\LLTQ\ETL\feiqiu-ETL\etl_billiards\tests\testdata_json")
-        
+
+        data_dir = self.config.get(
+            "manual.data_dir",
+            r"c:\dev\LLTQ\ETL\feiqiu-ETL\etl_billiards\tests\testdata_json",
+        )
         if not os.path.exists(data_dir):
-            self.logger.error(f"Data directory not found: {data_dir}")
+            self.logger.error("Data directory not found: %s", data_dir)
             return {"status": "error", "message": "Directory not found"}
 
-        total_files = 0
-        total_rows = 0
+        counts = {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}
 
-        for filename in os.listdir(data_dir):
+        for filename in sorted(os.listdir(data_dir)):
             if not filename.endswith(".json"):
                 continue
 
-            # Determine target table
-            target_table = None
-            for key, table in self.FILE_MAPPING.items():
-                if key in filename:
-                    target_table = table
-                    break
-            
-            if not target_table:
-                self.logger.warning(f"No mapping found for file: {filename}, skipping.")
+            filepath = os.path.join(data_dir, filename)
+            try:
+                with open(filepath, "r", encoding="utf-8") as fh:
+                    raw_entries = json.load(fh)
+            except Exception:
+                counts["errors"] += 1
+                self.logger.exception("Failed to read %s", filename)
                 continue
 
-            self.logger.info(f"Ingesting {filename} into {target_table}")
-            
+            if not isinstance(raw_entries, list):
+                raw_entries = [raw_entries]
+
+            records = self._normalize_records(raw_entries)
+            if not records:
+                counts["skipped"] += 1
+                continue
+
+            target_table = self._match_by_filename(filename) or self._match_by_content(
+                records, raw_entries
+            )
+            if not target_table:
+                self.logger.warning("No mapping found for file: %s", filename)
+                counts["skipped"] += 1
+                continue
+
+            self.logger.info("Ingesting %s into %s", filename, target_table)
+
             try:
-                with open(os.path.join(data_dir, filename), 'r', encoding='utf-8') as f:
-                    data = json.load(f)
-                
-                if not isinstance(data, list):
-                    data = [data]
-
-                # Prepare rows for GenericODSLoader
-                # We need to adapt the data to what GenericODSLoader expects (or update it)
-                # GenericODSLoader expects dicts. It handles normalization.
-                # But we need to ensure the primary keys are present in the payload or extracted.
-                # The GenericODSLoader might need configuration for PK extraction if it's not standard.
-                # For now, let's assume the payload IS the row, and we wrap it.
-                
-                # Actually, GenericODSLoader.upsert_rows expects the raw API result list.
-                # It calls _normalize_row.
-                # We need to make sure _normalize_row works for these files.
-                # Most files have 'id' or similar.
-                
-                # Let's instantiate a loader for this table
-                # We need to know the PK for the table.
-                # This is usually defined in ODS_TASK_CLASSES but here we are dynamic.
-                # We might need a simpler loader or reuse GenericODSLoader with specific PK config.
-                
-                # For simplicity, let's use a custom ingestion here that mimics GenericODSLoader but is file-aware.
-                rows_to_insert = []
-                for item in data:
-                    # Extract Store ID (usually in siteProfile or data root)
-                    store_id = self._extract_store_id(item) or self.config.get("app.store_id")
-                    
-                    # Extract PK (id, orderSettleId, etc.)
-                    pk_val = self._extract_pk(item, target_table)
-                    
-                    if not pk_val:
-                        # Try to find 'id' in the item
-                        pk_val = item.get("id")
-                    
-                    if not pk_val:
-                         # Special case for Ticket Detail
-                        if "ods_ticket_detail" in target_table:
-                            pk_val = item.get("orderSettleId")
-
-                    if not pk_val:
+                rows = []
+                for record in records:
+                    site_id = self._extract_store_id(record) or self.config.get(
+                        "app.store_id"
+                    )
+                    pk_value = self._extract_pk(record, target_table)
+                    pk_tuple = self._ensure_tuple(pk_value)
+                    if not all(value not in (None, "") for value in pk_tuple):
                         continue
 
                     row = {
-                        "store_id": store_id,
-                        "payload": json.dumps(item, ensure_ascii=False),
+                        "site_id": site_id,
+                        "payload": json.dumps(record, ensure_ascii=False),
                         "source_file": filename,
-                        "fetched_at": datetime.now()
+                        "fetched_at": datetime.now(),
                     }
-                    
-                    # Add specific PK column
-                    pk_col = self._get_pk_column(target_table)
-                    row[pk_col] = pk_val
-                    
-                    rows_to_insert.append(row)
+                    for column, value in zip(
+                        self._get_conflict_columns(target_table), pk_tuple
+                    ):
+                        row[column] = value
+                    self._enrich_row(row, record, target_table)
+                    rows.append(row)
 
-                if rows_to_insert:
-                    self._bulk_insert(target_table, rows_to_insert)
-                    total_rows += len(rows_to_insert)
-                    total_files += 1
+                if rows:
+                    self._bulk_insert(target_table, rows)
+                    counts["inserted"] += len(rows)
+                else:
+                    counts["skipped"] += 1
+                counts["fetched"] += 1
 
-            except Exception as e:
-                self.logger.error(f"Error processing {filename}: {e}", exc_info=True)
+            except Exception:
+                counts["errors"] += 1
+                self.logger.exception("Error processing %s", filename)
+                self.db.rollback()
 
-        return {"status": "success", "files_processed": total_files, "rows_inserted": total_rows}
+        try:
+            self.db.commit()
+        except Exception:
+            self.db.rollback()
+            raise
 
-    def _extract_store_id(self, item):
-        # Try common paths
-        if "store_id" in item: return item["store_id"]
-        if "siteProfile" in item and "id" in item["siteProfile"]: return item["siteProfile"]["id"]
-        if "data" in item and "data" in item["data"] and "siteId" in item["data"]["data"]: return item["data"]["data"]["siteId"]
+        return self._build_result("SUCCESS", counts)
+
+    # ------------------------------------------------------------------ helpers
+    def _match_by_filename(self, filename: str) -> str | None:
+        for keywords, table in self.FILE_MAPPING:
+            if any(keyword and keyword in filename for keyword in keywords):
+                return table
         return None
 
-    def _extract_pk(self, item, table):
-        # Helper to find PK based on table
+    def _match_by_content(
+        self, records: list[dict], raw_entries: list[dict]
+    ) -> str | None:
+        """
+        Map content to PRD ODS tables.
+        """
+        sample_record = records[0] if records else None
+        wrapper = self._extract_sample(raw_entries)
+        data_node = wrapper.get("data") if isinstance(wrapper, dict) else None
+        data_keys = set(data_node.keys()) if isinstance(data_node, dict) else set()
+        record_keys = set(sample_record.keys()) if isinstance(sample_record, dict) else set()
+
+        # Data node based hints
+        if "tenantMemberInfos" in data_keys:
+            return "billiards_ods.ods_member_profile"
+        if "tenantMemberCards" in data_keys:
+            return "billiards_ods.ods_member_card"
+        if "queryDeliveryRecordsList" in data_keys:
+            return "billiards_ods.ods_inventory_change"
+        if "goodsStockA" in data_keys or "rangeStartStock" in data_keys:
+            return "billiards_ods.ods_inventory_stock"
+        if "goodsCategoryList" in data_keys:
+            return "billiards_ods.ods_goods_category"
+        if "orderAssistantDetails" in data_keys:
+            return "billiards_ods.ods_assistant_service_log"
+        if "abolitionAssistants" in data_keys:
+            return "billiards_ods.ods_assistant_cancel_log"
+        if "siteTableUseDetailsList" in data_keys:
+            return "billiards_ods.ods_table_use_log"
+        if "taiFeeAdjustInfos" in data_keys:
+            return "billiards_ods.ods_table_fee_adjust"
+        if "orderGoodsLedgers" in data_keys or "orderGoodsList" in data_keys:
+            return "billiards_ods.ods_store_sale_item"
+        if "tenantGoodsList" in data_keys:
+            return "billiards_ods.ods_store_product"
+        if "packageCouponList" in data_keys:
+            return "billiards_ods.ods_group_package"
+        if "settleList" in data_keys and "total" in data_keys:
+            return "billiards_ods.ods_order_settle"
+
+        # Record key based hints
+        if sample_record:
+            if {"pay_amount", "pay_status"} <= record_keys or {"payAmount", "payStatus"} <= record_keys:
+                return "billiards_ods.ods_payment_record"
+            if "refundAmount" in record_keys or "refund_amount" in record_keys:
+                return "billiards_ods.ods_refund_record"
+            if "orderSettleId" in record_keys or "order_settle_id" in record_keys:
+                return "billiards_ods.ods_order_receipt_detail"
+            if "coupon_channel" in record_keys or "groupPackageId" in record_keys:
+                return "billiards_ods.ods_platform_coupon_log"
+            if "packageId" in record_keys or "package_id" in record_keys:
+                return "billiards_ods.ods_group_package_log"
+            if "memberCardId" in record_keys or "cardId" in record_keys:
+                return "billiards_ods.ods_member_card"
+            if "memberId" in record_keys:
+                return "billiards_ods.ods_member_profile"
+            if "siteGoodsId" in record_keys and "currentStock" in record_keys:
+                return "billiards_ods.ods_inventory_stock"
+            if "goodsId" in record_keys:
+                return "billiards_ods.ods_product"
+
+        return None
+
+    def _extract_sample(self, payloads: Iterable[dict]) -> dict:
+        for item in payloads:
+            if isinstance(item, dict):
+                return item
+        return {}
+
+    def _normalize_records(self, payloads: list[dict]) -> list[dict]:
+        records: list[dict] = []
+        for payload in payloads:
+            records.extend(self._unwrap_payload(payload))
+        return records
+
+    def _unwrap_payload(self, payload) -> list[dict]:
+        if isinstance(payload, dict):
+            data_node = payload.get("data")
+            extra_keys = set(payload.keys()) - {"data"} - self.WRAPPER_META_KEYS
+            if isinstance(data_node, dict) and not extra_keys:
+                flattened: list[dict] = []
+                found_list = False
+                for value in data_node.values():
+                    if isinstance(value, list):
+                        flattened.extend(value)
+                        found_list = True
+                if found_list:
+                    return flattened
+                return [data_node]
+            return [payload]
+
+        if isinstance(payload, list):
+            flattened: list[dict] = []
+            for item in payload:
+                flattened.extend(self._unwrap_payload(item))
+            return flattened
+
+        return []
+
+    def _extract_store_id(self, item: dict):
+        """Extract site_id from record/siteProfile wrappers."""
+        site_profile = item.get("siteProfile") or item.get("site_profile")
+        if isinstance(site_profile, dict) and site_profile.get("id"):
+            return site_profile["id"]
+
+        for key in ("site_id", "siteId", "register_site_id"):
+            if item.get(key):
+                return item[key]
+
+        data_node = item.get("data")
+        if isinstance(data_node, dict):
+            return data_node.get("siteId") or data_node.get("site_id")
+
+        return None
+
+    def _extract_pk(self, item: dict, table: str):
+        if "ods_order_receipt_detail" in table:
+            return item.get("orderSettleId") or item.get("order_settle_id") or item.get("id")
         if "ods_order_settle" in table:
-            # Check for nested structure in some files
-            if "settleList" in item and "settleList" in item["settleList"]:
-                return item["settleList"]["settleList"].get("id")
+            settle = item.get("settleList") or item.get("settle") or item
+            if isinstance(settle, dict):
+                return settle.get("id") or settle.get("settleId") or item.get("id")
             return item.get("id")
+
+        if "ods_payment_record" in table:
+            return item.get("payId") or item.get("id")
+
+        if "ods_refund_record" in table:
+            return item.get("refundId") or item.get("id")
+
+        if "ods_platform_coupon_log" in table:
+            return item.get("couponId") or item.get("id")
+
+        if "ods_assistant_service_log" in table or "ods_table_use_log" in table:
+            return item.get("ledgerId") or item.get("ledger_id") or item.get("id")
+
+        if "ods_assistant_cancel_log" in table:
+            return item.get("cancel_id") or item.get("cancelId") or item.get("abolishId") or item.get("id")
+
+        if "ods_store_sale_item" in table:
+            return (
+                item.get("sale_item_id")
+                or item.get("saleItemId")
+                or item.get("orderGoodsId")
+                or item.get("order_goods_id")
+                or item.get("id")
+            )
+
+        if "ods_inventory_change" in table:
+            return item.get("siteGoodsStockId") or item.get("id")
+
+        if "ods_inventory_stock" in table:
+            return (
+                item.get("siteGoodsId")
+                or item.get("id"),
+                item.get("snapshotKey") or item.get("snapshot_key") or "default",
+            )
+
+        if "ods_member_card" in table:
+            return item.get("cardId") or item.get("memberCardId") or item.get("id")
+
+        if "ods_member_profile" in table:
+            return item.get("memberId") or item.get("id")
+
+        if "ods_group_package_log" in table:
+            return item.get("usage_id") or item.get("usageId") or item.get("couponId") or item.get("id")
+
+        if "ods_group_package" in table:
+            return item.get("package_id") or item.get("packageId") or item.get("groupPackageId") or item.get("id")
+
+        if "ods_goods_category" in table:
+            return item.get("category_id") or item.get("categoryId") or item.get("id")
+
+        if "ods_table_fee_adjust" in table:
+            return item.get("adjust_id") or item.get("adjustId") or item.get("id")
+
+        if "ods_table_info" in table:
+            return item.get("table_id") or item.get("tableId") or item.get("id")
+
+        if "ods_assistant_account" in table:
+            return item.get("assistantId") or item.get("assistant_id") or item.get("id")
+
+        if "ods_store_product" in table:
+            return item.get("siteGoodsId") or item.get("site_goods_id") or item.get("id")
+
+        if "ods_product" in table:
+            return item.get("goodsId") or item.get("goods_id") or item.get("id")
+
+        if "ods_balance_change" in table:
+            return item.get("change_id") or item.get("changeId") or item.get("id")
+
+        if "ods_recharge_record" in table:
+            return item.get("recharge_id") or item.get("rechargeId") or item.get("id")
+
         return item.get("id")
 
-    def _get_pk_column(self, table):
-        if "ods_ticket_detail" in table: return "order_settle_id"
-        if "ods_order_settle" in table: return "order_settle_id"
-        if "ods_payment" in table: return "pay_id"
-        if "ods_member" in table: return "member_id"
-        if "ods_assistant_ledger" in table: return "ledger_id"
-        if "ods_goods_ledger" in table: return "order_goods_id"
-        if "ods_inventory_change" in table: return "change_id"
-        if "ods_assistant_abolish" in table: return "abolish_id"
-        if "ods_coupon_verify" in table: return "coupon_id"
-        if "ods_member_card" in table: return "card_id"
-        if "ods_package_coupon" in table: return "package_id"
-        return "id" # Fallback
+    def _get_conflict_columns(self, table: str) -> list[str]:
+        if "ods_order_receipt_detail" in table:
+            return ["order_settle_id"]
+        if "ods_payment_record" in table:
+            return ["pay_id"]
+        if "ods_refund_record" in table:
+            return ["refund_id"]
+        if "ods_platform_coupon_log" in table:
+            return ["coupon_id"]
+        if "ods_assistant_service_log" in table or "ods_table_use_log" in table:
+            return ["ledger_id"]
+        if "ods_assistant_cancel_log" in table:
+            return ["cancel_id"]
+        if "ods_store_sale_item" in table:
+            return ["sale_item_id"]
+        if "ods_order_settle" in table:
+            return ["order_settle_id"]
+        if "ods_inventory_change" in table:
+            return ["change_id"]
+        if "ods_inventory_stock" in table:
+            return ["site_goods_id", "snapshot_key"]
+        if "ods_member_card" in table:
+            return ["card_id"]
+        if "ods_member_profile" in table:
+            return ["member_id"]
+        if "ods_group_package_log" in table:
+            return ["usage_id"]
+        if "ods_group_package" in table:
+            return ["package_id"]
+        if "ods_goods_category" in table:
+            return ["category_id"]
+        if "ods_table_info" in table:
+            return ["table_id"]
+        if "ods_table_fee_adjust" in table:
+            return ["adjust_id"]
+        if "ods_assistant_account" in table:
+            return ["assistant_id"]
+        if "ods_store_product" in table:
+            return ["site_goods_id"]
+        if "ods_product" in table:
+            return ["goods_id"]
+        if "ods_balance_change" in table:
+            return ["change_id"]
+        if "ods_recharge_record" in table:
+            return ["recharge_id"]
+        return ["id"]
+
+    def _enrich_row(self, row: dict, record: dict, table: str):
+        """Best-effort populate important columns from payload for PRD ODS schema."""
+        def pick(obj, *keys):
+            for k in keys:
+                if isinstance(obj, dict) and obj.get(k) not in (None, ""):
+                    return obj.get(k)
+            return None
+
+        if "ods_member_profile" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["member_name"] = pick(record, "name", "memberName")
+            row["nickname"] = record.get("nickname")
+            row["mobile"] = record.get("mobile")
+            row["gender"] = record.get("sex")
+            row["birthday"] = record.get("birthday")
+            row["register_time"] = record.get("register_time") or record.get("registerTime")
+            row["member_type_id"] = pick(record, "cardTypeId", "member_type_id")
+            row["member_type_name"] = record.get("cardTypeName")
+            row["status"] = pick(record, "status", "state")
+            row["balance"] = record.get("balance")
+            row["points"] = record.get("points") or record.get("point")
+            row["last_visit_time"] = record.get("lastVisitTime")
+            row["wechat_id"] = record.get("wechatId")
+            row["alipay_id"] = record.get("alipayId")
+            row["member_card_no"] = record.get("cardNo")
+            row["remarks"] = record.get("remark")
+
+        if "ods_member_card" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["member_id"] = pick(record, "memberId", "member_id")
+            row["card_type_id"] = record.get("cardTypeId")
+            row["card_type_name"] = record.get("cardTypeName")
+            row["card_balance"] = record.get("balance")
+            row["discount_rate"] = record.get("discount") or record.get("discount_rate")
+            row["valid_start_date"] = record.get("validStart")
+            row["valid_end_date"] = record.get("validEnd")
+            row["last_consume_time"] = record.get("lastConsumeTime")
+            row["status"] = record.get("status")
+            row["activate_time"] = record.get("activateTime")
+            row["deactivate_time"] = record.get("cancelTime")
+            row["issuer_id"] = record.get("issuerId")
+            row["issuer_name"] = record.get("issuerName")
+
+        if "ods_recharge_record" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["member_id"] = pick(record, "memberId", "member_id")
+            row["recharge_amount"] = record.get("amount") or record.get("rechargeAmount")
+            row["gift_amount"] = record.get("giftAmount")
+            row["pay_method"] = record.get("payType") or record.get("pay_method")
+            row["pay_trade_no"] = record.get("payTradeNo")
+            row["order_trade_no"] = record.get("orderTradeNo")
+            row["recharge_time"] = record.get("createTime") or record.get("rechargeTime")
+            row["status"] = record.get("status")
+            row["operator_id"] = record.get("operatorId")
+            row["operator_name"] = record.get("operatorName")
+
+        if "ods_balance_change" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["site_id"] = row.get("site_id") or pick(record, "siteId", "site_id")
+            row["member_id"] = pick(record, "memberId", "member_id")
+            row["change_amount"] = record.get("change_amount")
+            row["balance_before"] = record.get("before_balance")
+            row["balance_after"] = record.get("after_balance")
+            row["change_type"] = record.get("from_type") or record.get("type")
+            row["relate_id"] = record.get("relate_id")
+            row["pay_method"] = record.get("pay_type")
+            row["remark"] = record.get("remark")
+            row["operator_id"] = record.get("operatorId")
+            row["operator_name"] = record.get("operatorName")
+            row["change_time"] = record.get("create_time") or record.get("changeTime")
+            row["is_deleted"] = record.get("is_delete") or record.get("is_deleted")
+            row["source_file"] = row.get("source_file")
+            row["fetched_at"] = row.get("fetched_at")
+
+        if "ods_assistant_account" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["assistant_name"] = record.get("assistantName") or record.get("name")
+            row["mobile"] = record.get("mobile")
+            row["team_id"] = record.get("teamId")
+            row["team_name"] = record.get("teamName")
+            row["status"] = record.get("status")
+            row["hired_date"] = record.get("hireDate")
+            row["left_date"] = record.get("leaveDate")
+
+        if "ods_assistant_service_log" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["assistant_id"] = record.get("assistantId")
+            row["service_type"] = record.get("serviceType")
+            row["order_trade_no"] = record.get("orderTradeNo")
+            row["order_settle_id"] = record.get("orderSettleId")
+            row["start_time"] = record.get("startTime")
+            row["end_time"] = record.get("endTime")
+            row["duration_minutes"] = record.get("duration")
+            row["original_fee"] = record.get("originFee") or record.get("original_fee")
+            row["discount_amount"] = record.get("discountAmount")
+            row["final_fee"] = record.get("finalFee") or record.get("final_fee")
+            row["member_id"] = record.get("memberId")
+            row["status"] = record.get("status")
+
+        if "ods_assistant_cancel_log" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["ledger_id"] = record.get("ledgerId")
+            row["assistant_id"] = record.get("assistantId")
+            row["order_trade_no"] = record.get("orderTradeNo")
+            row["reason"] = record.get("reason")
+            row["cancel_time"] = record.get("cancel_time") or record.get("cancelTime")
+            row["operator_id"] = record.get("operatorId")
+            row["operator_name"] = record.get("operatorName")
+
+        if "ods_table_info" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["table_code"] = record.get("tableCode")
+            row["table_name"] = record.get("tableName")
+            row["table_type"] = record.get("tableType")
+            row["area_name"] = record.get("areaName")
+            row["status"] = record.get("status")
+            row["created_time"] = record.get("createTime")
+            row["updated_time"] = record.get("updateTime")
+
+        if "ods_table_use_log" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["table_id"] = record.get("tableId")
+            row["order_trade_no"] = record.get("orderTradeNo")
+            row["order_settle_id"] = record.get("orderSettleId")
+            row["start_time"] = record.get("startTime")
+            row["end_time"] = record.get("endTime")
+            row["duration_minutes"] = record.get("duration")
+            row["original_table_fee"] = record.get("originFee") or record.get("original_table_fee")
+            row["discount_amount"] = record.get("discountAmount")
+            row["final_table_fee"] = record.get("finalFee") or record.get("final_table_fee")
+            row["member_id"] = record.get("memberId")
+            row["status"] = record.get("status")
+
+        if "ods_table_fee_adjust" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["ledger_id"] = record.get("ledgerId")
+            row["order_trade_no"] = record.get("orderTradeNo")
+            row["discount_amount"] = record.get("discountAmount")
+            row["reason"] = record.get("reason")
+            row["operator_id"] = record.get("operatorId")
+            row["operator_name"] = record.get("operatorName")
+            row["created_at"] = record.get("created_at") or record.get("createTime")
+
+        if "ods_store_product" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["goods_id"] = record.get("goodsId")
+            row["goods_name"] = record.get("goodsName")
+            row["category_id"] = record.get("categoryId")
+            row["category_name"] = record.get("categoryName")
+            row["sale_price"] = record.get("salePrice")
+            row["cost_price"] = record.get("costPrice")
+            row["status"] = record.get("status")
+
+        if "ods_store_sale_item" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["order_trade_no"] = record.get("orderTradeNo")
+            row["order_settle_id"] = record.get("orderSettleId")
+            row["goods_id"] = record.get("goodsId")
+            row["goods_name"] = record.get("goodsName")
+            row["category_id"] = record.get("categoryId")
+            row["quantity"] = record.get("quantity")
+            row["original_amount"] = record.get("originalAmount")
+            row["discount_amount"] = record.get("discountAmount")
+            row["final_amount"] = record.get("finalAmount")
+            row["is_gift"] = record.get("isGift")
+            row["sale_time"] = record.get("saleTime")
+
+        if "ods_group_package_log" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["package_id"] = record.get("packageId")
+            row["coupon_id"] = record.get("couponId")
+            row["order_trade_no"] = record.get("orderTradeNo")
+            row["order_settle_id"] = record.get("orderSettleId")
+            row["member_id"] = record.get("memberId")
+            row["status"] = record.get("status")
+            row["used_time"] = record.get("usedTime")
+            row["deduct_amount"] = record.get("deductAmount")
+            row["settle_price"] = record.get("settlePrice")
+
+        if "ods_group_package" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["package_name"] = record.get("packageName")
+            row["platform_code"] = record.get("platformCode")
+            row["status"] = record.get("status")
+            row["face_price"] = record.get("facePrice")
+            row["settle_price"] = record.get("settlePrice")
+            row["valid_from"] = record.get("validFrom")
+            row["valid_to"] = record.get("validTo")
+
+        if "ods_platform_coupon_log" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["platform_code"] = record.get("platformCode")
+            row["verify_code"] = record.get("verifyCode")
+            row["order_trade_no"] = record.get("orderTradeNo")
+            row["order_settle_id"] = record.get("orderSettleId")
+            row["member_id"] = record.get("memberId")
+            row["status"] = record.get("status")
+            row["used_time"] = record.get("usedTime")
+            row["deduct_amount"] = record.get("deductAmount")
+            row["settle_price"] = record.get("settlePrice")
+
+        if "ods_payment_record" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["order_trade_no"] = record.get("orderTradeNo")
+            row["order_settle_id"] = record.get("orderSettleId")
+            row["member_id"] = record.get("memberId")
+            row["pay_method_code"] = record.get("payMethodCode") or record.get("pay_type")
+            row["pay_method_name"] = record.get("payMethodName")
+            row["pay_amount"] = record.get("payAmount")
+            row["pay_time"] = record.get("payTime")
+            row["relate_type"] = record.get("relateType")
+            row["relate_id"] = record.get("relateId")
+
+        if "ods_refund_record" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["order_trade_no"] = record.get("orderTradeNo")
+            row["order_settle_id"] = record.get("orderSettleId")
+            row["member_id"] = record.get("memberId")
+            row["pay_method_code"] = record.get("payMethodCode")
+            row["refund_amount"] = record.get("refundAmount")
+            row["refund_time"] = record.get("refundTime")
+            row["status"] = record.get("status")
+
+        if "ods_inventory_change" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["site_goods_id"] = record.get("siteGoodsId")
+            row["goods_id"] = record.get("goodsId")
+            row["change_amount"] = record.get("changeAmount")
+            row["before_stock"] = record.get("beforeStock")
+            row["after_stock"] = record.get("afterStock")
+            row["change_type"] = record.get("changeType")
+            row["relate_id"] = record.get("relateId")
+            row["remark"] = record.get("remark")
+            row["operator_id"] = record.get("operatorId")
+            row["operator_name"] = record.get("operatorName")
+            row["change_time"] = record.get("changeTime")
+
+        if "ods_inventory_stock" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["goods_id"] = record.get("goodsId")
+            row["current_stock"] = record.get("currentStock")
+            row["cost_price"] = record.get("costPrice")
+
+        if "ods_goods_category" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["category_name"] = record.get("categoryName")
+            row["parent_id"] = record.get("parentId")
+            row["level_no"] = record.get("levelNo")
+            row["status"] = record.get("status")
+            row["remark"] = record.get("remark")
+
+        if "ods_order_receipt_detail" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["order_trade_no"] = record.get("orderTradeNo")
+            row["receipt_no"] = record.get("receiptNo")
+            row["receipt_time"] = record.get("receiptTime")
+            row["total_amount"] = record.get("totalAmount")
+            row["discount_amount"] = record.get("discountAmount")
+            row["final_amount"] = record.get("finalAmount")
+            row["member_id"] = record.get("memberId")
+            row["snapshot_raw"] = record.get("siteProfile") or record.get("site_profile")
+
+        if "ods_order_settle" in table:
+            settle = record.get("settleList") if isinstance(record.get("settleList"), dict) else record
+            if isinstance(settle, dict):
+                row["tenant_id"] = pick(settle, "tenantId", "tenant_id")
+                row["settle_relate_id"] = settle.get("settleRelateId")
+                row["settle_name"] = settle.get("settleName")
+                row["settle_type"] = settle.get("settleType")
+                row["settle_status"] = settle.get("settleStatus")
+                row["member_id"] = settle.get("memberId")
+                row["member_phone"] = settle.get("memberPhone")
+                row["table_id"] = settle.get("tableId")
+                row["consume_money"] = settle.get("consumeMoney")
+                row["table_charge_money"] = settle.get("tableChargeMoney")
+                row["goods_money"] = settle.get("goodsMoney")
+                row["service_money"] = settle.get("serviceMoney")
+                row["assistant_pd_money"] = settle.get("assistantPdMoney")
+                row["assistant_cx_money"] = settle.get("assistantCxMoney")
+                row["pay_amount"] = settle.get("payAmount")
+                row["coupon_amount"] = settle.get("couponAmount")
+                row["card_amount"] = settle.get("cardAmount")
+                row["balance_amount"] = settle.get("balanceAmount")
+                row["refund_amount"] = settle.get("refundAmount")
+                row["prepay_money"] = settle.get("prepayMoney")
+                row["adjust_amount"] = settle.get("adjustAmount")
+                row["rounding_amount"] = settle.get("roundingAmount")
+                row["payment_method"] = settle.get("paymentMethod")
+                row["create_time"] = settle.get("createTime")
+                row["pay_time"] = settle.get("payTime")
+                row["operator_id"] = settle.get("operatorId")
+                row["operator_name"] = settle.get("operatorName")
+
+        if "ods_product" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+            row["goods_id"] = record.get("goodsId")
+            row["goods_name"] = record.get("goodsName")
+            row["goods_code"] = record.get("goodsCode")
+            row["category_id"] = record.get("categoryId")
+            row["category_name"] = record.get("categoryName")
+            row["unit"] = record.get("unit")
+            row["price"] = record.get("price")
+            row["status"] = record.get("status")
+
+        if "ods_platform_coupon_log" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+
+        if "ods_table_use_log" in table:
+            row["tenant_id"] = pick(record, "tenantId", "tenant_id")
+
+    def _ensure_tuple(self, value):
+        if isinstance(value, tuple):
+            return value
+        return (value,)
+
+    def _bulk_insert(self, table: str, rows: list[dict]):
+        if not rows:
+            return
+
+        columns = list(rows[0].keys())
+        col_clause = ", ".join(columns)
+        val_clause = ", ".join(f"%({col})s" for col in columns)
+        conflict_cols = ["site_id"] + self._get_conflict_columns(table)
+        conflict_clause = ", ".join(conflict_cols)
 
-    def _bulk_insert(self, table, rows):
-        if not rows: return
-        
-        keys = list(rows[0].keys())
-        cols = ", ".join(keys)
-        vals = ", ".join([f"%({k})s" for k in keys])
-        
-        # Determine PK col for conflict
-        pk_col = self._get_pk_column(table)
-        
         sql = f"""
-            INSERT INTO {table} ({cols})
-            VALUES ({vals})
-            ON CONFLICT (store_id, {pk_col}) DO UPDATE SET
+            INSERT INTO {table} ({col_clause})
+            VALUES ({val_clause})
+            ON CONFLICT ({conflict_clause}) DO UPDATE SET
                 payload = EXCLUDED.payload,
                 fetched_at = EXCLUDED.fetched_at,
-                source_file = EXCLUDED.source_file;
+                source_file = EXCLUDED.source_file
         """
         self.db.batch_execute(sql, rows)

etl_billiards/tasks/members_task.py

@@ -1,73 +1,72 @@
 # -*- coding: utf-8 -*-
 """会员ETL任务"""
 import json
-from .base_task import BaseTask
+
+from .base_task import BaseTask, TaskContext
 from loaders.dimensions.member import MemberLoader
 from models.parsers import TypeParser
 
+
 class MembersTask(BaseTask):
     """会员ETL任务"""
-    
+
     def get_task_code(self) -> str:
         return "MEMBERS"
-    
-    def execute(self) -> dict:
-        """执行会员ETL"""
-        self.logger.info(f"开始执行 {self.get_task_code()} 任务")
-        
-        params = {
-            "storeId": self.config.get("app.store_id"),
+
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params({"siteId": context.store_id})
+        records, _ = self.api.get_paginated(
+            endpoint="/MemberProfile/GetTenantMemberList",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+            list_key="tenantMemberInfos",
+        )
+        return {"records": records}
+
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            parsed_row = self._parse_member(raw, context.store_id)
+            if parsed_row:
+                parsed.append(parsed_row)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
         }
-        
-        try:
-            records, pages_meta = self.api.get_paginated(
-                endpoint="/MemberProfile/GetTenantMemberList",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data",)
-            )
-            
-            parsed_records = []
-            for rec in records:
-                parsed = self._parse_member(rec)
-                if parsed:
-                    parsed_records.append(parsed)
-            
-            loader = MemberLoader(self.db)
-            store_id = self.config.get("app.store_id")
-            inserted, updated, skipped = loader.upsert_members(parsed_records, store_id)
-            
-            self.db.commit()
-            
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0
-            }
-            
-            self.logger.info(f"{self.get_task_code()} 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-            
-        except Exception as e:
-            self.db.rollback()
-            self.logger.error(f"{self.get_task_code()} 失败", exc_info=True)
-            raise
-    
-    def _parse_member(self, raw: dict) -> dict:
+
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = MemberLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_members(
+            transformed["records"], context.store_id
+        )
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
+
+    def _parse_member(self, raw: dict, store_id: int) -> dict | None:
         """解析会员记录"""
         try:
+            member_id = TypeParser.parse_int(raw.get("memberId"))
+            if not member_id:
+                return None
             return {
-                "store_id": self.config.get("app.store_id"),
-                "member_id": TypeParser.parse_int(raw.get("memberId")),
+                "store_id": store_id,
+                "member_id": member_id,
                 "member_name": raw.get("memberName"),
                 "phone": raw.get("phone"),
                 "balance": TypeParser.parse_decimal(raw.get("balance")),
                 "status": raw.get("status"),
                 "register_time": TypeParser.parse_timestamp(raw.get("registerTime"), self.tz),
-                "raw_data": json.dumps(raw, ensure_ascii=False)
+                "raw_data": json.dumps(raw, ensure_ascii=False),
             }
-        except Exception as e:
-            self.logger.warning(f"解析会员记录失败: {e}, 原始数据: {raw}")
+        except Exception as exc:
+            self.logger.warning("解析会员记录失败: %s, 原始数据: %s", exc, raw)
             return None

etl_billiards/tasks/ods_tasks.py

@@ -38,10 +38,12 @@ class OdsTaskSpec:
     pk_columns: Tuple[ColumnSpec, ...] = ()
     extra_columns: Tuple[ColumnSpec, ...] = ()
     include_page_size: bool = False
-    include_page_no: bool = True
+    include_page_no: bool = False
     include_source_file: bool = True
     include_source_endpoint: bool = True
     requires_window: bool = True
+    time_fields: Tuple[str, str] | None = ("startTime", "endTime")
+    include_site_id: bool = True
     description: str = ""
     extra_params: Dict[str, Any] = field(default_factory=dict)
 
@@ -65,7 +67,7 @@ class BaseOdsTask(BaseTask):
         page_size = self.config.get("api.page_size", 200)
         params = self._build_params(spec, store_id)
         columns = self._resolve_columns(spec)
-        conflict_columns = ["store_id"] + [col.column for col in spec.pk_columns]
+        conflict_columns = ["site_id"] + [col.column for col in spec.pk_columns]
         loader = GenericODSLoader(
             self.db,
             spec.table_name,
@@ -117,16 +119,21 @@ class BaseOdsTask(BaseTask):
             raise
 
     def _build_params(self, spec: OdsTaskSpec, store_id: int) -> dict:
-        params: dict[str, Any] = {"storeId": store_id}
-        params.update(spec.extra_params)
-        if spec.requires_window:
+        base: dict[str, Any] = {}
+        if spec.include_site_id:
+            base["siteId"] = store_id
+        if spec.requires_window and spec.time_fields:
             window_start, window_end, _ = self._get_time_window()
-            params["startTime"] = TypeParser.format_timestamp(window_start, self.tz)
-            params["endTime"] = TypeParser.format_timestamp(window_end, self.tz)
+            start_key, end_key = spec.time_fields
+            base[start_key] = TypeParser.format_timestamp(window_start, self.tz)
+            base[end_key] = TypeParser.format_timestamp(window_end, self.tz)
+
+        params = self._merge_common_params(base)
+        params.update(spec.extra_params)
         return params
 
     def _resolve_columns(self, spec: OdsTaskSpec) -> List[str]:
-        columns: List[str] = ["store_id"]
+        columns: List[str] = ["site_id"]
         seen = set(columns)
         for col_spec in list(spec.pk_columns) + list(spec.extra_columns):
             if col_spec.column not in seen:
@@ -166,7 +173,7 @@ class BaseOdsTask(BaseTask):
         page_size_value: int | None,
         source_file: str | None,
     ) -> dict | None:
-        row: dict[str, Any] = {"store_id": store_id}
+        row: dict[str, Any] = {"site_id": store_id}
 
         for col_spec in spec.pk_columns + spec.extra_columns:
             value = self._extract_value(record, col_spec)
@@ -238,19 +245,33 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
         code="ODS_ORDER_SETTLE",
         class_name="OdsOrderSettleTask",
         table_name="billiards_ods.ods_order_settle",
-        endpoint="/order/list",
+        endpoint="/Site/GetAllOrderSettleList",
         data_path=("data",),
-        pk_columns=(_int_col("order_settle_id", "orderSettleId", "order_settle_id", "id", required=True),),
-        extra_columns=(_int_col("order_trade_no", "orderTradeNo", "order_trade_no"),),
-        include_page_size=True,
+        list_key="settleList",
+        pk_columns=(
+            _int_col(
+                "order_settle_id",
+                "orderSettleId",
+                "order_settle_id",
+                "settleList.id",
+                "id",
+                required=True,
+            ),
+        ),
+        extra_columns=(
+            _int_col("order_trade_no", "orderTradeNo", "order_trade_no", "settleList.orderTradeNo"),
+        ),
+        include_page_size=False,
+        time_fields=("rangeStartTime", "rangeEndTime"),
         description="订单/结算 ODS 原始记录",
     ),
     OdsTaskSpec(
         code="ODS_TABLE_USE",
         class_name="OdsTableUseTask",
-        table_name="billiards_ods.ods_table_use_detail",
-        endpoint="/Table/UseDetailList",
-        data_path=("data", "siteTableUseDetailsList"),
+        table_name="billiards_ods.ods_table_use_log",
+        endpoint="/Site/GetSiteTableOrderDetails",
+        data_path=("data",),
+        list_key="siteTableUseDetailsList",
         pk_columns=(_int_col("ledger_id", "id", required=True),),
         extra_columns=(
             _int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
@@ -261,9 +282,10 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
     OdsTaskSpec(
         code="ODS_ASSISTANT_LEDGER",
         class_name="OdsAssistantLedgerTask",
-        table_name="billiards_ods.ods_assistant_ledger",
-        endpoint="/Assistant/LedgerList",
-        data_path=("data", "orderAssistantDetails"),
+        table_name="billiards_ods.ods_assistant_service_log",
+        endpoint="/AssistantPerformance/GetOrderAssistantDetails",
+        data_path=("data",),
+        list_key="orderAssistantDetails",
         pk_columns=(_int_col("ledger_id", "id", required=True),),
         extra_columns=(
             _int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
@@ -274,18 +296,20 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
     OdsTaskSpec(
         code="ODS_ASSISTANT_ABOLISH",
         class_name="OdsAssistantAbolishTask",
-        table_name="billiards_ods.ods_assistant_abolish",
-        endpoint="/Assistant/AbolishList",
-        data_path=("data", "abolitionAssistants"),
+        table_name="billiards_ods.ods_assistant_cancel_log",
+        endpoint="/AssistantPerformance/GetAbolitionAssistant",
+        data_path=("data",),
+        list_key="abolitionAssistants",
         pk_columns=(_int_col("abolish_id", "id", required=True),),
         description="助教作废记录 ODS",
     ),
     OdsTaskSpec(
         code="ODS_GOODS_LEDGER",
         class_name="OdsGoodsLedgerTask",
-        table_name="billiards_ods.ods_goods_ledger",
-        endpoint="/Order/GoodsLedgerList",
-        data_path=("data", "orderGoodsLedgers"),
+        table_name="billiards_ods.ods_store_sale_item",
+        endpoint="/TenantGoods/GetGoodsSalesList",
+        data_path=("data",),
+        list_key="orderGoodsLedgers",
         pk_columns=(_int_col("order_goods_id", "orderGoodsId", "id", required=True),),
         extra_columns=(
             _int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
@@ -296,8 +320,8 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
     OdsTaskSpec(
         code="ODS_PAYMENT",
         class_name="OdsPaymentTask",
-        table_name="billiards_ods.ods_payment",
-        endpoint="/pay/records",
+        table_name="billiards_ods.ods_payment_record",
+        endpoint="/PayLog/GetPayLogListPage",
         data_path=("data",),
         pk_columns=(_int_col("pay_id", "payId", "id", required=True),),
         extra_columns=(
@@ -305,14 +329,15 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
             _int_col("relate_id", "relate_id", "relateId"),
         ),
         include_page_size=False,
+        time_fields=("StartPayTime", "EndPayTime"),
         description="支付流水 ODS",
     ),
     OdsTaskSpec(
         code="ODS_REFUND",
         class_name="OdsRefundTask",
-        table_name="billiards_ods.ods_refund",
-        endpoint="/Pay/RefundList",
-        data_path=(),
+        table_name="billiards_ods.ods_refund_record",
+        endpoint="/Order/GetRefundPayLogList",
+        data_path=("data",),
         pk_columns=(_int_col("refund_id", "id", required=True),),
         extra_columns=(
             ColumnSpec(column="relate_type", sources=("relate_type", "relateType")),
@@ -323,18 +348,19 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
     OdsTaskSpec(
         code="ODS_COUPON_VERIFY",
         class_name="OdsCouponVerifyTask",
-        table_name="billiards_ods.ods_coupon_verify",
-        endpoint="/Coupon/UsageList",
-        data_path=(),
+        table_name="billiards_ods.ods_platform_coupon_log",
+        endpoint="/Promotion/GetOfflineCouponConsumePageList",
+        data_path=("data",),
         pk_columns=(_int_col("coupon_id", "id", "couponId", required=True),),
         description="平台验券/团购流水 ODS",
     ),
     OdsTaskSpec(
         code="ODS_MEMBER",
         class_name="OdsMemberTask",
-        table_name="billiards_ods.ods_member",
+        table_name="billiards_ods.ods_member_profile",
         endpoint="/MemberProfile/GetTenantMemberList",
         data_path=("data",),
+        list_key="tenantMemberInfos",
         pk_columns=(_int_col("member_id", "memberId", required=True),),
         requires_window=False,
         description="会员档案 ODS",
@@ -343,8 +369,9 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
         code="ODS_MEMBER_CARD",
         class_name="OdsMemberCardTask",
         table_name="billiards_ods.ods_member_card",
-        endpoint="/MemberCard/List",
-        data_path=("data", "tenantMemberCards"),
+        endpoint="/MemberProfile/GetTenantMemberCardList",
+        data_path=("data",),
+        list_key="tenantMemberCards",
         pk_columns=(_int_col("card_id", "tenantMemberCardId", "cardId", required=True),),
         requires_window=False,
         description="会员卡/储值卡 ODS",
@@ -352,9 +379,10 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
     OdsTaskSpec(
         code="ODS_PACKAGE",
         class_name="OdsPackageTask",
-        table_name="billiards_ods.ods_package_coupon",
-        endpoint="/Package/List",
-        data_path=("data", "packageCouponList"),
+        table_name="billiards_ods.ods_group_package",
+        endpoint="/PackageCoupon/QueryPackageCouponList",
+        data_path=("data",),
+        list_key="packageCouponList",
         pk_columns=(_int_col("package_id", "id", "packageId", required=True),),
         requires_window=False,
         description="团购/套餐定义 ODS",
@@ -363,8 +391,8 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
         code="ODS_INVENTORY_STOCK",
         class_name="OdsInventoryStockTask",
         table_name="billiards_ods.ods_inventory_stock",
-        endpoint="/Inventory/StockSummary",
-        data_path=(),
+        endpoint="/TenantGoods/GetGoodsStockReport",
+        data_path=("data",),
         pk_columns=(
             _int_col("site_goods_id", "siteGoodsId", required=True),
             ColumnSpec(column="snapshot_key", default="default", required=True),
@@ -376,8 +404,9 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
         code="ODS_INVENTORY_CHANGE",
         class_name="OdsInventoryChangeTask",
         table_name="billiards_ods.ods_inventory_change",
-        endpoint="/Inventory/ChangeList",
-        data_path=("data", "queryDeliveryRecordsList"),
+        endpoint="/GoodsStockManage/QueryGoodsOutboundReceipt",
+        data_path=("data",),
+        list_key="queryDeliveryRecordsList",
         pk_columns=(_int_col("change_id", "siteGoodsStockId", "id", required=True),),
         description="库存变动 ODS",
     ),

etl_billiards/tasks/orders_task.py

@@ -1,80 +1,77 @@
 # -*- coding: utf-8 -*-
 """订单ETL任务"""
 import json
-from .base_task import BaseTask
+
+from .base_task import BaseTask, TaskContext
 from loaders.facts.order import OrderLoader
 from models.parsers import TypeParser
 
+
 class OrdersTask(BaseTask):
     """订单数据ETL任务"""
-    
+
     def get_task_code(self) -> str:
         return "ORDERS"
-    
-    def execute(self) -> dict:
-        """执行订单数据ETL"""
-        self.logger.info(f"开始执行 {self.get_task_code()} 任务")
-        
-        # 1. 获取时间窗口
-        window_start, window_end, window_minutes = self._get_time_window()
-        
-        # 2. 调用API获取数据
-        params = {
-            "storeId": self.config.get("app.store_id"),
-            "startTime": TypeParser.format_timestamp(window_start, self.tz),
-            "endTime": TypeParser.format_timestamp(window_end, self.tz),
-        }
-        
-        try:
-            records, pages_meta = self.api.get_paginated(
-                endpoint="/order/list",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data",)
-            )
-            
-            # 3. 解析并清洗数据
-            parsed_records = []
-            for rec in records:
-                parsed = self._parse_order(rec)
-                if parsed:
-                    parsed_records.append(parsed)
-            
-            # 4. 加载数据
-            loader = OrderLoader(self.db)
-            store_id = self.config.get("app.store_id")
-            inserted, updated, skipped = loader.upsert_orders(
-                parsed_records,
-                store_id
-            )
-            
-            # 5. 提交事务
-            self.db.commit()
-            
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0
+
+    # ------------------------------------------------------------------ E/T/L hooks
+    def extract(self, context: TaskContext) -> dict:
+        """调用 API 拉取订单记录"""
+        params = self._merge_common_params(
+            {
+                "siteId": context.store_id,
+                "rangeStartTime": TypeParser.format_timestamp(context.window_start, self.tz),
+                "rangeEndTime": TypeParser.format_timestamp(context.window_end, self.tz),
             }
-            
-            self.logger.info(
-                f"{self.get_task_code()} 完成: {counts}"
-            )
-            
-            return self._build_result("SUCCESS", counts)
-            
-        except Exception as e:
-            self.db.rollback()
-            self.logger.error(f"{self.get_task_code()} 失败", exc_info=True)
-            raise
-    
-    def _parse_order(self, raw: dict) -> dict:
+        )
+        records, pages_meta = self.api.get_paginated(
+            endpoint="/Site/GetAllOrderSettleList",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+            list_key="settleList",
+        )
+        return {"records": records, "meta": pages_meta}
+
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        """解析原始订单 JSON"""
+        parsed_records = []
+        skipped = 0
+
+        for rec in extracted.get("records", []):
+            parsed = self._parse_order(rec, context.store_id)
+            if parsed:
+                parsed_records.append(parsed)
+            else:
+                skipped += 1
+
+        return {
+            "records": parsed_records,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
+        }
+
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        """写入 fact_order"""
+        loader = OrderLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_orders(
+            transformed["records"], context.store_id
+        )
+
+        counts = {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
+        return counts
+
+    # ------------------------------------------------------------------ helpers
+    def _parse_order(self, raw: dict, store_id: int) -> dict | None:
         """解析单条订单记录"""
         try:
             return {
-                "store_id": self.config.get("app.store_id"),
+                "store_id": store_id,
                 "order_id": TypeParser.parse_int(raw.get("orderId")),
                 "order_no": raw.get("orderNo"),
                 "member_id": TypeParser.parse_int(raw.get("memberId")),
@@ -87,8 +84,8 @@ class OrdersTask(BaseTask):
                 "pay_status": raw.get("payStatus"),
                 "order_status": raw.get("orderStatus"),
                 "remark": raw.get("remark"),
-                "raw_data": json.dumps(raw, ensure_ascii=False)
+                "raw_data": json.dumps(raw, ensure_ascii=False),
             }
-        except Exception as e:
-            self.logger.warning(f"解析订单失败: {e}, 原始数据: {raw}")
+        except Exception as exc:
+            self.logger.warning("解析订单失败: %s, 原始数据: %s", exc, raw)
             return None

etl_billiards/tasks/packages_task.py

@@ -3,7 +3,7 @@
 
 import json
 
-from .base_task import BaseTask
+from .base_task import BaseTask, TaskContext
 from loaders.dimensions.package import PackageDefinitionLoader
 from models.parsers import TypeParser
 
@@ -14,49 +14,48 @@ class PackagesDefTask(BaseTask):
     def get_task_code(self) -> str:
         return "PACKAGES_DEF"
 
-    def execute(self) -> dict:
-        self.logger.info("开始执行 PACKAGES_DEF 任务")
-        params = {"storeId": self.config.get("app.store_id")}
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params({"siteId": context.store_id})
+        records, _ = self.api.get_paginated(
+            endpoint="/PackageCoupon/QueryPackageCouponList",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+            list_key="packageCouponList",
+        )
+        return {"records": records}
 
-        try:
-            records, _ = self.api.get_paginated(
-                endpoint="/Package/List",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data", "packageCouponList"),
-            )
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            mapped = self._parse_package(raw, context.store_id)
+            if mapped:
+                parsed.append(mapped)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
+        }
 
-            parsed = []
-            for raw in records:
-                mapped = self._parse_package(raw)
-                if mapped:
-                    parsed.append(mapped)
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = PackageDefinitionLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_packages(transformed["records"])
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
 
-            loader = PackageDefinitionLoader(self.db)
-            inserted, updated, skipped = loader.upsert_packages(parsed)
-
-            self.db.commit()
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0,
-            }
-            self.logger.info(f"PACKAGES_DEF 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-        except Exception:
-            self.db.rollback()
-            self.logger.error("PACKAGES_DEF 失败", exc_info=True)
-            raise
-
-    def _parse_package(self, raw: dict) -> dict | None:
+    def _parse_package(self, raw: dict, store_id: int) -> dict | None:
         package_id = TypeParser.parse_int(raw.get("id"))
         if not package_id:
-            self.logger.warning("跳过缺少 id 的套餐数据: %s", raw)
+            self.logger.warning("跳过缺少 package id 的套餐记录: %s", raw)
             return None
 
-        store_id = self.config.get("app.store_id")
         return {
             "store_id": store_id,
             "package_id": package_id,

etl_billiards/tasks/payments_task.py

@@ -1,68 +1,70 @@
 # -*- coding: utf-8 -*-
 """支付记录ETL任务"""
 import json
-from .base_task import BaseTask
+
+from .base_task import BaseTask, TaskContext
 from loaders.facts.payment import PaymentLoader
 from models.parsers import TypeParser
 
+
 class PaymentsTask(BaseTask):
-    """支付记录ETL任务"""
-    
+    """支付记录 E/T/L 任务"""
+
     def get_task_code(self) -> str:
         return "PAYMENTS"
-    
-    def execute(self) -> dict:
-        """执行支付记录ETL"""
-        self.logger.info(f"开始执行 {self.get_task_code()} 任务")
-        
-        window_start, window_end, window_minutes = self._get_time_window()
-        
-        params = {
-            "storeId": self.config.get("app.store_id"),
-            "startTime": TypeParser.format_timestamp(window_start, self.tz),
-            "endTime": TypeParser.format_timestamp(window_end, self.tz),
-        }
-        
-        try:
-            records, pages_meta = self.api.get_paginated(
-                endpoint="/pay/records",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data",)
-            )
-            
-            parsed_records = []
-            for rec in records:
-                parsed = self._parse_payment(rec)
-                if parsed:
-                    parsed_records.append(parsed)
-            
-            loader = PaymentLoader(self.db)
-            store_id = self.config.get("app.store_id")
-            inserted, updated, skipped = loader.upsert_payments(parsed_records, store_id)
-            
-            self.db.commit()
-            
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0
+
+    # ------------------------------------------------------------------ E/T/L hooks
+    def extract(self, context: TaskContext) -> dict:
+        """调用 API 抓取支付记录"""
+        params = self._merge_common_params(
+            {
+                "siteId": context.store_id,
+                "StartPayTime": TypeParser.format_timestamp(context.window_start, self.tz),
+                "EndPayTime": TypeParser.format_timestamp(context.window_end, self.tz),
             }
-            
-            self.logger.info(f"{self.get_task_code()} 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-            
-        except Exception as e:
-            self.db.rollback()
-            self.logger.error(f"{self.get_task_code()} 失败", exc_info=True)
-            raise
-    
-    def _parse_payment(self, raw: dict) -> dict:
+        )
+        records, pages_meta = self.api.get_paginated(
+            endpoint="/PayLog/GetPayLogListPage",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+        )
+        return {"records": records, "meta": pages_meta}
+
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        """解析支付 JSON"""
+        parsed, skipped = [], 0
+        for rec in extracted.get("records", []):
+            cleaned = self._parse_payment(rec, context.store_id)
+            if cleaned:
+                parsed.append(cleaned)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
+        }
+
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        """写入 fact_payment"""
+        loader = PaymentLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_payments(
+            transformed["records"], context.store_id
+        )
+        counts = {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
+        return counts
+
+    # ------------------------------------------------------------------ helpers
+    def _parse_payment(self, raw: dict, store_id: int) -> dict | None:
         """解析支付记录"""
         try:
-            store_id = self.config.get("app.store_id")
             return {
                 "store_id": store_id,
                 "pay_id": TypeParser.parse_int(raw.get("payId") or raw.get("id")),
@@ -75,7 +77,9 @@ class PaymentsTask(BaseTask):
                 ),
                 "relate_type": raw.get("relateType") or raw.get("relate_type"),
                 "relate_id": TypeParser.parse_int(raw.get("relateId") or raw.get("relate_id")),
-                "site_id": TypeParser.parse_int(raw.get("siteId") or raw.get("site_id") or store_id),
+                "site_id": TypeParser.parse_int(
+                    raw.get("siteId") or raw.get("site_id") or store_id
+                ),
                 "tenant_id": TypeParser.parse_int(raw.get("tenantId") or raw.get("tenant_id")),
                 "pay_time": TypeParser.parse_timestamp(raw.get("payTime"), self.tz),
                 "create_time": TypeParser.parse_timestamp(
@@ -89,16 +93,19 @@ class PaymentsTask(BaseTask):
                     or raw.get("fee_amount")
                 ),
                 "discount_amount": TypeParser.parse_decimal(
-                    raw.get("discountAmount") or raw.get("couponAmount") or raw.get("discount_amount")
+                    raw.get("discountAmount")
+                    or raw.get("couponAmount")
+                    or raw.get("discount_amount")
                 ),
                 "pay_type": raw.get("payType"),
                 "payment_method": raw.get("paymentMethod") or raw.get("payment_method"),
-                "online_pay_channel": raw.get("onlinePayChannel") or raw.get("online_pay_channel"),
+                "online_pay_channel": raw.get("onlinePayChannel")
+                or raw.get("online_pay_channel"),
                 "pay_status": raw.get("payStatus"),
                 "pay_terminal": raw.get("payTerminal") or raw.get("pay_terminal"),
                 "remark": raw.get("remark"),
                 "raw_data": json.dumps(raw, ensure_ascii=False),
             }
-        except Exception as e:
-            self.logger.warning(f"解析支付记录失败: {e}, 原始数据: {raw}")
+        except Exception as exc:
+            self.logger.warning("解析支付记录失败: %s, 原始数据: %s", exc, raw)
             return None

etl_billiards/tasks/products_task.py

@@ -3,7 +3,7 @@
 
 import json
 
-from .base_task import BaseTask
+from .base_task import BaseTask, TaskContext
 from loaders.dimensions.product import ProductLoader
 from models.parsers import TypeParser
 
@@ -12,95 +12,56 @@ class ProductsTask(BaseTask):
     """商品维度 ETL 任务"""
 
     def get_task_code(self) -> str:
-        """任务代码，应与 etl_admin.etl_task.task_code 一致"""
         return "PRODUCTS"
 
-    def execute(self) -> dict:
-        """
-        执行商品档案 ETL
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params({"siteId": context.store_id})
+        records, _ = self.api.get_paginated(
+            endpoint="/TenantGoods/QueryTenantGoods",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+            list_key="tenantGoodsList",
+        )
+        return {"records": records}
 
-        流程：
-        1. 调用上游 /TenantGoods/QueryTenantGoods 分页拉取商品列表
-        2. 解析/清洗字段
-        3. 通过 ProductLoader 写入 dim_product 和 dim_product_price_scd
-        """
-        self.logger.info(f"开始执行 {self.get_task_code()} 任务")
-
-        params = {
-            "storeId": self.config.get("app.store_id"),
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            parsed_row = self._parse_product(raw, context.store_id)
+            if parsed_row:
+                parsed.append(parsed_row)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
         }
 
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = ProductLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_products(
+            transformed["records"], context.store_id
+        )
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
+
+    def _parse_product(self, raw: dict, store_id: int) -> dict | None:
         try:
-            # 1. 分页拉取数据
-            records, pages_meta = self.api.get_paginated(
-                endpoint="/TenantGoods/QueryTenantGoods",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data",),
-            )
-
-            # 2. 解析/清洗
-            parsed_records = []
-            for raw in records:
-                parsed = self._parse_product(raw)
-                if parsed:
-                    parsed_records.append(parsed)
-
-            # 3. 加载入库（维度主表 + 价格SCD2）
-            loader = ProductLoader(self.db)
-            store_id = self.config.get("app.store_id")
-            inserted, updated, skipped = loader.upsert_products(
-                parsed_records, store_id
-            )
-
-            # 4. 提交事务
-            self.db.commit()
-
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0,
-            }
-
-            self.logger.info(f"{self.get_task_code()} 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-
-        except Exception:
-            # 明确回滚，避免部分成功
-            self.db.rollback()
-            self.logger.error(f"{self.get_task_code()} 失败", exc_info=True)
-            raise
-
-    def _parse_product(self, raw: dict) -> dict | None:
-        """
-        解析单条商品记录，字段映射参考旧版 upsert_dim_product_and_price_scd
-
-        上游字段示例：
-        - siteGoodsId / tenantGoodsId / productId
-        - goodsName / productName
-        - tenantGoodsCategoryId / goodsCategoryId / categoryName / goodsCategorySecondId
-        - goodsUnit
-        - costPrice / goodsPrice / salePrice
-        - goodsState / status
-        - supplierId / barcode / isCombo
-        - createTime / updateTime
-        """
-        try:
-            product_id = (
-                TypeParser.parse_int(
-                    raw.get("siteGoodsId")
-                    or raw.get("tenantGoodsId")
-                    or raw.get("productId")
-                )
+            product_id = TypeParser.parse_int(
+                raw.get("siteGoodsId") or raw.get("tenantGoodsId") or raw.get("productId")
             )
             if not product_id:
-                # 主键缺失，直接跳过
                 return None
 
             return {
-                "store_id": self.config.get("app.store_id"),
+                "store_id": store_id,
                 "product_id": product_id,
                 "site_product_id": TypeParser.parse_int(raw.get("siteGoodsId")),
                 "product_name": raw.get("goodsName") or raw.get("productName"),
@@ -108,15 +69,12 @@ class ProductsTask(BaseTask):
                     raw.get("tenantGoodsCategoryId") or raw.get("goodsCategoryId")
                 ),
                 "category_name": raw.get("categoryName"),
-                "second_category_id": TypeParser.parse_int(
-                    raw.get("goodsCategorySecondId")
-                ),
+                "second_category_id": TypeParser.parse_int(raw.get("goodsCategorySecondId")),
                 "unit": raw.get("goodsUnit"),
                 "cost_price": TypeParser.parse_decimal(raw.get("costPrice")),
                 "sale_price": TypeParser.parse_decimal(
                     raw.get("goodsPrice") or raw.get("salePrice")
                 ),
-                # 旧版这里就是 None，如后面有明确字段可以再补
                 "allow_discount": None,
                 "status": raw.get("goodsState") or raw.get("status"),
                 "supplier_id": TypeParser.parse_int(raw.get("supplierId"))
@@ -126,14 +84,10 @@ class ProductsTask(BaseTask):
                 "is_combo": bool(raw.get("isCombo"))
                 if raw.get("isCombo") is not None
                 else None,
-                "created_time": TypeParser.parse_timestamp(
-                    raw.get("createTime"), self.tz
-                ),
-                "updated_time": TypeParser.parse_timestamp(
-                    raw.get("updateTime"), self.tz
-                ),
+                "created_time": TypeParser.parse_timestamp(raw.get("createTime"), self.tz),
+                "updated_time": TypeParser.parse_timestamp(raw.get("updateTime"), self.tz),
                 "raw_data": json.dumps(raw, ensure_ascii=False),
             }
-        except Exception as e:
-            self.logger.warning(f"解析商品记录失败: {e}, 原始数据: {raw}")
-            return None
+        except Exception as exc:
+            self.logger.warning("解析商品记录失败: %s, 原始数据: %s", exc, raw)
+            return None

etl_billiards/tasks/refunds_task.py

@@ -3,7 +3,7 @@
 
 import json
 
-from .base_task import BaseTask
+from .base_task import BaseTask, TaskContext
 from loaders.facts.refund import RefundLoader
 from models.parsers import TypeParser
 
@@ -14,54 +14,53 @@ class RefundsTask(BaseTask):
     def get_task_code(self) -> str:
         return "REFUNDS"
 
-    def execute(self) -> dict:
-        self.logger.info("开始执行 REFUNDS 任务")
-        window_start, window_end, _ = self._get_time_window()
-        params = {
-            "storeId": self.config.get("app.store_id"),
-            "startTime": TypeParser.format_timestamp(window_start, self.tz),
-            "endTime": TypeParser.format_timestamp(window_end, self.tz),
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params(
+            {
+                "siteId": context.store_id,
+                "startTime": TypeParser.format_timestamp(context.window_start, self.tz),
+                "endTime": TypeParser.format_timestamp(context.window_end, self.tz),
+            }
+        )
+        records, _ = self.api.get_paginated(
+            endpoint="/Order/GetRefundPayLogList",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+        )
+        return {"records": records}
+
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            mapped = self._parse_refund(raw, context.store_id)
+            if mapped:
+                parsed.append(mapped)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
         }
 
-        try:
-            records, _ = self.api.get_paginated(
-                endpoint="/Pay/RefundList",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=(),
-            )
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = RefundLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_refunds(transformed["records"])
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
 
-            parsed = []
-            for raw in records:
-                mapped = self._parse_refund(raw)
-                if mapped:
-                    parsed.append(mapped)
-
-            loader = RefundLoader(self.db)
-            inserted, updated, skipped = loader.upsert_refunds(parsed)
-
-            self.db.commit()
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0,
-            }
-            self.logger.info(f"REFUNDS 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-        except Exception:
-            self.db.rollback()
-            self.logger.error("REFUNDS 失败", exc_info=True)
-            raise
-
-    def _parse_refund(self, raw: dict) -> dict | None:
+    def _parse_refund(self, raw: dict, store_id: int) -> dict | None:
         refund_id = TypeParser.parse_int(raw.get("id"))
         if not refund_id:
-            self.logger.warning("跳过缺少 id 的退款记录: %s", raw)
+            self.logger.warning("跳过缺少退款ID的数据: %s", raw)
             return None
 
-        store_id = self.config.get("app.store_id")
         return {
             "store_id": store_id,
             "refund_id": refund_id,

etl_billiards/tasks/table_discount_task.py

@@ -3,7 +3,7 @@
 
 import json
 
-from .base_task import BaseTask
+from .base_task import BaseTask, TaskContext
 from loaders.facts.table_discount import TableDiscountLoader
 from models.parsers import TypeParser
 
@@ -14,55 +14,55 @@ class TableDiscountTask(BaseTask):
     def get_task_code(self) -> str:
         return "TABLE_DISCOUNT"
 
-    def execute(self) -> dict:
-        self.logger.info("开始执行 TABLE_DISCOUNT 任务")
-        window_start, window_end, _ = self._get_time_window()
-        params = {
-            "storeId": self.config.get("app.store_id"),
-            "startTime": TypeParser.format_timestamp(window_start, self.tz),
-            "endTime": TypeParser.format_timestamp(window_end, self.tz),
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params(
+            {
+                "siteId": context.store_id,
+                "startTime": TypeParser.format_timestamp(context.window_start, self.tz),
+                "endTime": TypeParser.format_timestamp(context.window_end, self.tz),
+            }
+        )
+        records, _ = self.api.get_paginated(
+            endpoint="/Site/GetTaiFeeAdjustList",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+            list_key="taiFeeAdjustInfos",
+        )
+        return {"records": records}
+
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            mapped = self._parse_discount(raw, context.store_id)
+            if mapped:
+                parsed.append(mapped)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
         }
 
-        try:
-            records, _ = self.api.get_paginated(
-                endpoint="/Table/AdjustList",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data", "taiFeeAdjustInfos"),
-            )
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = TableDiscountLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_discounts(transformed["records"])
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
 
-            parsed = []
-            for raw in records:
-                mapped = self._parse_discount(raw)
-                if mapped:
-                    parsed.append(mapped)
-
-            loader = TableDiscountLoader(self.db)
-            inserted, updated, skipped = loader.upsert_discounts(parsed)
-
-            self.db.commit()
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0,
-            }
-            self.logger.info(f"TABLE_DISCOUNT 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-        except Exception:
-            self.db.rollback()
-            self.logger.error("TABLE_DISCOUNT 失败", exc_info=True)
-            raise
-
-    def _parse_discount(self, raw: dict) -> dict | None:
+    def _parse_discount(self, raw: dict, store_id: int) -> dict | None:
         discount_id = TypeParser.parse_int(raw.get("id"))
         if not discount_id:
-            self.logger.warning("跳过缺少 id 的台费折扣记录: %s", raw)
+            self.logger.warning("跳过缺少折扣ID的记录: %s", raw)
             return None
 
         table_profile = raw.get("tableProfile") or {}
-        store_id = self.config.get("app.store_id")
         return {
             "store_id": store_id,
             "discount_id": discount_id,

etl_billiards/tasks/tables_task.py

@@ -3,7 +3,7 @@
 
 import json
 
-from .base_task import BaseTask
+from .base_task import BaseTask, TaskContext
 from loaders.dimensions.table import TableLoader
 from models.parsers import TypeParser
 
@@ -14,49 +14,48 @@ class TablesTask(BaseTask):
     def get_task_code(self) -> str:
         return "TABLES"
 
-    def execute(self) -> dict:
-        self.logger.info("开始执行 TABLES 任务")
-        params = {"storeId": self.config.get("app.store_id")}
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params({"siteId": context.store_id})
+        records, _ = self.api.get_paginated(
+            endpoint="/Table/GetSiteTables",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+            list_key="siteTables",
+        )
+        return {"records": records}
 
-        try:
-            records, _ = self.api.get_paginated(
-                endpoint="/Table/GetSiteTables",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data", "siteTables"),
-            )
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            mapped = self._parse_table(raw, context.store_id)
+            if mapped:
+                parsed.append(mapped)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
+        }
 
-            parsed = []
-            for raw in records:
-                mapped = self._parse_table(raw)
-                if mapped:
-                    parsed.append(mapped)
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = TableLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_tables(transformed["records"])
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
 
-            loader = TableLoader(self.db)
-            inserted, updated, skipped = loader.upsert_tables(parsed)
-
-            self.db.commit()
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0,
-            }
-            self.logger.info(f"TABLES 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-        except Exception:
-            self.db.rollback()
-            self.logger.error("TABLES 失败", exc_info=True)
-            raise
-
-    def _parse_table(self, raw: dict) -> dict | None:
+    def _parse_table(self, raw: dict, store_id: int) -> dict | None:
         table_id = TypeParser.parse_int(raw.get("id"))
         if not table_id:
             self.logger.warning("跳过缺少 table_id 的台桌记录: %s", raw)
             return None
 
-        store_id = self.config.get("app.store_id")
         return {
             "store_id": store_id,
             "table_id": table_id,

etl_billiards/tasks/topups_task.py

@@ -3,7 +3,7 @@
 
 import json
 
-from .base_task import BaseTask
+from .base_task import BaseTask, TaskContext
 from loaders.facts.topup import TopupLoader
 from models.parsers import TypeParser
 
@@ -14,55 +14,55 @@ class TopupsTask(BaseTask):
     def get_task_code(self) -> str:
         return "TOPUPS"
 
-    def execute(self) -> dict:
-        self.logger.info("开始执行 TOPUPS 任务")
-        window_start, window_end, _ = self._get_time_window()
-        params = {
-            "storeId": self.config.get("app.store_id"),
-            "startTime": TypeParser.format_timestamp(window_start, self.tz),
-            "endTime": TypeParser.format_timestamp(window_end, self.tz),
+    def extract(self, context: TaskContext) -> dict:
+        params = self._merge_common_params(
+            {
+                "siteId": context.store_id,
+                "rangeStartTime": TypeParser.format_timestamp(context.window_start, self.tz),
+                "rangeEndTime": TypeParser.format_timestamp(context.window_end, self.tz),
+            }
+        )
+        records, _ = self.api.get_paginated(
+            endpoint="/Site/GetRechargeSettleList",
+            params=params,
+            page_size=self.config.get("api.page_size", 200),
+            data_path=("data",),
+            list_key="settleList",
+        )
+        return {"records": records}
+
+    def transform(self, extracted: dict, context: TaskContext) -> dict:
+        parsed, skipped = [], 0
+        for raw in extracted.get("records", []):
+            mapped = self._parse_topup(raw, context.store_id)
+            if mapped:
+                parsed.append(mapped)
+            else:
+                skipped += 1
+        return {
+            "records": parsed,
+            "fetched": len(extracted.get("records", [])),
+            "skipped": skipped,
         }
 
-        try:
-            records, _ = self.api.get_paginated(
-                endpoint="/Topup/SettleList",
-                params=params,
-                page_size=self.config.get("api.page_size", 200),
-                data_path=("data", "settleList"),
-            )
+    def load(self, transformed: dict, context: TaskContext) -> dict:
+        loader = TopupLoader(self.db)
+        inserted, updated, loader_skipped = loader.upsert_topups(transformed["records"])
+        return {
+            "fetched": transformed["fetched"],
+            "inserted": inserted,
+            "updated": updated,
+            "skipped": transformed["skipped"] + loader_skipped,
+            "errors": 0,
+        }
 
-            parsed = []
-            for raw in records:
-                mapped = self._parse_topup(raw)
-                if mapped:
-                    parsed.append(mapped)
-
-            loader = TopupLoader(self.db)
-            inserted, updated, skipped = loader.upsert_topups(parsed)
-
-            self.db.commit()
-            counts = {
-                "fetched": len(records),
-                "inserted": inserted,
-                "updated": updated,
-                "skipped": skipped,
-                "errors": 0,
-            }
-            self.logger.info(f"TOPUPS 完成: {counts}")
-            return self._build_result("SUCCESS", counts)
-        except Exception:
-            self.db.rollback()
-            self.logger.error("TOPUPS 失败", exc_info=True)
-            raise
-
-    def _parse_topup(self, raw: dict) -> dict | None:
+    def _parse_topup(self, raw: dict, store_id: int) -> dict | None:
         node = raw.get("settleList") if isinstance(raw.get("settleList"), dict) else raw
         topup_id = TypeParser.parse_int(node.get("id"))
         if not topup_id:
-            self.logger.warning("跳过缺少 id 的充值结算: %s", raw)
+            self.logger.warning("跳过缺少充值ID的记录: %s", raw)
             return None
 
-        store_id = self.config.get("app.store_id")
         return {
             "store_id": store_id,
             "topup_id": topup_id,