ODS 完成

etl_billiards/tasks/ods_tasks.py

@@ -38,10 +38,12 @@ class OdsTaskSpec:
     pk_columns: Tuple[ColumnSpec, ...] = ()
     extra_columns: Tuple[ColumnSpec, ...] = ()
     include_page_size: bool = False
-    include_page_no: bool = True
+    include_page_no: bool = False
     include_source_file: bool = True
     include_source_endpoint: bool = True
     requires_window: bool = True
+    time_fields: Tuple[str, str] | None = ("startTime", "endTime")
+    include_site_id: bool = True
     description: str = ""
     extra_params: Dict[str, Any] = field(default_factory=dict)
 
@@ -65,7 +67,7 @@ class BaseOdsTask(BaseTask):
         page_size = self.config.get("api.page_size", 200)
         params = self._build_params(spec, store_id)
         columns = self._resolve_columns(spec)
-        conflict_columns = ["store_id"] + [col.column for col in spec.pk_columns]
+        conflict_columns = ["site_id"] + [col.column for col in spec.pk_columns]
         loader = GenericODSLoader(
             self.db,
             spec.table_name,
@@ -117,16 +119,21 @@ class BaseOdsTask(BaseTask):
             raise
 
     def _build_params(self, spec: OdsTaskSpec, store_id: int) -> dict:
-        params: dict[str, Any] = {"storeId": store_id}
-        params.update(spec.extra_params)
-        if spec.requires_window:
+        base: dict[str, Any] = {}
+        if spec.include_site_id:
+            base["siteId"] = store_id
+        if spec.requires_window and spec.time_fields:
             window_start, window_end, _ = self._get_time_window()
-            params["startTime"] = TypeParser.format_timestamp(window_start, self.tz)
-            params["endTime"] = TypeParser.format_timestamp(window_end, self.tz)
+            start_key, end_key = spec.time_fields
+            base[start_key] = TypeParser.format_timestamp(window_start, self.tz)
+            base[end_key] = TypeParser.format_timestamp(window_end, self.tz)
+
+        params = self._merge_common_params(base)
+        params.update(spec.extra_params)
         return params
 
     def _resolve_columns(self, spec: OdsTaskSpec) -> List[str]:
-        columns: List[str] = ["store_id"]
+        columns: List[str] = ["site_id"]
         seen = set(columns)
         for col_spec in list(spec.pk_columns) + list(spec.extra_columns):
             if col_spec.column not in seen:
@@ -166,7 +173,7 @@ class BaseOdsTask(BaseTask):
         page_size_value: int | None,
         source_file: str | None,
     ) -> dict | None:
-        row: dict[str, Any] = {"store_id": store_id}
+        row: dict[str, Any] = {"site_id": store_id}
 
         for col_spec in spec.pk_columns + spec.extra_columns:
             value = self._extract_value(record, col_spec)
@@ -238,19 +245,33 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
         code="ODS_ORDER_SETTLE",
         class_name="OdsOrderSettleTask",
         table_name="billiards_ods.ods_order_settle",
-        endpoint="/order/list",
+        endpoint="/Site/GetAllOrderSettleList",
         data_path=("data",),
-        pk_columns=(_int_col("order_settle_id", "orderSettleId", "order_settle_id", "id", required=True),),
-        extra_columns=(_int_col("order_trade_no", "orderTradeNo", "order_trade_no"),),
-        include_page_size=True,
+        list_key="settleList",
+        pk_columns=(
+            _int_col(
+                "order_settle_id",
+                "orderSettleId",
+                "order_settle_id",
+                "settleList.id",
+                "id",
+                required=True,
+            ),
+        ),
+        extra_columns=(
+            _int_col("order_trade_no", "orderTradeNo", "order_trade_no", "settleList.orderTradeNo"),
+        ),
+        include_page_size=False,
+        time_fields=("rangeStartTime", "rangeEndTime"),
         description="订单/结算 ODS 原始记录",
     ),
     OdsTaskSpec(
         code="ODS_TABLE_USE",
         class_name="OdsTableUseTask",
-        table_name="billiards_ods.ods_table_use_detail",
-        endpoint="/Table/UseDetailList",
-        data_path=("data", "siteTableUseDetailsList"),
+        table_name="billiards_ods.ods_table_use_log",
+        endpoint="/Site/GetSiteTableOrderDetails",
+        data_path=("data",),
+        list_key="siteTableUseDetailsList",
         pk_columns=(_int_col("ledger_id", "id", required=True),),
         extra_columns=(
             _int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
@@ -261,9 +282,10 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
     OdsTaskSpec(
         code="ODS_ASSISTANT_LEDGER",
         class_name="OdsAssistantLedgerTask",
-        table_name="billiards_ods.ods_assistant_ledger",
-        endpoint="/Assistant/LedgerList",
-        data_path=("data", "orderAssistantDetails"),
+        table_name="billiards_ods.ods_assistant_service_log",
+        endpoint="/AssistantPerformance/GetOrderAssistantDetails",
+        data_path=("data",),
+        list_key="orderAssistantDetails",
         pk_columns=(_int_col("ledger_id", "id", required=True),),
         extra_columns=(
             _int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
@@ -274,18 +296,20 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
     OdsTaskSpec(
         code="ODS_ASSISTANT_ABOLISH",
         class_name="OdsAssistantAbolishTask",
-        table_name="billiards_ods.ods_assistant_abolish",
-        endpoint="/Assistant/AbolishList",
-        data_path=("data", "abolitionAssistants"),
+        table_name="billiards_ods.ods_assistant_cancel_log",
+        endpoint="/AssistantPerformance/GetAbolitionAssistant",
+        data_path=("data",),
+        list_key="abolitionAssistants",
         pk_columns=(_int_col("abolish_id", "id", required=True),),
         description="助教作废记录 ODS",
     ),
     OdsTaskSpec(
         code="ODS_GOODS_LEDGER",
         class_name="OdsGoodsLedgerTask",
-        table_name="billiards_ods.ods_goods_ledger",
-        endpoint="/Order/GoodsLedgerList",
-        data_path=("data", "orderGoodsLedgers"),
+        table_name="billiards_ods.ods_store_sale_item",
+        endpoint="/TenantGoods/GetGoodsSalesList",
+        data_path=("data",),
+        list_key="orderGoodsLedgers",
         pk_columns=(_int_col("order_goods_id", "orderGoodsId", "id", required=True),),
         extra_columns=(
             _int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
@@ -296,8 +320,8 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
     OdsTaskSpec(
         code="ODS_PAYMENT",
         class_name="OdsPaymentTask",
-        table_name="billiards_ods.ods_payment",
-        endpoint="/pay/records",
+        table_name="billiards_ods.ods_payment_record",
+        endpoint="/PayLog/GetPayLogListPage",
         data_path=("data",),
         pk_columns=(_int_col("pay_id", "payId", "id", required=True),),
         extra_columns=(
@@ -305,14 +329,15 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
             _int_col("relate_id", "relate_id", "relateId"),
         ),
         include_page_size=False,
+        time_fields=("StartPayTime", "EndPayTime"),
         description="支付流水 ODS",
     ),
     OdsTaskSpec(
         code="ODS_REFUND",
         class_name="OdsRefundTask",
-        table_name="billiards_ods.ods_refund",
-        endpoint="/Pay/RefundList",
-        data_path=(),
+        table_name="billiards_ods.ods_refund_record",
+        endpoint="/Order/GetRefundPayLogList",
+        data_path=("data",),
         pk_columns=(_int_col("refund_id", "id", required=True),),
         extra_columns=(
             ColumnSpec(column="relate_type", sources=("relate_type", "relateType")),
@@ -323,18 +348,19 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
     OdsTaskSpec(
         code="ODS_COUPON_VERIFY",
         class_name="OdsCouponVerifyTask",
-        table_name="billiards_ods.ods_coupon_verify",
-        endpoint="/Coupon/UsageList",
-        data_path=(),
+        table_name="billiards_ods.ods_platform_coupon_log",
+        endpoint="/Promotion/GetOfflineCouponConsumePageList",
+        data_path=("data",),
         pk_columns=(_int_col("coupon_id", "id", "couponId", required=True),),
         description="平台验券/团购流水 ODS",
     ),
     OdsTaskSpec(
         code="ODS_MEMBER",
         class_name="OdsMemberTask",
-        table_name="billiards_ods.ods_member",
+        table_name="billiards_ods.ods_member_profile",
         endpoint="/MemberProfile/GetTenantMemberList",
         data_path=("data",),
+        list_key="tenantMemberInfos",
         pk_columns=(_int_col("member_id", "memberId", required=True),),
         requires_window=False,
         description="会员档案 ODS",
@@ -343,8 +369,9 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
         code="ODS_MEMBER_CARD",
         class_name="OdsMemberCardTask",
         table_name="billiards_ods.ods_member_card",
-        endpoint="/MemberCard/List",
-        data_path=("data", "tenantMemberCards"),
+        endpoint="/MemberProfile/GetTenantMemberCardList",
+        data_path=("data",),
+        list_key="tenantMemberCards",
         pk_columns=(_int_col("card_id", "tenantMemberCardId", "cardId", required=True),),
         requires_window=False,
         description="会员卡/储值卡 ODS",
@@ -352,9 +379,10 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
     OdsTaskSpec(
         code="ODS_PACKAGE",
         class_name="OdsPackageTask",
-        table_name="billiards_ods.ods_package_coupon",
-        endpoint="/Package/List",
-        data_path=("data", "packageCouponList"),
+        table_name="billiards_ods.ods_group_package",
+        endpoint="/PackageCoupon/QueryPackageCouponList",
+        data_path=("data",),
+        list_key="packageCouponList",
         pk_columns=(_int_col("package_id", "id", "packageId", required=True),),
         requires_window=False,
         description="团购/套餐定义 ODS",
@@ -363,8 +391,8 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
         code="ODS_INVENTORY_STOCK",
         class_name="OdsInventoryStockTask",
         table_name="billiards_ods.ods_inventory_stock",
-        endpoint="/Inventory/StockSummary",
-        data_path=(),
+        endpoint="/TenantGoods/GetGoodsStockReport",
+        data_path=("data",),
         pk_columns=(
             _int_col("site_goods_id", "siteGoodsId", required=True),
             ColumnSpec(column="snapshot_key", default="default", required=True),
@@ -376,8 +404,9 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
         code="ODS_INVENTORY_CHANGE",
         class_name="OdsInventoryChangeTask",
         table_name="billiards_ods.ods_inventory_change",
-        endpoint="/Inventory/ChangeList",
-        data_path=("data", "queryDeliveryRecordsList"),
+        endpoint="/GoodsStockManage/QueryGoodsOutboundReceipt",
+        data_path=("data",),
+        list_key="queryDeliveryRecordsList",
         pk_columns=(_int_col("change_id", "siteGoodsStockId", "id", required=True),),
         description="库存变动 ODS",
     ),