合并

etl_billiards/tasks/payments_dwd_task.py

@@ -1,8 +1,9 @@
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
 from .base_dwd_task import BaseDwdTask
 from loaders.facts.payment import PaymentLoader
 from models.parsers import TypeParser
 import json
+from utils.windowing import build_window_segments
 
 class PaymentsDwdTask(BaseDwdTask):
     """
@@ -17,8 +18,20 @@ class PaymentsDwdTask(BaseDwdTask):
     def execute(self) -> dict:
         self.logger.info(f"Starting {self.get_task_code()} task")
 
-        window_start, window_end, _ = self._get_time_window()
-        self.logger.info(f"Processing window: {window_start} to {window_end}")
+        base_start, base_end, _ = self._get_time_window()
+        segments = build_window_segments(
+            self.config,
+            base_start,
+            base_end,
+            tz=self.tz,
+            override_only=True,
+        )
+        if not segments:
+            segments = [(base_start, base_end)]
+
+        total_segments = len(segments)
+        if total_segments > 1:
+            self.logger.info(f"{self.get_task_code()}: ????? {total_segments} ?")
 
         loader = PaymentLoader(self.db, logger=self.logger)
         store_id = self.config.get("app.store_id")
@@ -27,36 +40,42 @@ class PaymentsDwdTask(BaseDwdTask):
         total_updated = 0
         total_skipped = 0
 
-        # Iterate ODS Data
-        batches = self.iter_ods_rows(
-            table_name="billiards_ods.ods_payment_record",
-            columns=["site_id", "pay_id", "payload", "fetched_at"],
-            start_time=window_start,
-            end_time=window_end
-        )
+        for idx, (window_start, window_end) in enumerate(segments, start=1):
+            self.logger.info(
+                f"Processing window {idx}/{total_segments}: {window_start} to {window_end}"
+            )
+            batches = self.iter_ods_rows(
+                table_name="billiards_ods.payment_transactions",
+                columns=["site_id", "pay_id", "payload", "fetched_at"],
+                start_time=window_start,
+                end_time=window_end
+            )
 
-        for batch in batches:
-            if not batch:
-                continue
-            
-            parsed_rows = []
-            for row in batch:
-                payload = self.parse_payload(row)
-                if not payload:
+            for batch in batches:
+                if not batch:
                     continue
-                
-                parsed = self._parse_payment(payload, store_id)
-                if parsed:
-                    parsed_rows.append(parsed)
-            
-            if parsed_rows:
-                inserted, updated, skipped = loader.upsert_payments(parsed_rows, store_id)
-                total_inserted += inserted
-                total_updated += updated
-                total_skipped += skipped
 
-        self.db.commit()
-        
+                parsed_rows = []
+                for row in batch:
+                    payload = self.parse_payload(row)
+                    if not payload:
+                        continue
+
+                    parsed = self._parse_payment(payload, store_id)
+                    if parsed:
+                        parsed_rows.append(parsed)
+
+                if parsed_rows:
+                    inserted, updated, skipped = loader.upsert_payments(parsed_rows, store_id)
+                    total_inserted += inserted
+                    total_updated += updated
+                    total_skipped += skipped
+
+            self.db.commit()
+
+        overall_start = segments[0][0]
+        overall_end = segments[-1][1]
+
         self.logger.info(
             "Task %s completed. inserted=%s updated=%s skipped=%s",
             self.get_task_code(),
@@ -64,7 +83,7 @@ class PaymentsDwdTask(BaseDwdTask):
             total_updated,
             total_skipped,
         )
-        
+
         return {
             "status": "SUCCESS",
             "counts": {
@@ -72,8 +91,8 @@ class PaymentsDwdTask(BaseDwdTask):
                 "updated": total_updated,
                 "skipped": total_skipped,
             },
-            "window_start": window_start,
-            "window_end": window_end,
+            "window_start": overall_start,
+            "window_end": overall_end,
         }
 
     def _parse_payment(self, raw: dict, store_id: int) -> dict:
@@ -136,3 +155,4 @@ class PaymentsDwdTask(BaseDwdTask):
         except Exception as e:
             self.logger.warning(f"Error parsing payment: {e}")
             return None
+