阶段性更新

etl_billiards/tasks/members_dwd_task.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+from .base_dwd_task import BaseDwdTask
+from loaders.dimensions.member import MemberLoader
+from models.parsers import TypeParser
+import json
+
+class MembersDwdTask(BaseDwdTask):
+    """
+    DWD Task: Process Member Records from ODS to Dimension Table
+    Source: billiards_ods.ods_member
+    Target: billiards.dim_member
+    """
+
+    def get_task_code(self) -> str:
+        return "MEMBERS_DWD"
+
+    def execute(self) -> dict:
+        self.logger.info(f"Starting {self.get_task_code()} task")
+
+        window_start, window_end, _ = self._get_time_window()
+        self.logger.info(f"Processing window: {window_start} to {window_end}")
+
+        loader = MemberLoader(self.db)
+        store_id = self.config.get("app.store_id")
+
+        total_inserted = 0
+        total_updated = 0
+        total_errors = 0
+
+        # Iterate ODS Data
+        batches = self.iter_ods_rows(
+            table_name="billiards_ods.ods_member",
+            columns=["store_id", "member_id", "payload", "fetched_at"],
+            start_time=window_start,
+            end_time=window_end
+        )
+
+        for batch in batches:
+            if not batch:
+                continue
+            
+            parsed_rows = []
+            for row in batch:
+                payload = self.parse_payload(row)
+                if not payload:
+                    continue
+                
+                parsed = self._parse_member(payload, store_id)
+                if parsed:
+                    parsed_rows.append(parsed)
+            
+            if parsed_rows:
+                inserted, updated, skipped = loader.upsert_members(parsed_rows, store_id)
+                total_inserted += inserted
+                total_updated += updated
+
+        self.db.commit()
+        
+        self.logger.info(f"Task {self.get_task_code()} completed. Inserted: {total_inserted}, Updated: {total_updated}")
+        
+        return {
+            "status": "success",
+            "inserted": total_inserted,
+            "updated": total_updated,
+            "window_start": window_start.isoformat(),
+            "window_end": window_end.isoformat()
+        }
+
+    def _parse_member(self, raw: dict, store_id: int) -> dict:
+        """Parse ODS payload into Dim structure"""
+        try:
+            # Handle both API structure (camelCase) and manual structure
+            member_id = raw.get("id") or raw.get("memberId")
+            if not member_id:
+                return None
+
+            return {
+                "store_id": store_id,
+                "member_id": member_id,
+                "member_name": raw.get("name") or raw.get("memberName"),
+                "phone": raw.get("phone") or raw.get("mobile"),
+                "balance": raw.get("balance", 0),
+                "status": str(raw.get("status", "NORMAL")),
+                "register_time": raw.get("createTime") or raw.get("registerTime"),
+                "raw_data": json.dumps(raw, ensure_ascii=False)
+            }
+        except Exception as e:
+            self.logger.warning(f"Error parsing member: {e}")
+            return None