90 lines
3.0 KiB
Python
90 lines
3.0 KiB
Python
# -*- coding: utf-8 -*-
|
|
from .base_dwd_task import BaseDwdTask
|
|
from loaders.dimensions.member import MemberLoader
|
|
from models.parsers import TypeParser
|
|
import json
|
|
|
|
class MembersDwdTask(BaseDwdTask):
|
|
"""
|
|
DWD Task: Process Member Records from ODS to Dimension Table
|
|
Source: billiards_ods.ods_member_profile
|
|
Target: billiards.dim_member
|
|
"""
|
|
|
|
def get_task_code(self) -> str:
|
|
return "MEMBERS_DWD"
|
|
|
|
def execute(self) -> dict:
|
|
self.logger.info(f"Starting {self.get_task_code()} task")
|
|
|
|
window_start, window_end, _ = self._get_time_window()
|
|
self.logger.info(f"Processing window: {window_start} to {window_end}")
|
|
|
|
loader = MemberLoader(self.db)
|
|
store_id = self.config.get("app.store_id")
|
|
|
|
total_inserted = 0
|
|
total_updated = 0
|
|
total_errors = 0
|
|
|
|
# Iterate ODS Data
|
|
batches = self.iter_ods_rows(
|
|
table_name="billiards_ods.ods_member_profile",
|
|
columns=["site_id", "member_id", "payload", "fetched_at"],
|
|
start_time=window_start,
|
|
end_time=window_end
|
|
)
|
|
|
|
for batch in batches:
|
|
if not batch:
|
|
continue
|
|
|
|
parsed_rows = []
|
|
for row in batch:
|
|
payload = self.parse_payload(row)
|
|
if not payload:
|
|
continue
|
|
|
|
parsed = self._parse_member(payload, store_id)
|
|
if parsed:
|
|
parsed_rows.append(parsed)
|
|
|
|
if parsed_rows:
|
|
inserted, updated, skipped = loader.upsert_members(parsed_rows, store_id)
|
|
total_inserted += inserted
|
|
total_updated += updated
|
|
|
|
self.db.commit()
|
|
|
|
self.logger.info(f"Task {self.get_task_code()} completed. Inserted: {total_inserted}, Updated: {total_updated}")
|
|
|
|
return {
|
|
"status": "success",
|
|
"inserted": total_inserted,
|
|
"updated": total_updated,
|
|
"window_start": window_start.isoformat(),
|
|
"window_end": window_end.isoformat()
|
|
}
|
|
|
|
def _parse_member(self, raw: dict, store_id: int) -> dict:
|
|
"""Parse ODS payload into Dim structure"""
|
|
try:
|
|
# Handle both API structure (camelCase) and manual structure
|
|
member_id = raw.get("id") or raw.get("memberId")
|
|
if not member_id:
|
|
return None
|
|
|
|
return {
|
|
"store_id": store_id,
|
|
"member_id": member_id,
|
|
"member_name": raw.get("name") or raw.get("memberName"),
|
|
"phone": raw.get("phone") or raw.get("mobile"),
|
|
"balance": raw.get("balance", 0),
|
|
"status": str(raw.get("status", "NORMAL")),
|
|
"register_time": raw.get("createTime") or raw.get("registerTime"),
|
|
"raw_data": json.dumps(raw, ensure_ascii=False)
|
|
}
|
|
except Exception as e:
|
|
self.logger.warning(f"Error parsing member: {e}")
|
|
return None
|