ODS 完成
This commit is contained in:
@@ -1,68 +1,70 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""支付记录ETL任务"""
|
||||
import json
|
||||
from .base_task import BaseTask
|
||||
|
||||
from .base_task import BaseTask, TaskContext
|
||||
from loaders.facts.payment import PaymentLoader
|
||||
from models.parsers import TypeParser
|
||||
|
||||
|
||||
class PaymentsTask(BaseTask):
|
||||
"""支付记录ETL任务"""
|
||||
|
||||
"""支付记录 E/T/L 任务"""
|
||||
|
||||
def get_task_code(self) -> str:
|
||||
return "PAYMENTS"
|
||||
|
||||
def execute(self) -> dict:
|
||||
"""执行支付记录ETL"""
|
||||
self.logger.info(f"开始执行 {self.get_task_code()} 任务")
|
||||
|
||||
window_start, window_end, window_minutes = self._get_time_window()
|
||||
|
||||
params = {
|
||||
"storeId": self.config.get("app.store_id"),
|
||||
"startTime": TypeParser.format_timestamp(window_start, self.tz),
|
||||
"endTime": TypeParser.format_timestamp(window_end, self.tz),
|
||||
}
|
||||
|
||||
try:
|
||||
records, pages_meta = self.api.get_paginated(
|
||||
endpoint="/pay/records",
|
||||
params=params,
|
||||
page_size=self.config.get("api.page_size", 200),
|
||||
data_path=("data",)
|
||||
)
|
||||
|
||||
parsed_records = []
|
||||
for rec in records:
|
||||
parsed = self._parse_payment(rec)
|
||||
if parsed:
|
||||
parsed_records.append(parsed)
|
||||
|
||||
loader = PaymentLoader(self.db)
|
||||
store_id = self.config.get("app.store_id")
|
||||
inserted, updated, skipped = loader.upsert_payments(parsed_records, store_id)
|
||||
|
||||
self.db.commit()
|
||||
|
||||
counts = {
|
||||
"fetched": len(records),
|
||||
"inserted": inserted,
|
||||
"updated": updated,
|
||||
"skipped": skipped,
|
||||
"errors": 0
|
||||
|
||||
# ------------------------------------------------------------------ E/T/L hooks
|
||||
def extract(self, context: TaskContext) -> dict:
|
||||
"""调用 API 抓取支付记录"""
|
||||
params = self._merge_common_params(
|
||||
{
|
||||
"siteId": context.store_id,
|
||||
"StartPayTime": TypeParser.format_timestamp(context.window_start, self.tz),
|
||||
"EndPayTime": TypeParser.format_timestamp(context.window_end, self.tz),
|
||||
}
|
||||
|
||||
self.logger.info(f"{self.get_task_code()} 完成: {counts}")
|
||||
return self._build_result("SUCCESS", counts)
|
||||
|
||||
except Exception as e:
|
||||
self.db.rollback()
|
||||
self.logger.error(f"{self.get_task_code()} 失败", exc_info=True)
|
||||
raise
|
||||
|
||||
def _parse_payment(self, raw: dict) -> dict:
|
||||
)
|
||||
records, pages_meta = self.api.get_paginated(
|
||||
endpoint="/PayLog/GetPayLogListPage",
|
||||
params=params,
|
||||
page_size=self.config.get("api.page_size", 200),
|
||||
data_path=("data",),
|
||||
)
|
||||
return {"records": records, "meta": pages_meta}
|
||||
|
||||
def transform(self, extracted: dict, context: TaskContext) -> dict:
|
||||
"""解析支付 JSON"""
|
||||
parsed, skipped = [], 0
|
||||
for rec in extracted.get("records", []):
|
||||
cleaned = self._parse_payment(rec, context.store_id)
|
||||
if cleaned:
|
||||
parsed.append(cleaned)
|
||||
else:
|
||||
skipped += 1
|
||||
return {
|
||||
"records": parsed,
|
||||
"fetched": len(extracted.get("records", [])),
|
||||
"skipped": skipped,
|
||||
}
|
||||
|
||||
def load(self, transformed: dict, context: TaskContext) -> dict:
|
||||
"""写入 fact_payment"""
|
||||
loader = PaymentLoader(self.db)
|
||||
inserted, updated, loader_skipped = loader.upsert_payments(
|
||||
transformed["records"], context.store_id
|
||||
)
|
||||
counts = {
|
||||
"fetched": transformed["fetched"],
|
||||
"inserted": inserted,
|
||||
"updated": updated,
|
||||
"skipped": transformed["skipped"] + loader_skipped,
|
||||
"errors": 0,
|
||||
}
|
||||
return counts
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
def _parse_payment(self, raw: dict, store_id: int) -> dict | None:
|
||||
"""解析支付记录"""
|
||||
try:
|
||||
store_id = self.config.get("app.store_id")
|
||||
return {
|
||||
"store_id": store_id,
|
||||
"pay_id": TypeParser.parse_int(raw.get("payId") or raw.get("id")),
|
||||
@@ -75,7 +77,9 @@ class PaymentsTask(BaseTask):
|
||||
),
|
||||
"relate_type": raw.get("relateType") or raw.get("relate_type"),
|
||||
"relate_id": TypeParser.parse_int(raw.get("relateId") or raw.get("relate_id")),
|
||||
"site_id": TypeParser.parse_int(raw.get("siteId") or raw.get("site_id") or store_id),
|
||||
"site_id": TypeParser.parse_int(
|
||||
raw.get("siteId") or raw.get("site_id") or store_id
|
||||
),
|
||||
"tenant_id": TypeParser.parse_int(raw.get("tenantId") or raw.get("tenant_id")),
|
||||
"pay_time": TypeParser.parse_timestamp(raw.get("payTime"), self.tz),
|
||||
"create_time": TypeParser.parse_timestamp(
|
||||
@@ -89,16 +93,19 @@ class PaymentsTask(BaseTask):
|
||||
or raw.get("fee_amount")
|
||||
),
|
||||
"discount_amount": TypeParser.parse_decimal(
|
||||
raw.get("discountAmount") or raw.get("couponAmount") or raw.get("discount_amount")
|
||||
raw.get("discountAmount")
|
||||
or raw.get("couponAmount")
|
||||
or raw.get("discount_amount")
|
||||
),
|
||||
"pay_type": raw.get("payType"),
|
||||
"payment_method": raw.get("paymentMethod") or raw.get("payment_method"),
|
||||
"online_pay_channel": raw.get("onlinePayChannel") or raw.get("online_pay_channel"),
|
||||
"online_pay_channel": raw.get("onlinePayChannel")
|
||||
or raw.get("online_pay_channel"),
|
||||
"pay_status": raw.get("payStatus"),
|
||||
"pay_terminal": raw.get("payTerminal") or raw.get("pay_terminal"),
|
||||
"remark": raw.get("remark"),
|
||||
"raw_data": json.dumps(raw, ensure_ascii=False),
|
||||
}
|
||||
except Exception as e:
|
||||
self.logger.warning(f"解析支付记录失败: {e}, 原始数据: {raw}")
|
||||
except Exception as exc:
|
||||
self.logger.warning("解析支付记录失败: %s, 原始数据: %s", exc, raw)
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user