401 lines
14 KiB
Python
401 lines
14 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""ODS ingestion tasks."""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from typing import Any, Callable, Dict, Iterable, List, Sequence, Tuple, Type
|
|
|
|
from loaders.ods import GenericODSLoader
|
|
from models.parsers import TypeParser
|
|
from .base_task import BaseTask
|
|
|
|
|
|
ColumnTransform = Callable[[Any], Any]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ColumnSpec:
|
|
"""Mapping between DB column and source JSON field."""
|
|
|
|
column: str
|
|
sources: Tuple[str, ...] = ()
|
|
required: bool = False
|
|
default: Any = None
|
|
transform: ColumnTransform | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class OdsTaskSpec:
|
|
"""Definition of a single ODS ingestion task."""
|
|
|
|
code: str
|
|
class_name: str
|
|
table_name: str
|
|
endpoint: str
|
|
data_path: Tuple[str, ...] = ("data",)
|
|
list_key: str | None = None
|
|
pk_columns: Tuple[ColumnSpec, ...] = ()
|
|
extra_columns: Tuple[ColumnSpec, ...] = ()
|
|
include_page_size: bool = False
|
|
include_page_no: bool = True
|
|
include_source_file: bool = True
|
|
include_source_endpoint: bool = True
|
|
requires_window: bool = True
|
|
description: str = ""
|
|
extra_params: Dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
class BaseOdsTask(BaseTask):
|
|
"""Shared functionality for ODS ingestion tasks."""
|
|
|
|
SPEC: OdsTaskSpec
|
|
|
|
def get_task_code(self) -> str:
|
|
return self.SPEC.code
|
|
|
|
def execute(self) -> dict:
|
|
spec = self.SPEC
|
|
self.logger.info("开始执行 %s (ODS)", spec.code)
|
|
|
|
store_id = TypeParser.parse_int(self.config.get("app.store_id"))
|
|
if not store_id:
|
|
raise ValueError("app.store_id 未配置，无法执行 ODS 任务")
|
|
|
|
page_size = self.config.get("api.page_size", 200)
|
|
params = self._build_params(spec, store_id)
|
|
columns = self._resolve_columns(spec)
|
|
conflict_columns = ["store_id"] + [col.column for col in spec.pk_columns]
|
|
loader = GenericODSLoader(
|
|
self.db,
|
|
spec.table_name,
|
|
columns,
|
|
conflict_columns,
|
|
)
|
|
|
|
counts = {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}
|
|
source_file = self._resolve_source_file_hint(spec)
|
|
|
|
try:
|
|
for page_no, page_records, _, _ in self.api.iter_paginated(
|
|
endpoint=spec.endpoint,
|
|
params=params,
|
|
page_size=page_size,
|
|
data_path=spec.data_path,
|
|
list_key=spec.list_key,
|
|
):
|
|
rows: List[dict] = []
|
|
for raw in page_records:
|
|
row = self._build_row(
|
|
spec=spec,
|
|
store_id=store_id,
|
|
record=raw,
|
|
page_no=page_no if spec.include_page_no else None,
|
|
page_size_value=len(page_records)
|
|
if spec.include_page_size
|
|
else None,
|
|
source_file=source_file,
|
|
)
|
|
if row is None:
|
|
counts["skipped"] += 1
|
|
continue
|
|
rows.append(row)
|
|
|
|
inserted, updated, _ = loader.upsert_rows(rows)
|
|
counts["inserted"] += inserted
|
|
counts["updated"] += updated
|
|
counts["fetched"] += len(page_records)
|
|
|
|
self.db.commit()
|
|
self.logger.info("%s ODS 任务完成: %s", spec.code, counts)
|
|
return self._build_result("SUCCESS", counts)
|
|
|
|
except Exception:
|
|
self.db.rollback()
|
|
counts["errors"] += 1
|
|
self.logger.error("%s ODS 任务失败", spec.code, exc_info=True)
|
|
raise
|
|
|
|
def _build_params(self, spec: OdsTaskSpec, store_id: int) -> dict:
|
|
params: dict[str, Any] = {"storeId": store_id}
|
|
params.update(spec.extra_params)
|
|
if spec.requires_window:
|
|
window_start, window_end, _ = self._get_time_window()
|
|
params["startTime"] = TypeParser.format_timestamp(window_start, self.tz)
|
|
params["endTime"] = TypeParser.format_timestamp(window_end, self.tz)
|
|
return params
|
|
|
|
def _resolve_columns(self, spec: OdsTaskSpec) -> List[str]:
|
|
columns: List[str] = ["store_id"]
|
|
seen = set(columns)
|
|
for col_spec in list(spec.pk_columns) + list(spec.extra_columns):
|
|
if col_spec.column not in seen:
|
|
columns.append(col_spec.column)
|
|
seen.add(col_spec.column)
|
|
|
|
if spec.include_page_no and "page_no" not in seen:
|
|
columns.append("page_no")
|
|
seen.add("page_no")
|
|
|
|
if spec.include_page_size and "page_size" not in seen:
|
|
columns.append("page_size")
|
|
seen.add("page_size")
|
|
|
|
if spec.include_source_file and "source_file" not in seen:
|
|
columns.append("source_file")
|
|
seen.add("source_file")
|
|
|
|
if spec.include_source_endpoint and "source_endpoint" not in seen:
|
|
columns.append("source_endpoint")
|
|
seen.add("source_endpoint")
|
|
|
|
if "fetched_at" not in seen:
|
|
columns.append("fetched_at")
|
|
seen.add("fetched_at")
|
|
if "payload" not in seen:
|
|
columns.append("payload")
|
|
|
|
return columns
|
|
|
|
def _build_row(
|
|
self,
|
|
spec: OdsTaskSpec,
|
|
store_id: int,
|
|
record: dict,
|
|
page_no: int | None,
|
|
page_size_value: int | None,
|
|
source_file: str | None,
|
|
) -> dict | None:
|
|
row: dict[str, Any] = {"store_id": store_id}
|
|
|
|
for col_spec in spec.pk_columns + spec.extra_columns:
|
|
value = self._extract_value(record, col_spec)
|
|
if value is None and col_spec.required:
|
|
self.logger.warning(
|
|
"%s 缺少必填字段 %s，原始记录: %s",
|
|
spec.code,
|
|
col_spec.column,
|
|
record,
|
|
)
|
|
return None
|
|
row[col_spec.column] = value
|
|
|
|
if spec.include_page_no:
|
|
row["page_no"] = page_no
|
|
if spec.include_page_size:
|
|
row["page_size"] = page_size_value
|
|
if spec.include_source_file:
|
|
row["source_file"] = source_file
|
|
if spec.include_source_endpoint:
|
|
row["source_endpoint"] = spec.endpoint
|
|
|
|
row["fetched_at"] = datetime.now(self.tz)
|
|
row["payload"] = record
|
|
return row
|
|
|
|
def _extract_value(self, record: dict, spec: ColumnSpec):
|
|
value = None
|
|
for key in spec.sources:
|
|
value = self._dig(record, key)
|
|
if value is not None:
|
|
break
|
|
if value is None and spec.default is not None:
|
|
value = spec.default
|
|
if value is not None and spec.transform:
|
|
value = spec.transform(value)
|
|
return value
|
|
|
|
@staticmethod
|
|
def _dig(record: Any, path: str | None):
|
|
if not path:
|
|
return None
|
|
current = record
|
|
for part in path.split("."):
|
|
if isinstance(current, dict):
|
|
current = current.get(part)
|
|
else:
|
|
return None
|
|
return current
|
|
|
|
def _resolve_source_file_hint(self, spec: OdsTaskSpec) -> str | None:
|
|
resolver = getattr(self.api, "get_source_hint", None)
|
|
if callable(resolver):
|
|
return resolver(spec.endpoint)
|
|
return None
|
|
|
|
|
|
def _int_col(name: str, *sources: str, required: bool = False) -> ColumnSpec:
|
|
return ColumnSpec(
|
|
column=name,
|
|
sources=sources,
|
|
required=required,
|
|
transform=TypeParser.parse_int,
|
|
)
|
|
|
|
|
|
ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
|
OdsTaskSpec(
|
|
code="ODS_ORDER_SETTLE",
|
|
class_name="OdsOrderSettleTask",
|
|
table_name="billiards_ods.ods_order_settle",
|
|
endpoint="/order/list",
|
|
data_path=("data",),
|
|
pk_columns=(_int_col("order_settle_id", "orderSettleId", "order_settle_id", "id", required=True),),
|
|
extra_columns=(_int_col("order_trade_no", "orderTradeNo", "order_trade_no"),),
|
|
include_page_size=True,
|
|
description="订单/结算 ODS 原始记录",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_TABLE_USE",
|
|
class_name="OdsTableUseTask",
|
|
table_name="billiards_ods.ods_table_use_detail",
|
|
endpoint="/Table/UseDetailList",
|
|
data_path=("data", "siteTableUseDetailsList"),
|
|
pk_columns=(_int_col("ledger_id", "id", required=True),),
|
|
extra_columns=(
|
|
_int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
|
|
_int_col("order_settle_id", "order_settle_id", "orderSettleId"),
|
|
),
|
|
description="台费/开台流水 ODS",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_ASSISTANT_LEDGER",
|
|
class_name="OdsAssistantLedgerTask",
|
|
table_name="billiards_ods.ods_assistant_ledger",
|
|
endpoint="/Assistant/LedgerList",
|
|
data_path=("data", "orderAssistantDetails"),
|
|
pk_columns=(_int_col("ledger_id", "id", required=True),),
|
|
extra_columns=(
|
|
_int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
|
|
_int_col("order_settle_id", "order_settle_id", "orderSettleId"),
|
|
),
|
|
description="助教流水 ODS",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_ASSISTANT_ABOLISH",
|
|
class_name="OdsAssistantAbolishTask",
|
|
table_name="billiards_ods.ods_assistant_abolish",
|
|
endpoint="/Assistant/AbolishList",
|
|
data_path=("data", "abolitionAssistants"),
|
|
pk_columns=(_int_col("abolish_id", "id", required=True),),
|
|
description="助教作废记录 ODS",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_GOODS_LEDGER",
|
|
class_name="OdsGoodsLedgerTask",
|
|
table_name="billiards_ods.ods_goods_ledger",
|
|
endpoint="/Order/GoodsLedgerList",
|
|
data_path=("data", "orderGoodsLedgers"),
|
|
pk_columns=(_int_col("order_goods_id", "orderGoodsId", "id", required=True),),
|
|
extra_columns=(
|
|
_int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
|
|
_int_col("order_settle_id", "order_settle_id", "orderSettleId"),
|
|
),
|
|
description="商品销售流水 ODS",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_PAYMENT",
|
|
class_name="OdsPaymentTask",
|
|
table_name="billiards_ods.ods_payment",
|
|
endpoint="/pay/records",
|
|
data_path=("data",),
|
|
pk_columns=(_int_col("pay_id", "payId", "id", required=True),),
|
|
extra_columns=(
|
|
ColumnSpec(column="relate_type", sources=("relate_type", "relateType")),
|
|
_int_col("relate_id", "relate_id", "relateId"),
|
|
),
|
|
include_page_size=False,
|
|
description="支付流水 ODS",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_REFUND",
|
|
class_name="OdsRefundTask",
|
|
table_name="billiards_ods.ods_refund",
|
|
endpoint="/Pay/RefundList",
|
|
data_path=(),
|
|
pk_columns=(_int_col("refund_id", "id", required=True),),
|
|
extra_columns=(
|
|
ColumnSpec(column="relate_type", sources=("relate_type", "relateType")),
|
|
_int_col("relate_id", "relate_id", "relateId"),
|
|
),
|
|
description="退款流水 ODS",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_COUPON_VERIFY",
|
|
class_name="OdsCouponVerifyTask",
|
|
table_name="billiards_ods.ods_coupon_verify",
|
|
endpoint="/Coupon/UsageList",
|
|
data_path=(),
|
|
pk_columns=(_int_col("coupon_id", "id", "couponId", required=True),),
|
|
description="平台验券/团购流水 ODS",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_MEMBER",
|
|
class_name="OdsMemberTask",
|
|
table_name="billiards_ods.ods_member",
|
|
endpoint="/MemberProfile/GetTenantMemberList",
|
|
data_path=("data",),
|
|
pk_columns=(_int_col("member_id", "memberId", required=True),),
|
|
requires_window=False,
|
|
description="会员档案 ODS",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_MEMBER_CARD",
|
|
class_name="OdsMemberCardTask",
|
|
table_name="billiards_ods.ods_member_card",
|
|
endpoint="/MemberCard/List",
|
|
data_path=("data", "tenantMemberCards"),
|
|
pk_columns=(_int_col("card_id", "tenantMemberCardId", "cardId", required=True),),
|
|
requires_window=False,
|
|
description="会员卡/储值卡 ODS",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_PACKAGE",
|
|
class_name="OdsPackageTask",
|
|
table_name="billiards_ods.ods_package_coupon",
|
|
endpoint="/Package/List",
|
|
data_path=("data", "packageCouponList"),
|
|
pk_columns=(_int_col("package_id", "id", "packageId", required=True),),
|
|
requires_window=False,
|
|
description="团购/套餐定义 ODS",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_INVENTORY_STOCK",
|
|
class_name="OdsInventoryStockTask",
|
|
table_name="billiards_ods.ods_inventory_stock",
|
|
endpoint="/Inventory/StockSummary",
|
|
data_path=(),
|
|
pk_columns=(
|
|
_int_col("site_goods_id", "siteGoodsId", required=True),
|
|
ColumnSpec(column="snapshot_key", default="default", required=True),
|
|
),
|
|
requires_window=False,
|
|
description="库存汇总 ODS",
|
|
),
|
|
OdsTaskSpec(
|
|
code="ODS_INVENTORY_CHANGE",
|
|
class_name="OdsInventoryChangeTask",
|
|
table_name="billiards_ods.ods_inventory_change",
|
|
endpoint="/Inventory/ChangeList",
|
|
data_path=("data", "queryDeliveryRecordsList"),
|
|
pk_columns=(_int_col("change_id", "siteGoodsStockId", "id", required=True),),
|
|
description="库存变动 ODS",
|
|
),
|
|
)
|
|
|
|
|
|
def _build_task_class(spec: OdsTaskSpec) -> Type[BaseOdsTask]:
|
|
attrs = {
|
|
"SPEC": spec,
|
|
"__doc__": spec.description or f"ODS ingestion task {spec.code}",
|
|
"__module__": __name__,
|
|
}
|
|
return type(spec.class_name, (BaseOdsTask,), attrs)
|
|
|
|
|
|
ODS_TASK_CLASSES: Dict[str, Type[BaseOdsTask]] = {
|
|
spec.code: _build_task_class(spec) for spec in ODS_TASK_SPECS
|
|
}
|
|
|
|
__all__ = ["ODS_TASK_CLASSES", "ODS_TASK_SPECS", "BaseOdsTask"]
|