DWD完成
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# -*- coding: utf-8 -*-
|
||||
"""ODS ingestion tasks."""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -62,11 +62,11 @@ class BaseOdsTask(BaseTask):
|
||||
|
||||
def execute(self) -> dict:
|
||||
spec = self.SPEC
|
||||
self.logger.info("开始执行 %s (ODS)", spec.code)
|
||||
self.logger.info("寮€濮嬫墽琛?%s (ODS)", spec.code)
|
||||
|
||||
store_id = TypeParser.parse_int(self.config.get("app.store_id"))
|
||||
if not store_id:
|
||||
raise ValueError("app.store_id 未配置,无法执行 ODS 任务")
|
||||
raise ValueError("app.store_id 鏈厤缃紝鏃犳硶鎵ц ODS 浠诲姟")
|
||||
|
||||
page_size = self.config.get("api.page_size", 200)
|
||||
params = self._build_params(spec, store_id)
|
||||
@@ -122,13 +122,13 @@ class BaseOdsTask(BaseTask):
|
||||
counts["fetched"] += len(page_records)
|
||||
|
||||
self.db.commit()
|
||||
self.logger.info("%s ODS 任务完成: %s", spec.code, counts)
|
||||
self.logger.info("%s ODS 浠诲姟瀹屾垚: %s", spec.code, counts)
|
||||
return self._build_result("SUCCESS", counts)
|
||||
|
||||
except Exception:
|
||||
self.db.rollback()
|
||||
counts["errors"] += 1
|
||||
self.logger.error("%s ODS 任务失败", spec.code, exc_info=True)
|
||||
self.logger.error("%s ODS 浠诲姟澶辫触", spec.code, exc_info=True)
|
||||
raise
|
||||
|
||||
def _build_params(self, spec: OdsTaskSpec, store_id: int) -> dict:
|
||||
@@ -201,7 +201,7 @@ class BaseOdsTask(BaseTask):
|
||||
value = self._extract_value(record, col_spec)
|
||||
if value is None and col_spec.required:
|
||||
self.logger.warning(
|
||||
"%s 缺少必填字段 %s,原始记录: %s",
|
||||
"%s 缂哄皯蹇呭~瀛楁 %s锛屽師濮嬭褰? %s",
|
||||
spec.code,
|
||||
col_spec.column,
|
||||
record,
|
||||
@@ -265,9 +265,38 @@ def _int_col(name: str, *sources: str, required: bool = False) -> ColumnSpec:
|
||||
)
|
||||
|
||||
|
||||
def _decimal_col(name: str, *sources: str) -> ColumnSpec:
|
||||
"""??????????????"""
|
||||
return ColumnSpec(
|
||||
column=name,
|
||||
sources=sources,
|
||||
transform=lambda v: TypeParser.parse_decimal(v, 2),
|
||||
)
|
||||
|
||||
|
||||
def _bool_col(name: str, *sources: str) -> ColumnSpec:
|
||||
"""??????????????0/1?true/false ???"""
|
||||
|
||||
def _to_bool(value):
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
s = str(value).strip().lower()
|
||||
if s in {"1", "true", "t", "yes", "y"}:
|
||||
return True
|
||||
if s in {"0", "false", "f", "no", "n"}:
|
||||
return False
|
||||
return bool(value)
|
||||
|
||||
return ColumnSpec(column=name, sources=sources, transform=_to_bool)
|
||||
|
||||
|
||||
|
||||
|
||||
ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
OdsTaskSpec(
|
||||
code="ODS_ASSISTANT_ACCOUNTS",
|
||||
code="ODS_ASSISTANT_ACCOUNT",
|
||||
class_name="OdsAssistantAccountsTask",
|
||||
table_name="billiards_ods.assistant_accounts_master",
|
||||
endpoint="/PersonnelManagement/SearchAssistantInfo",
|
||||
@@ -281,10 +310,10 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_fetched_at=False,
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
description="助教账号档案 ODS:SearchAssistantInfo -> assistantInfos 原始 JSON",
|
||||
description="鍔╂暀璐﹀彿妗f ODS锛歋earchAssistantInfo -> assistantInfos 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_ORDER_SETTLE",
|
||||
code="ODS_SETTLEMENT_RECORDS",
|
||||
class_name="OdsOrderSettleTask",
|
||||
table_name="billiards_ods.settlement_records",
|
||||
endpoint="/Site/GetAllOrderSettleList",
|
||||
@@ -299,7 +328,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="结账记录 ODS:GetAllOrderSettleList -> settleList 原始 JSON",
|
||||
description="缁撹处璁板綍 ODS锛欸etAllOrderSettleList -> settleList 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_TABLE_USE",
|
||||
@@ -317,7 +346,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="台费计费流水 ODS:GetSiteTableOrderDetails -> siteTableUseDetailsList 原始 JSON",
|
||||
description="鍙拌垂璁¤垂娴佹按 ODS锛欸etSiteTableOrderDetails -> siteTableUseDetailsList 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_ASSISTANT_LEDGER",
|
||||
@@ -334,7 +363,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_fetched_at=False,
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
description="助教服务流水 ODS:GetOrderAssistantDetails -> orderAssistantDetails 原始 JSON",
|
||||
description="鍔╂暀鏈嶅姟娴佹按 ODS锛欸etOrderAssistantDetails -> orderAssistantDetails 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_ASSISTANT_ABOLISH",
|
||||
@@ -351,10 +380,10 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_fetched_at=False,
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
description="助教废除记录 ODS:GetAbolitionAssistant -> abolitionAssistants 原始 JSON",
|
||||
description="鍔╂暀搴熼櫎璁板綍 ODS锛欸etAbolitionAssistant -> abolitionAssistants 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_GOODS_LEDGER",
|
||||
code="ODS_STORE_GOODS_SALES",
|
||||
class_name="OdsGoodsLedgerTask",
|
||||
table_name="billiards_ods.store_goods_sales_records",
|
||||
endpoint="/TenantGoods/GetGoodsSalesList",
|
||||
@@ -369,7 +398,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="门店商品销售流水 ODS:GetGoodsSalesList -> orderGoodsLedgers 原始 JSON",
|
||||
description="闂ㄥ簵鍟嗗搧閿€鍞祦姘?ODS锛欸etGoodsSalesList -> orderGoodsLedgers 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_PAYMENT",
|
||||
@@ -386,7 +415,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="支付流水 ODS:GetPayLogListPage 原始 JSON",
|
||||
description="鏀粯娴佹按 ODS锛欸etPayLogListPage 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_REFUND",
|
||||
@@ -403,10 +432,10 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="退款流水 ODS:GetRefundPayLogList 原始 JSON",
|
||||
description="閫€娆炬祦姘?ODS锛欸etRefundPayLogList 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_COUPON_VERIFY",
|
||||
code="ODS_PLATFORM_COUPON",
|
||||
class_name="OdsCouponVerifyTask",
|
||||
table_name="billiards_ods.platform_coupon_redemption_records",
|
||||
endpoint="/Promotion/GetOfflineCouponConsumePageList",
|
||||
@@ -420,7 +449,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="平台/团购券核销 ODS:GetOfflineCouponConsumePageList 原始 JSON",
|
||||
description="骞冲彴/鍥㈣喘鍒告牳閿€ ODS锛欸etOfflineCouponConsumePageList 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_MEMBER",
|
||||
@@ -438,7 +467,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="会员档案 ODS:GetTenantMemberList -> tenantMemberInfos 原始 JSON",
|
||||
description="浼氬憳妗f ODS锛欸etTenantMemberList -> tenantMemberInfos 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_MEMBER_CARD",
|
||||
@@ -456,7 +485,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="会员储值卡 ODS:GetTenantMemberCardList -> tenantMemberCards 原始 JSON",
|
||||
description="浼氬憳鍌ㄥ€煎崱 ODS锛欸etTenantMemberCardList -> tenantMemberCards 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_MEMBER_BALANCE",
|
||||
@@ -474,7 +503,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="会员余额变动 ODS:GetMemberCardBalanceChange -> tenantMemberCardLogs 原始 JSON",
|
||||
description="浼氬憳浣欓鍙樺姩 ODS锛欸etMemberCardBalanceChange -> tenantMemberCardLogs 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_RECHARGE_SETTLE",
|
||||
@@ -483,19 +512,83 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
endpoint="/Site/GetRechargeSettleList",
|
||||
data_path=("data",),
|
||||
list_key="settleList",
|
||||
pk_columns=(),
|
||||
pk_columns=(_int_col("recharge_order_id", "settleList.id", "id", required=True),),
|
||||
extra_columns=(
|
||||
_int_col("tenant_id", "settleList.tenantId", "tenantId"),
|
||||
_int_col("site_id", "settleList.siteId", "siteId", "siteProfile.id"),
|
||||
ColumnSpec("site_name_snapshot", sources=("siteProfile.shop_name", "settleList.siteName")),
|
||||
_int_col("member_id", "settleList.memberId", "memberId"),
|
||||
ColumnSpec("member_name_snapshot", sources=("settleList.memberName", "memberName")),
|
||||
ColumnSpec("member_phone_snapshot", sources=("settleList.memberPhone", "memberPhone")),
|
||||
_int_col("tenant_member_card_id", "settleList.tenantMemberCardId", "tenantMemberCardId"),
|
||||
ColumnSpec("member_card_type_name", sources=("settleList.memberCardTypeName", "memberCardTypeName")),
|
||||
_int_col("settle_relate_id", "settleList.settleRelateId", "settleRelateId"),
|
||||
_int_col("settle_type", "settleList.settleType", "settleType"),
|
||||
ColumnSpec("settle_name", sources=("settleList.settleName", "settleName")),
|
||||
_int_col("is_first", "settleList.isFirst", "isFirst"),
|
||||
_int_col("settle_status", "settleList.settleStatus", "settleStatus"),
|
||||
_decimal_col("pay_amount", "settleList.payAmount", "payAmount"),
|
||||
_decimal_col("refund_amount", "settleList.refundAmount", "refundAmount"),
|
||||
_decimal_col("point_amount", "settleList.pointAmount", "pointAmount"),
|
||||
_decimal_col("cash_amount", "settleList.cashAmount", "cashAmount"),
|
||||
_decimal_col("online_amount", "settleList.onlineAmount", "onlineAmount"),
|
||||
_decimal_col("balance_amount", "settleList.balanceAmount", "balanceAmount"),
|
||||
_decimal_col("card_amount", "settleList.cardAmount", "cardAmount"),
|
||||
_decimal_col("coupon_amount", "settleList.couponAmount", "couponAmount"),
|
||||
_decimal_col("recharge_card_amount", "settleList.rechargeCardAmount", "rechargeCardAmount"),
|
||||
_decimal_col("gift_card_amount", "settleList.giftCardAmount", "giftCardAmount"),
|
||||
_decimal_col("prepay_money", "settleList.prepayMoney", "prepayMoney"),
|
||||
_decimal_col("consume_money", "settleList.consumeMoney", "consumeMoney"),
|
||||
_decimal_col("goods_money", "settleList.goodsMoney", "goodsMoney"),
|
||||
_decimal_col("real_goods_money", "settleList.realGoodsMoney", "realGoodsMoney"),
|
||||
_decimal_col("table_charge_money", "settleList.tableChargeMoney", "tableChargeMoney"),
|
||||
_decimal_col("service_money", "settleList.serviceMoney", "serviceMoney"),
|
||||
_decimal_col("activity_discount", "settleList.activityDiscount", "activityDiscount"),
|
||||
_decimal_col("all_coupon_discount", "settleList.allCouponDiscount", "allCouponDiscount"),
|
||||
_decimal_col("goods_promotion_money", "settleList.goodsPromotionMoney", "goodsPromotionMoney"),
|
||||
_decimal_col("assistant_promotion_money", "settleList.assistantPromotionMoney", "assistantPromotionMoney"),
|
||||
_decimal_col("assistant_pd_money", "settleList.assistantPdMoney", "assistantPdMoney"),
|
||||
_decimal_col("assistant_cx_money", "settleList.assistantCxMoney", "assistantCxMoney"),
|
||||
_decimal_col("assistant_manual_discount", "settleList.assistantManualDiscount", "assistantManualDiscount"),
|
||||
_decimal_col("coupon_sale_amount", "settleList.couponSaleAmount", "couponSaleAmount"),
|
||||
_decimal_col("member_discount_amount", "settleList.memberDiscountAmount", "memberDiscountAmount"),
|
||||
_decimal_col("point_discount_price", "settleList.pointDiscountPrice", "pointDiscountPrice"),
|
||||
_decimal_col("point_discount_cost", "settleList.pointDiscountCost", "pointDiscountCost"),
|
||||
_decimal_col("adjust_amount", "settleList.adjustAmount", "adjustAmount"),
|
||||
_decimal_col("rounding_amount", "settleList.roundingAmount", "roundingAmount"),
|
||||
_int_col("payment_method", "settleList.paymentMethod", "paymentMethod"),
|
||||
_bool_col("can_be_revoked", "settleList.canBeRevoked", "canBeRevoked"),
|
||||
_bool_col("is_bind_member", "settleList.isBindMember", "isBindMember"),
|
||||
_bool_col("is_activity", "settleList.isActivity", "isActivity"),
|
||||
_bool_col("is_use_coupon", "settleList.isUseCoupon", "isUseCoupon"),
|
||||
_bool_col("is_use_discount", "settleList.isUseDiscount", "isUseDiscount"),
|
||||
_int_col("operator_id", "settleList.operatorId", "operatorId"),
|
||||
ColumnSpec("operator_name_snapshot", sources=("settleList.operatorName", "operatorName")),
|
||||
_int_col("salesman_user_id", "settleList.salesManUserId", "salesmanUserId", "salesManUserId"),
|
||||
ColumnSpec("salesman_name", sources=("settleList.salesManName", "salesmanName", "settleList.salesmanName")),
|
||||
ColumnSpec("order_remark", sources=("settleList.orderRemark", "orderRemark")),
|
||||
_int_col("table_id", "settleList.tableId", "tableId"),
|
||||
_int_col("serial_number", "settleList.serialNumber", "serialNumber"),
|
||||
_int_col("revoke_order_id", "settleList.revokeOrderId", "revokeOrderId"),
|
||||
ColumnSpec("revoke_order_name", sources=("settleList.revokeOrderName", "revokeOrderName")),
|
||||
ColumnSpec("revoke_time", sources=("settleList.revokeTime", "revokeTime")),
|
||||
ColumnSpec("create_time", sources=("settleList.createTime", "createTime")),
|
||||
ColumnSpec("pay_time", sources=("settleList.payTime", "payTime")),
|
||||
ColumnSpec("site_profile", sources=("siteProfile",)),
|
||||
),
|
||||
include_site_column=False,
|
||||
include_source_endpoint=False,
|
||||
include_source_endpoint=True,
|
||||
include_page_no=False,
|
||||
include_page_size=False,
|
||||
include_fetched_at=False,
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
include_fetched_at=True,
|
||||
include_record_index=False,
|
||||
conflict_columns_override=None,
|
||||
requires_window=False,
|
||||
description="会员充值结算 ODS:GetRechargeSettleList -> settleList 原始 JSON",
|
||||
description="?????? ODS?GetRechargeSettleList -> data.settleList ????",
|
||||
),
|
||||
|
||||
OdsTaskSpec(
|
||||
code="ODS_PACKAGE",
|
||||
code="ODS_GROUP_PACKAGE",
|
||||
class_name="OdsPackageTask",
|
||||
table_name="billiards_ods.group_buy_packages",
|
||||
endpoint="/PackageCoupon/QueryPackageCouponList",
|
||||
@@ -510,7 +603,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="团购套餐定义 ODS:QueryPackageCouponList -> packageCouponList 原始 JSON",
|
||||
description="鍥㈣喘濂楅瀹氫箟 ODS锛歈ueryPackageCouponList -> packageCouponList 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_GROUP_BUY_REDEMPTION",
|
||||
@@ -528,7 +621,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="团购套餐核销 ODS:GetSiteTableUseDetails -> siteTableUseDetailsList 原始 JSON",
|
||||
description="鍥㈣喘濂楅鏍搁攢 ODS锛欸etSiteTableUseDetails -> siteTableUseDetailsList 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_INVENTORY_STOCK",
|
||||
@@ -545,7 +638,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="库存汇总 ODS:GetGoodsStockReport 原始 JSON",
|
||||
description="搴撳瓨姹囨€?ODS锛欸etGoodsStockReport 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_INVENTORY_CHANGE",
|
||||
@@ -562,7 +655,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_fetched_at=False,
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
description="库存变化记录 ODS:QueryGoodsOutboundReceipt -> queryDeliveryRecordsList 原始 JSON",
|
||||
description="搴撳瓨鍙樺寲璁板綍 ODS锛歈ueryGoodsOutboundReceipt -> queryDeliveryRecordsList 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_TABLES",
|
||||
@@ -580,7 +673,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="台桌维表 ODS:GetSiteTables -> siteTables 原始 JSON",
|
||||
description="鍙版缁磋〃 ODS锛欸etSiteTables -> siteTables 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_GOODS_CATEGORY",
|
||||
@@ -598,7 +691,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="库存商品分类树 ODS:QueryPrimarySecondaryCategory -> goodsCategoryList 原始 JSON",
|
||||
description="搴撳瓨鍟嗗搧鍒嗙被鏍?ODS锛歈ueryPrimarySecondaryCategory -> goodsCategoryList 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_STORE_GOODS",
|
||||
@@ -616,10 +709,10 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="门店商品档案 ODS:GetGoodsInventoryList -> orderGoodsList 原始 JSON",
|
||||
description="闂ㄥ簵鍟嗗搧妗f ODS锛欸etGoodsInventoryList -> orderGoodsList 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_TABLE_DISCOUNT",
|
||||
code="ODS_TABLE_FEE_DISCOUNT",
|
||||
class_name="OdsTableDiscountTask",
|
||||
table_name="billiards_ods.table_fee_discount_records",
|
||||
endpoint="/Site/GetTaiFeeAdjustList",
|
||||
@@ -634,7 +727,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="台费折扣/调账 ODS:GetTaiFeeAdjustList -> taiFeeAdjustInfos 原始 JSON",
|
||||
description="鍙拌垂鎶樻墸/璋冭处 ODS锛欸etTaiFeeAdjustList -> taiFeeAdjustInfos 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_TENANT_GOODS",
|
||||
@@ -652,7 +745,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
include_record_index=True,
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
description="租户商品档案 ODS:QueryTenantGoods -> tenantGoodsList 原始 JSON",
|
||||
description="绉熸埛鍟嗗搧妗f ODS锛歈ueryTenantGoods -> tenantGoodsList 鍘熷 JSON",
|
||||
),
|
||||
OdsTaskSpec(
|
||||
code="ODS_SETTLEMENT_TICKET",
|
||||
@@ -671,7 +764,7 @@ ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
|
||||
conflict_columns_override=("source_file", "record_index"),
|
||||
requires_window=False,
|
||||
include_site_id=False,
|
||||
description="结账小票详情 ODS:GetOrderSettleTicketNew 原始 JSON",
|
||||
description="缁撹处灏忕エ璇︽儏 ODS锛欸etOrderSettleTicketNew 鍘熷 JSON",
|
||||
),
|
||||
)
|
||||
|
||||
@@ -725,7 +818,7 @@ class OdsSettlementTicketTask(BaseOdsTask):
|
||||
|
||||
if not candidates:
|
||||
self.logger.info(
|
||||
"%s: 窗口[%s ~ %s] 未发现需要抓取的小票",
|
||||
"%s: 绐楀彛[%s ~ %s] 鏈彂鐜伴渶瑕佹姄鍙栫殑灏忕エ",
|
||||
spec.code,
|
||||
context.window_start,
|
||||
context.window_end,
|
||||
@@ -755,7 +848,7 @@ class OdsSettlementTicketTask(BaseOdsTask):
|
||||
counts["updated"] += updated
|
||||
self.db.commit()
|
||||
self.logger.info(
|
||||
"%s: 小票抓取完成,候选=%s 插入=%s 更新=%s 跳过=%s",
|
||||
"%s: 灏忕エ鎶撳彇瀹屾垚锛屽€欓€?%s 鎻掑叆=%s 鏇存柊=%s 璺宠繃=%s",
|
||||
spec.code,
|
||||
len(candidates),
|
||||
inserted,
|
||||
@@ -767,7 +860,7 @@ class OdsSettlementTicketTask(BaseOdsTask):
|
||||
except Exception:
|
||||
counts["errors"] += 1
|
||||
self.db.rollback()
|
||||
self.logger.error("%s: 小票抓取失败", spec.code, exc_info=True)
|
||||
self.logger.error("%s: 灏忕エ鎶撳彇澶辫触", spec.code, exc_info=True)
|
||||
raise
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
@@ -782,7 +875,7 @@ class OdsSettlementTicketTask(BaseOdsTask):
|
||||
try:
|
||||
rows = self.db.query(sql)
|
||||
except Exception:
|
||||
self.logger.warning("查询已有小票失败,按空集处理", exc_info=True)
|
||||
self.logger.warning("鏌ヨ宸叉湁灏忕エ澶辫触锛屾寜绌洪泦澶勭悊", exc_info=True)
|
||||
return set()
|
||||
|
||||
return {
|
||||
@@ -819,7 +912,7 @@ class OdsSettlementTicketTask(BaseOdsTask):
|
||||
try:
|
||||
rows = self.db.query(sql, params)
|
||||
except Exception:
|
||||
self.logger.warning("读取支付流水以获取结算单ID失败,将尝试调用支付接口回退", exc_info=True)
|
||||
self.logger.warning("璇诲彇鏀粯娴佹按浠ヨ幏鍙栫粨绠楀崟ID澶辫触锛屽皢灏濊瘯璋冪敤鏀粯鎺ュ彛鍥為€€", exc_info=True)
|
||||
return set()
|
||||
|
||||
return {
|
||||
@@ -853,7 +946,7 @@ class OdsSettlementTicketTask(BaseOdsTask):
|
||||
if relate_id:
|
||||
candidate_ids.add(relate_id)
|
||||
except Exception:
|
||||
self.logger.warning("调用支付接口获取结算单ID失败,当前批次将跳过回退来源", exc_info=True)
|
||||
self.logger.warning("璋冪敤鏀粯鎺ュ彛鑾峰彇缁撶畻鍗旾D澶辫触锛屽綋鍓嶆壒娆″皢璺宠繃鍥為€€鏉ユ簮", exc_info=True)
|
||||
return candidate_ids
|
||||
|
||||
def _fetch_ticket_payload(self, order_settle_id: int):
|
||||
@@ -869,10 +962,10 @@ class OdsSettlementTicketTask(BaseOdsTask):
|
||||
payload = response
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"调用小票接口失败 orderSettleId=%s", order_settle_id, exc_info=True
|
||||
"璋冪敤灏忕エ鎺ュ彛澶辫触 orderSettleId=%s", order_settle_id, exc_info=True
|
||||
)
|
||||
if isinstance(payload, dict) and isinstance(payload.get("data"), list) and len(payload["data"]) == 1:
|
||||
# 本地桩/回放可能把响应包装成单元素 list,这里展开以贴近真实结构
|
||||
# 鏈湴妗?鍥炴斁鍙兘鎶婂搷搴斿寘瑁呮垚鍗曞厓绱?list锛岃繖閲屽睍寮€浠ヨ创杩戠湡瀹炵粨鏋?
|
||||
payload = payload["data"][0]
|
||||
return payload
|
||||
|
||||
@@ -899,27 +992,29 @@ def _build_task_class(spec: OdsTaskSpec) -> Type[BaseOdsTask]:
|
||||
|
||||
|
||||
ENABLED_ODS_CODES = {
|
||||
"ODS_ASSISTANT_ACCOUNTS",
|
||||
"ODS_ASSISTANT_ACCOUNT",
|
||||
"ODS_ASSISTANT_LEDGER",
|
||||
"ODS_ASSISTANT_ABOLISH",
|
||||
"ODS_INVENTORY_CHANGE",
|
||||
"ODS_INVENTORY_STOCK",
|
||||
"ODS_PACKAGE",
|
||||
"ODS_GROUP_PACKAGE",
|
||||
"ODS_GROUP_BUY_REDEMPTION",
|
||||
"ODS_MEMBER",
|
||||
"ODS_MEMBER_BALANCE",
|
||||
"ODS_MEMBER_CARD",
|
||||
"ODS_PAYMENT",
|
||||
"ODS_REFUND",
|
||||
"ODS_COUPON_VERIFY",
|
||||
"ODS_PLATFORM_COUPON",
|
||||
"ODS_RECHARGE_SETTLE",
|
||||
"ODS_TABLE_USE",
|
||||
"ODS_TABLES",
|
||||
"ODS_GOODS_CATEGORY",
|
||||
"ODS_STORE_GOODS",
|
||||
"ODS_TABLE_DISCOUNT",
|
||||
"ODS_TABLE_FEE_DISCOUNT",
|
||||
"ODS_STORE_GOODS_SALES",
|
||||
"ODS_TENANT_GOODS",
|
||||
"ODS_SETTLEMENT_TICKET",
|
||||
"ODS_ORDER_SETTLE",
|
||||
"ODS_SETTLEMENT_RECORDS",
|
||||
}
|
||||
|
||||
ODS_TASK_CLASSES: Dict[str, Type[BaseOdsTask]] = {
|
||||
@@ -931,3 +1026,4 @@ ODS_TASK_CLASSES: Dict[str, Type[BaseOdsTask]] = {
|
||||
ODS_TASK_CLASSES["ODS_SETTLEMENT_TICKET"] = OdsSettlementTicketTask
|
||||
|
||||
__all__ = ["ODS_TASK_CLASSES", "ODS_TASK_SPECS", "BaseOdsTask", "ENABLED_ODS_CODES"]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user