root/feiqiu-ETL
Template
1
0
Fork 0
Code Pull Requests Packages Releases Wiki Activity

ODS 完成

Browse Source
This commit is contained in:
Neo
2025-11-30 07:19:05 +08:00
parent b9b050bb5d
commit a6ad343092
81 changed files with 15695 additions and 227180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
etl_billiards/tasks/base_dwd_task.py
View File

@@ -45,7 +45,7 @@ class BaseDwdTask(BaseTask):
"""
offset = 0
cols_str = ", ".join(columns)
while True:
sql = f"""
SELECT {cols_str}
@@ -54,17 +54,17 @@ class BaseDwdTask(BaseTask):
ORDER BY {time_col} ASC
LIMIT %s OFFSET %s
"""
rows = self.db.fetch_all(sql, (start_time, end_time, batch_size, offset))
rows = self.db.query(sql, (start_time, end_time, batch_size, offset))
if not rows:
break
yield [dict(row) for row in rows]
yield rows
if len(rows) < batch_size:
break
offset += batch_size
def parse_payload(self, row: Dict[str, Any]) -> Dict[str, Any]:

426
etl_billiards/tasks/manual_ingest_task.py
View File

@@ -16,29 +16,31 @@ class ManualIngestTask(BaseTask):
Used when upstream API is unavailable and we need to replay captured payloads.
"""
# 仅保留已确认的新表映射，其余表待逐一校准后再添加
FILE_MAPPING: list[tuple[tuple[str, ...], str]] = [
(("会员档案",), "billiards_ods.ods_member_profile"),
(("储值卡列表", "储值卡"), "billiards_ods.ods_member_card"),
(("充值记录",), "billiards_ods.ods_recharge_record"),
(("余额变动",), "billiards_ods.ods_balance_change"),
(("助教账号",), "billiards_ods.ods_assistant_account"),
(("助教流水",), "billiards_ods.ods_assistant_service_log"),
(("助教废除", "助教作废"), "billiards_ods.ods_assistant_cancel_log"),
(("台桌列表",), "billiards_ods.ods_table_info"),
(("台费流水",), "billiards_ods.ods_table_use_log"),
(("台费打折",), "billiards_ods.ods_table_fee_adjust"),
(("商品档案",), "billiards_ods.ods_store_product"),
(("门店商品销售", "销售记录"), "billiards_ods.ods_store_sale_item"),
(("团购套餐定义", "套餐定义"), "billiards_ods.ods_group_package"),
(("团购套餐使用", "套餐使用"), "billiards_ods.ods_group_package_log"),
(("平台验券", "验券记录"), "billiards_ods.ods_platform_coupon_log"),
(("库存汇总",), "billiards_ods.ods_inventory_stock"),
(("库存变化记录1",), "billiards_ods.ods_inventory_change"),
(("库存变化记录2", "分类配置"), "billiards_ods.ods_goods_category"),
(("结账记录",), "billiards_ods.ods_order_settle"),
(("小票详情", "小票明细", "票详"), "billiards_ods.ods_order_receipt_detail"),
(("支付记录",), "billiards_ods.ods_payment_record"),
(("退款记录",), "billiards_ods.ods_refund_record"),
(("助教账号",), "billiards_ods.assistant_accounts_master"),
(("助教流水",), "billiards_ods.assistant_service_records"),
(("助教废除", "助教作废"), "billiards_ods.assistant_cancellation_records"),
(("库存变化记录1",), "billiards_ods.goods_stock_movements"),
(("库存汇总", "库存汇总记录"), "billiards_ods.goods_stock_summary"),
(("团购套餐", "套餐定义"), "billiards_ods.group_buy_packages"),
(("团购核销", "套餐使用", "团购使用"), "billiards_ods.group_buy_redemption_records"),
(("余额变动", "余额变更"), "billiards_ods.member_balance_changes"),
(("会员档案", "会员列表"), "billiards_ods.member_profiles"),
(("储值卡", "会员储值卡"), "billiards_ods.member_stored_value_cards"),
(("支付记录", "支付流水"), "billiards_ods.payment_transactions"),
(("平台验券", "团购验券", "平台券核销"), "billiards_ods.platform_coupon_redemption_records"),
(("充值结算", "充值记录"), "billiards_ods.recharge_settlements"),
(("退款流水", "退款记录"), "billiards_ods.refund_transactions"),
(("结账记录", "结算记录"), "billiards_ods.settlement_records"),
(("小票详情", "结账小票"), "billiards_ods.settlement_ticket_details"),
(("台桌列表", "台桌维表"), "billiards_ods.site_tables_master"),
(("商品分类树", "库存分类"), "billiards_ods.stock_goods_category_tree"),
(("门店商品档案",), "billiards_ods.store_goods_master"),
(("门店商品销售", "商品销售流水"), "billiards_ods.store_goods_sales_records"),
(("台费折扣", "台费调账"), "billiards_ods.table_fee_discount_records"),
(("台费流水", "台费计费"), "billiards_ods.table_fee_transactions"),
(("租户商品档案", "商品档案"), "billiards_ods.tenant_goods_master"),
]
WRAPPER_META_KEYS = {"code", "message", "msg", "success", "error", "status"}
@@ -253,7 +255,7 @@ class ManualIngestTask(BaseTask):
if isinstance(data_node, dict):
return data_node.get("siteId") or data_node.get("site_id")
return None
return self.config.get("app.store_id")
def _extract_pk(self, item: dict, table: str):
if "ods_order_receipt_detail" in table:
@@ -390,9 +392,28 @@ class ManualIngestTask(BaseTask):
if isinstance(obj, dict) and obj.get(k) not in (None, ""):
return obj.get(k)
return None
def to_bool(value):
if value in (None, ""):
return None
try:
return bool(int(value))
except Exception:
if isinstance(value, bool):
return value
return None
def to_int(value):
if value in (None, ""):
return None
try:
return int(value)
except Exception:
return None
if "ods_member_profile" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["system_member_id"] = record.get("system_member_id")
row["register_site_id"] = record.get("register_site_id")
row["site_name"] = record.get("site_name")
row["member_name"] = pick(record, "name", "memberName")
row["nickname"] = record.get("nickname")
row["mobile"] = record.get("mobile")
@@ -401,28 +422,44 @@ class ManualIngestTask(BaseTask):
row["register_time"] = record.get("register_time") or record.get("registerTime")
row["member_type_id"] = pick(record, "cardTypeId", "member_type_id")
row["member_type_name"] = record.get("cardTypeName")
row["member_card_grade_code"] = record.get("member_card_grade_code")
row["status"] = pick(record, "status", "state")
row["user_status"] = record.get("user_status")
row["balance"] = record.get("balance")
row["points"] = record.get("points") or record.get("point")
row["growth_value"] = record.get("growth_value")
row["last_visit_time"] = record.get("lastVisitTime")
row["wechat_id"] = record.get("wechatId")
row["alipay_id"] = record.get("alipayId")
row["member_card_no"] = record.get("cardNo")
row["referrer_member_id"] = record.get("referrer_member_id")
row["remarks"] = record.get("remark")
if "ods_member_card" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["member_id"] = pick(record, "memberId", "member_id")
row["tenant_member_id"] = record.get("tenant_member_id")
row["card_type_id"] = record.get("cardTypeId")
row["card_type_name"] = record.get("cardTypeName")
row["card_no"] = record.get("card_no") or record.get("cardNo")
row["card_physics_type"] = record.get("card_physics_type")
row["card_balance"] = record.get("balance")
row["denomination"] = record.get("denomination")
row["discount_rate"] = record.get("discount") or record.get("discount_rate")
row["valid_start_date"] = record.get("validStart")
row["valid_end_date"] = record.get("validEnd")
row["table_discount"] = record.get("table_discount")
row["goods_discount"] = record.get("goods_discount")
row["assistant_discount"] = record.get("assistant_discount")
row["assistant_reward_discount"] = record.get("assistant_reward_discount")
row["valid_start_date"] = record.get("validStart") or record.get("start_time")
row["valid_end_date"] = record.get("validEnd") or record.get("end_time")
row["disable_start_date"] = record.get("disable_start_time")
row["disable_end_date"] = record.get("disable_end_time")
row["last_consume_time"] = record.get("lastConsumeTime")
row["status"] = record.get("status")
row["is_delete"] = to_bool(record.get("is_delete"))
row["activate_time"] = record.get("activateTime")
row["deactivate_time"] = record.get("cancelTime")
row["register_site_id"] = record.get("register_site_id")
row["issuer_id"] = record.get("issuerId")
row["issuer_name"] = record.get("issuerName")
@@ -443,17 +480,25 @@ class ManualIngestTask(BaseTask):
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["site_id"] = row.get("site_id") or pick(record, "siteId", "site_id")
row["member_id"] = pick(record, "memberId", "member_id")
row["change_amount"] = record.get("change_amount")
row["balance_before"] = record.get("before_balance")
row["balance_after"] = record.get("after_balance")
row["tenant_member_id"] = record.get("tenant_member_id")
row["tenant_member_card_id"] = record.get("tenant_member_card_id")
row["member_name"] = record.get("memberName")
row["member_mobile"] = record.get("memberMobile")
row["change_amount"] = record.get("change_amount") or record.get("account_data")
row["balance_before"] = record.get("before_balance") or record.get("before")
row["balance_after"] = record.get("after_balance") or record.get("after")
row["change_type"] = record.get("from_type") or record.get("type")
row["payment_method"] = record.get("payment_method")
row["refund_amount"] = record.get("refund_amount")
row["relate_id"] = record.get("relate_id")
row["pay_method"] = record.get("pay_type")
row["register_site_id"] = record.get("register_site_id")
row["register_site_name"] = record.get("registerSiteName")
row["pay_site_name"] = record.get("paySiteName")
row["remark"] = record.get("remark")
row["operator_id"] = record.get("operatorId")
row["operator_name"] = record.get("operatorName")
row["change_time"] = record.get("create_time") or record.get("changeTime")
row["is_deleted"] = record.get("is_delete") or record.get("is_deleted")
row["is_deleted"] = to_bool(record.get("is_delete") or record.get("is_deleted"))
row["source_file"] = row.get("source_file")
row["fetched_at"] = row.get("fetched_at")
@@ -461,25 +506,44 @@ class ManualIngestTask(BaseTask):
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["assistant_name"] = record.get("assistantName") or record.get("name")
row["mobile"] = record.get("mobile")
row["assistant_no"] = record.get("assistant_no") or record.get("assistantNo")
row["team_id"] = record.get("teamId")
row["team_name"] = record.get("teamName")
row["group_id"] = record.get("group_id")
row["group_name"] = record.get("group_name")
row["job_num"] = record.get("job_num")
row["entry_type"] = record.get("entry_type")
row["leave_status"] = record.get("leave_status")
row["assistant_status"] = record.get("assistant_status")
row["allow_cx"] = to_bool(record.get("allow_cx"))
row["status"] = record.get("status")
row["hired_date"] = record.get("hireDate")
row["left_date"] = record.get("leaveDate")
if "ods_assistant_service_log" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["assistant_id"] = record.get("assistantId")
row["service_type"] = record.get("serviceType")
row["assistant_id"] = record.get("assistantId") or record.get("site_assistant_id")
row["assistant_name"] = record.get("assistantName") or record.get("ledger_name")
row["service_type"] = record.get("serviceType") or record.get("order_assistant_type")
row["order_trade_no"] = record.get("orderTradeNo")
row["order_settle_id"] = record.get("orderSettleId")
row["start_time"] = record.get("startTime")
row["end_time"] = record.get("endTime")
row["duration_minutes"] = record.get("duration")
row["original_fee"] = record.get("originFee") or record.get("original_fee")
row["discount_amount"] = record.get("discountAmount")
row["final_fee"] = record.get("finalFee") or record.get("final_fee")
row["member_id"] = record.get("memberId")
row["start_time"] = record.get("start_use_time") or record.get("startTime") or record.get("ledger_start_time")
row["end_time"] = record.get("end_time") or record.get("last_use_time") or record.get("ledger_end_time")
row["duration_seconds"] = record.get("real_use_seconds") or record.get("duration")
row["original_fee"] = record.get("originFee") or record.get("original_fee") or record.get("ledger_amount")
row["member_discount_amount"] = record.get("member_discount_amount")
row["manual_discount_amount"] = record.get("manual_discount_amount")
row["coupon_discount_amount"] = record.get("coupon_deduct_money")
row["final_fee"] = record.get("finalFee") or record.get("final_fee") or record.get("service_money")
row["member_id"] = record.get("memberId") or record.get("tenant_member_id")
row["operator_id"] = record.get("operator_id")
row["salesman_id"] = record.get("salesman_user_id")
row["is_canceled"] = bool(record.get("is_trash"))
row["cancel_time"] = record.get("trash_time")
row["skill_grade"] = record.get("skill_grade")
row["service_grade"] = record.get("service_grade")
row["composite_grade"] = record.get("composite_grade")
row["overall_score"] = record.get("sum_grade")
row["status"] = record.get("status")
if "ods_assistant_cancel_log" in table:
@@ -487,6 +551,13 @@ class ManualIngestTask(BaseTask):
row["ledger_id"] = record.get("ledgerId")
row["assistant_id"] = record.get("assistantId")
row["order_trade_no"] = record.get("orderTradeNo")
row["table_id"] = record.get("tableId")
row["table_area_id"] = record.get("tableAreaId")
row["table_area_name"] = record.get("tableArea")
row["table_name"] = record.get("tableName")
row["assistant_on"] = record.get("assistantOn")
row["pd_charge_minutes"] = record.get("pdChargeMinutes")
row["assistant_abolish_amount"] = record.get("assistantAbolishAmount")
row["reason"] = record.get("reason")
row["cancel_time"] = record.get("cancel_time") or record.get("cancelTime")
row["operator_id"] = record.get("operatorId")
@@ -498,22 +569,64 @@ class ManualIngestTask(BaseTask):
row["table_name"] = record.get("tableName")
row["table_type"] = record.get("tableType")
row["area_name"] = record.get("areaName")
row["site_table_area_id"] = record.get("site_table_area_id") or record.get("siteTableAreaId")
row["tenant_table_area_id"] = record.get("tenant_table_area_id") or record.get("tenantTableAreaId")
row["table_price"] = record.get("table_price") or record.get("tablePrice")
row["table_status"] = record.get("table_status") or record.get("tableStatusName") or record.get("table_status")
row["audit_status"] = record.get("audit_status")
row["show_status"] = record.get("show_status")
row["light_status"] = record.get("light_status")
row["virtual_table"] = to_bool(record.get("virtual_table"))
row["is_rest_area"] = to_bool(record.get("is_rest_area"))
row["charge_free"] = to_bool(record.get("charge_free"))
row["table_cloth_use_time"] = record.get("table_cloth_use_time")
row["table_cloth_use_cycle"] = record.get("table_cloth_use_Cycle")
row["status"] = record.get("status")
row["created_time"] = record.get("createTime")
row["updated_time"] = record.get("updateTime")
if "ods_table_use_log" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["table_id"] = record.get("tableId")
row["table_id"] = record.get("tableId") or record.get("site_table_id")
row["table_name"] = record.get("tableName") or record.get("ledger_name")
row["order_trade_no"] = record.get("orderTradeNo")
row["order_settle_id"] = record.get("orderSettleId")
row["start_time"] = record.get("startTime")
row["end_time"] = record.get("endTime")
row["duration_minutes"] = record.get("duration")
row["original_table_fee"] = record.get("originFee") or record.get("original_table_fee")
row["discount_amount"] = record.get("discountAmount")
row["final_table_fee"] = record.get("finalFee") or record.get("final_table_fee")
row["start_time"] = record.get("start_use_time") or record.get("startTime") or record.get("ledger_start_time")
row["end_time"] = record.get("end_time") or record.get("ledger_end_time")
row["duration_seconds"] = record.get("real_table_use_seconds") or record.get("duration")
row["billing_unit_price"] = record.get("ledger_unit_price")
row["billing_count"] = record.get("ledger_count")
row["original_table_fee"] = record.get("originFee") or record.get("original_table_fee") or record.get("ledger_amount")
row["member_discount_amount"] = record.get("member_discount_amount")
row["coupon_discount_amount"] = record.get("coupon_promotion_amount")
row["manual_discount_amount"] = record.get("adjust_amount") or record.get("discountAmount")
row["service_fee"] = record.get("mgmt_fee") or record.get("service_money")
row["final_table_fee"] = (
record.get("finalFee")
or record.get("final_table_fee")
or record.get("real_table_charge_money")
)
row["member_id"] = record.get("memberId")
row["operator_id"] = record.get("operator_id")
row["salesman_id"] = record.get("salesman_user_id")
row["is_canceled"] = bool(record.get("is_delete"))
row["cancel_time"] = record.get("cancel_time")
row["site_table_area_id"] = record.get("site_table_area_id")
row["tenant_table_area_id"] = record.get("tenant_table_area_id")
row["site_table_area_name"] = record.get("site_table_area_name")
row["is_single_order"] = to_bool(record.get("is_single_order"))
row["used_card_amount"] = record.get("used_card_amount")
row["adjust_amount"] = record.get("adjust_amount")
row["coupon_promotion_amount"] = record.get("coupon_promotion_amount")
row["service_money"] = record.get("service_money")
row["mgmt_fee"] = record.get("mgmt_fee")
row["fee_total"] = record.get("fee_total")
row["last_use_time"] = record.get("last_use_time")
row["ledger_start_time"] = record.get("ledger_start_time")
row["ledger_end_time"] = record.get("ledger_end_time")
row["ledger_status"] = record.get("ledger_status")
row["start_use_time"] = record.get("start_use_time")
row["add_clock_seconds"] = record.get("add_clock_seconds")
row["status"] = record.get("status")
if "ods_table_fee_adjust" in table:
@@ -532,23 +645,45 @@ class ManualIngestTask(BaseTask):
row["goods_name"] = record.get("goodsName")
row["category_id"] = record.get("categoryId")
row["category_name"] = record.get("categoryName")
row["unit"] = record.get("unit")
row["sale_price"] = record.get("salePrice")
row["cost_price"] = record.get("costPrice")
row["sale_num"] = record.get("sale_num")
row["stock_a"] = record.get("stock_A")
row["stock"] = record.get("stock")
row["provisional_total_cost"] = record.get("provisional_total_cost")
row["total_purchase_cost"] = record.get("total_purchase_cost")
row["batch_stock_quantity"] = record.get("batch_stock_quantity")
row["goods_state"] = record.get("goods_state")
row["status"] = record.get("status")
if "ods_store_sale_item" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["order_trade_no"] = record.get("orderTradeNo")
row["order_settle_id"] = record.get("orderSettleId")
row["goods_id"] = record.get("goodsId")
row["goods_name"] = record.get("goodsName")
row["category_id"] = record.get("categoryId")
row["quantity"] = record.get("quantity")
row["original_amount"] = record.get("originalAmount")
row["discount_amount"] = record.get("discountAmount")
row["final_amount"] = record.get("finalAmount")
row["is_gift"] = record.get("isGift")
row["sale_time"] = record.get("saleTime")
row["order_goods_id"] = record.get("order_goods_id") or record.get("orderGoodsId")
row["site_goods_id"] = record.get("site_goods_id") or record.get("siteGoodsId")
row["goods_id"] = record.get("tenant_goods_id") or record.get("goodsId")
row["goods_name"] = record.get("goodsName") or record.get("ledger_name")
row["category_id"] = record.get("categoryId") or record.get("tenant_goods_category_id")
row["quantity"] = record.get("quantity") or record.get("ledger_count")
row["unit_price"] = record.get("unitPrice") or record.get("ledger_unit_price")
row["original_amount"] = record.get("originalAmount") or record.get("ledger_amount")
row["discount_amount"] = record.get("discountAmount") or record.get("discount_money")
row["final_amount"] = record.get("finalAmount") or record.get("real_goods_money")
row["discount_price"] = record.get("discount_price")
row["cost_money"] = record.get("cost_money")
row["coupon_deduct_amount"] = record.get("coupon_deduct_money")
row["member_discount_amount"] = record.get("member_discount_amount")
row["point_discount_money"] = record.get("point_discount_money")
row["point_discount_cost"] = record.get("point_discount_money_cost")
row["sales_type"] = record.get("sales_type")
row["goods_remark"] = record.get("goods_remark")
row["is_gift"] = to_bool(record.get("isGift"))
row["sale_time"] = record.get("saleTime") or record.get("create_time")
row["salesman_id"] = record.get("salesman_user_id")
row["operator_id"] = record.get("operator_id")
row["is_refunded"] = to_bool(record.get("is_delete")) or bool(record.get("returns_number"))
if "ods_group_package_log" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
@@ -561,16 +696,29 @@ class ManualIngestTask(BaseTask):
row["used_time"] = record.get("usedTime")
row["deduct_amount"] = record.get("deductAmount")
row["settle_price"] = record.get("settlePrice")
row["coupon_code"] = record.get("coupon_code") or record.get("couponCode")
if "ods_group_package" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["package_name"] = record.get("packageName")
row["table_area_id"] = record.get("table_area_id") or record.get("tableAreaId")
row["table_area_name"] = record.get("tableAreaName")
row["platform_code"] = record.get("platformCode")
row["status"] = record.get("status")
row["face_price"] = record.get("facePrice")
row["settle_price"] = record.get("settlePrice")
row["valid_from"] = record.get("validFrom")
row["valid_to"] = record.get("validTo")
row["selling_price"] = record.get("selling_price")
row["duration"] = record.get("duration")
row["valid_from"] = record.get("validFrom") or record.get("start_time")
row["valid_to"] = record.get("validTo") or record.get("end_time")
row["start_time"] = record.get("start_time")
row["end_time"] = record.get("end_time")
row["is_enabled"] = to_bool(record.get("is_enabled"))
row["is_delete"] = to_bool(record.get("is_delete"))
row["package_type"] = record.get("type")
row["usable_count"] = record.get("usable_count")
row["creator_name"] = record.get("creator_name")
row["tenant_table_area_id"] = record.get("tenant_table_area_id")
if "ods_platform_coupon_log" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
@@ -589,10 +737,15 @@ class ManualIngestTask(BaseTask):
row["order_trade_no"] = record.get("orderTradeNo")
row["order_settle_id"] = record.get("orderSettleId")
row["member_id"] = record.get("memberId")
row["pay_method_code"] = record.get("payMethodCode") or record.get("pay_type")
row["pay_method_code"] = record.get("payMethodCode") or record.get("pay_type") or record.get("payment_method")
row["pay_method_name"] = record.get("payMethodName")
row["pay_status"] = record.get("pay_status")
row["pay_amount"] = record.get("payAmount")
row["pay_time"] = record.get("payTime")
row["online_pay_channel"] = record.get("online_pay_channel")
row["transaction_id"] = record.get("transaction_id")
row["operator_id"] = record.get("operator_id")
row["remark"] = record.get("remark")
row["relate_type"] = record.get("relateType")
row["relate_id"] = record.get("relateId")
@@ -601,30 +754,74 @@ class ManualIngestTask(BaseTask):
row["order_trade_no"] = record.get("orderTradeNo")
row["order_settle_id"] = record.get("orderSettleId")
row["member_id"] = record.get("memberId")
row["pay_sn"] = record.get("pay_sn")
row["pay_amount"] = record.get("pay_amount")
row["pay_status"] = record.get("pay_status")
row["is_revoke"] = to_bool(record.get("is_revoke"))
row["is_delete"] = to_bool(record.get("is_delete"))
row["online_pay_channel"] = record.get("online_pay_channel")
row["pay_method_code"] = record.get("payMethodCode")
row["refund_amount"] = record.get("refundAmount")
row["refund_time"] = record.get("refundTime")
row["action_type"] = record.get("action_type")
row["pay_terminal"] = record.get("pay_terminal")
row["pay_config_id"] = record.get("pay_config_id")
row["cashier_point_id"] = record.get("cashier_point_id")
row["operator_id"] = record.get("operator_id")
row["member_card_id"] = record.get("member_card_id")
row["balance_frozen_amount"] = record.get("balance_frozen_amount")
row["card_frozen_amount"] = record.get("card_frozen_amount")
row["round_amount"] = record.get("round_amount")
row["online_pay_type"] = record.get("online_pay_type")
row["channel_payer_id"] = record.get("channel_payer_id")
row["channel_pay_no"] = record.get("channel_pay_no")
row["check_status"] = record.get("check_status")
row["channel_fee"] = record.get("channel_fee")
row["relate_type"] = record.get("relate_type")
row["relate_id"] = record.get("relate_id")
row["status"] = record.get("status")
row["reason"] = record.get("reason")
row["related_payment_id"] = record.get("related_payment_id") or record.get("payment_id")
if "ods_inventory_change" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["site_goods_id"] = record.get("siteGoodsId")
row["goods_id"] = record.get("goodsId")
row["change_amount"] = record.get("changeAmount")
row["before_stock"] = record.get("beforeStock")
row["after_stock"] = record.get("afterStock")
row["stock_type"] = record.get("stockType")
row["change_amount"] = record.get("changeAmount") or record.get("changeNum")
row["before_stock"] = record.get("beforeStock") or record.get("startNum")
row["after_stock"] = record.get("afterStock") or record.get("endNum")
row["change_amount_alt"] = record.get("changeNumA")
row["before_stock_alt"] = record.get("startNumA")
row["after_stock_alt"] = record.get("endNumA")
row["change_type"] = record.get("changeType")
row["relate_id"] = record.get("relateId")
row["unit"] = record.get("unit")
row["price"] = record.get("price")
row["goods_category_id"] = record.get("goodsCategoryId")
row["goods_second_category_id"] = record.get("goodsSecondCategoryId")
row["remark"] = record.get("remark")
row["operator_id"] = record.get("operatorId")
row["operator_name"] = record.get("operatorName")
row["change_time"] = record.get("changeTime")
row["change_time"] = record.get("changeTime") or record.get("createTime")
if "ods_inventory_stock" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["goods_id"] = record.get("goodsId")
row["goods_name"] = record.get("goodsName")
row["goods_unit"] = record.get("goodsUnit")
row["goods_category_id"] = record.get("goodsCategoryId")
row["goods_second_category_id"] = record.get("goodsCategorySecondId")
row["range_start_stock"] = record.get("rangeStartStock")
row["range_end_stock"] = record.get("rangeEndStock")
row["range_in"] = record.get("rangeIn")
row["range_out"] = record.get("rangeOut")
row["range_inventory"] = record.get("rangeInventory")
row["range_sale"] = record.get("rangeSale")
row["range_sale_money"] = record.get("rangeSaleMoney")
row["current_stock"] = record.get("currentStock")
row["cost_price"] = record.get("costPrice")
row["category_name"] = record.get("categoryName")
if "ods_goods_category" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
@@ -635,25 +832,63 @@ class ManualIngestTask(BaseTask):
row["remark"] = record.get("remark")
if "ods_order_receipt_detail" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["order_trade_no"] = record.get("orderTradeNo")
row["receipt_no"] = record.get("receiptNo")
row["receipt_time"] = record.get("receiptTime")
row["total_amount"] = record.get("totalAmount")
row["discount_amount"] = record.get("discountAmount")
row["final_amount"] = record.get("finalAmount")
row["member_id"] = record.get("memberId")
row["snapshot_raw"] = record.get("siteProfile") or record.get("site_profile")
detail = record
if isinstance(record.get("data"), dict):
detail = record["data"].get("data", record["data"])
row["tenant_id"] = pick(detail, "tenantId", "tenant_id")
row["order_trade_no"] = detail.get("orderTradeNo")
row["order_settle_number"] = detail.get("orderSettleNumber")
row["settle_type"] = to_int(detail.get("settleType"))
row["receipt_no"] = detail.get("receiptNo")
row["receipt_time"] = detail.get("receiptTime")
row["total_amount"] = detail.get("totalAmount") or detail.get("ledgerAmount")
row["discount_amount"] = detail.get("discountAmount")
row["final_amount"] = detail.get("finalAmount")
row["actual_payment"] = detail.get("actualPayment")
row["ledger_amount"] = detail.get("ledgerAmount")
row["member_offer_amount"] = detail.get("memberOfferAmount")
row["delivery_fee"] = detail.get("deliveryFee")
row["adjust_amount"] = detail.get("adjustAmount")
row["payment_method"] = detail.get("paymentMethod")
row["pay_time"] = detail.get("payTime")
row["member_id"] = detail.get("memberId")
row["order_remark"] = detail.get("orderRemark")
row["cashier_name"] = detail.get("cashierName")
row["ticket_remark"] = detail.get("ticketRemark")
row["ticket_custom_content"] = detail.get("ticketCustomContent")
row["voucher_money"] = detail.get("voucherMoney")
row["reward_name"] = detail.get("rewardName")
row["consume_money"] = detail.get("consumeMoney")
row["refund_amount"] = detail.get("refundAmount")
row["balance_amount"] = detail.get("balanceAmount")
row["coupon_amount"] = detail.get("couponAmount")
row["member_deduct_amount"] = detail.get("memberDeductAmount")
row["prepay_money"] = detail.get("prepayMoney")
row["delivery_address"] = detail.get("deliveryAddress")
row["snapshot_raw"] = detail.get("siteProfile") or record.get("site_profile")
row["member_snapshot"] = detail.get("memberProfile")
if isinstance(row.get("snapshot_raw"), (dict, list)):
row["snapshot_raw"] = json.dumps(row["snapshot_raw"], ensure_ascii=False)
if isinstance(row.get("member_snapshot"), (dict, list)):
row["member_snapshot"] = json.dumps(row["member_snapshot"], ensure_ascii=False)
if "ods_order_settle" in table:
settle = record.get("settleList") if isinstance(record.get("settleList"), dict) else record
settle_node = record.get("settleList") or record.get("settle")
if not settle_node and isinstance(record.get("data"), dict):
settle_node = record["data"].get("settleList") or record["data"].get("settle")
if isinstance(settle_node, list):
settle = settle_node[0] if settle_node else {}
else:
settle = settle_node or record
if isinstance(settle, dict):
row["site_id"] = row.get("site_id") or settle.get("siteId")
row["tenant_id"] = pick(settle, "tenantId", "tenant_id")
row["settle_relate_id"] = settle.get("settleRelateId")
row["settle_name"] = settle.get("settleName")
row["settle_type"] = settle.get("settleType")
row["settle_status"] = settle.get("settleStatus")
row["member_id"] = settle.get("memberId")
row["member_name"] = settle.get("memberName")
row["member_phone"] = settle.get("memberPhone")
row["table_id"] = settle.get("tableId")
row["consume_money"] = settle.get("consumeMoney")
@@ -663,6 +898,9 @@ class ManualIngestTask(BaseTask):
row["assistant_pd_money"] = settle.get("assistantPdMoney")
row["assistant_cx_money"] = settle.get("assistantCxMoney")
row["pay_amount"] = settle.get("payAmount")
row["cash_amount"] = settle.get("cashAmount")
row["online_amount"] = settle.get("onlineAmount")
row["point_amount"] = settle.get("pointAmount")
row["coupon_amount"] = settle.get("couponAmount")
row["card_amount"] = settle.get("cardAmount")
row["balance_amount"] = settle.get("balanceAmount")
@@ -670,9 +908,33 @@ class ManualIngestTask(BaseTask):
row["prepay_money"] = settle.get("prepayMoney")
row["adjust_amount"] = settle.get("adjustAmount")
row["rounding_amount"] = settle.get("roundingAmount")
row["member_discount_amount"] = settle.get("memberDiscountAmount")
row["coupon_sale_amount"] = settle.get("couponSaleAmount")
row["goods_promotion_money"] = settle.get("goodsPromotionMoney")
row["assistant_promotion_money"] = settle.get("assistantPromotionMoney")
row["point_discount_price"] = settle.get("pointDiscountPrice")
row["point_discount_cost"] = settle.get("pointDiscountCost")
row["real_goods_money"] = settle.get("realGoodsMoney")
row["assistant_manual_discount"] = settle.get("assistantManualDiscount")
row["all_coupon_discount"] = settle.get("allCouponDiscount")
row["is_use_coupon"] = to_bool(settle.get("isUseCoupon"))
row["is_use_discount"] = to_bool(settle.get("isUseDiscount"))
row["is_activity"] = to_bool(settle.get("isActivity"))
row["is_bind_member"] = to_bool(settle.get("isBindMember"))
row["is_first"] = to_bool(settle.get("isFirst"))
row["recharge_card_amount"] = settle.get("rechargeCardAmount")
row["gift_card_amount"] = settle.get("giftCardAmount")
row["payment_method"] = settle.get("paymentMethod")
row["create_time"] = settle.get("createTime")
row["pay_time"] = settle.get("payTime")
row["revoke_order_id"] = settle.get("revokeOrderId")
row["revoke_order_name"] = settle.get("revokeOrderName")
row["revoke_time"] = settle.get("revokeTime")
row["can_be_revoked"] = settle.get("canBeRevoked")
row["serial_number"] = settle.get("serialNumber")
row["sales_man_name"] = settle.get("salesManName")
row["sales_man_user_id"] = settle.get("salesManUserId")
row["order_remark"] = settle.get("orderRemark")
row["operator_id"] = settle.get("operatorId")
row["operator_name"] = settle.get("operatorName")
@@ -683,12 +945,32 @@ class ManualIngestTask(BaseTask):
row["goods_code"] = record.get("goodsCode")
row["category_id"] = record.get("categoryId")
row["category_name"] = record.get("categoryName")
row["goods_second_category_id"] = record.get("goods_second_category_id") or record.get("goodsSecondCategoryId")
row["unit"] = record.get("unit")
row["price"] = record.get("price")
row["cost_price"] = record.get("cost_price")
row["market_price"] = record.get("market_price")
row["goods_state"] = record.get("goods_state")
row["goods_cover"] = record.get("goods_cover")
row["goods_bar_code"] = record.get("goods_bar_code")
row["able_discount"] = record.get("able_discount")
row["is_delete"] = record.get("is_delete")
row["status"] = record.get("status")
if "ods_platform_coupon_log" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
row["platform_code"] = record.get("platformCode")
row["verify_code"] = record.get("verifyCode")
row["coupon_code"] = record.get("coupon_code")
row["coupon_channel"] = record.get("coupon_channel")
row["order_trade_no"] = record.get("orderTradeNo")
row["order_settle_id"] = record.get("orderSettleId")
row["member_id"] = record.get("memberId")
row["status"] = record.get("use_status") or record.get("status")
row["used_time"] = record.get("consume_time") or record.get("usedTime")
row["deduct_amount"] = record.get("deductAmount")
row["settle_price"] = record.get("sale_price") or record.get("settlePrice")
row["coupon_money"] = record.get("coupon_money")
if "ods_table_use_log" in table:
row["tenant_id"] = pick(record, "tenantId", "tenant_id")

6
etl_billiards/tasks/members_dwd_task.py
View File

@@ -7,7 +7,7 @@ import json
class MembersDwdTask(BaseDwdTask):
"""
DWD Task: Process Member Records from ODS to Dimension Table
Source: billiards_ods.ods_member
Source: billiards_ods.ods_member_profile
Target: billiards.dim_member
"""
@@ -29,8 +29,8 @@ class MembersDwdTask(BaseDwdTask):
# Iterate ODS Data
batches = self.iter_ods_rows(
table_name="billiards_ods.ods_member",
columns=["store_id", "member_id", "payload", "fetched_at"],
table_name="billiards_ods.ods_member_profile",
columns=["site_id", "member_id", "payload", "fetched_at"],
start_time=window_start,
end_time=window_end
)

672
etl_billiards/tasks/ods_tasks.py
View File

@@ -41,11 +41,15 @@ class OdsTaskSpec:
include_page_no: bool = False
include_source_file: bool = True
include_source_endpoint: bool = True
include_record_index: bool = False
include_site_column: bool = True
include_fetched_at: bool = True
requires_window: bool = True
time_fields: Tuple[str, str] | None = ("startTime", "endTime")
include_site_id: bool = True
description: str = ""
extra_params: Dict[str, Any] = field(default_factory=dict)
conflict_columns_override: Tuple[str, ...] | None = None
class BaseOdsTask(BaseTask):
@@ -67,7 +71,13 @@ class BaseOdsTask(BaseTask):
page_size = self.config.get("api.page_size", 200)
params = self._build_params(spec, store_id)
columns = self._resolve_columns(spec)
conflict_columns = ["site_id"] + [col.column for col in spec.pk_columns]
if spec.conflict_columns_override:
conflict_columns = list(spec.conflict_columns_override)
else:
conflict_columns = []
if spec.include_site_column:
conflict_columns.append("site_id")
conflict_columns += [col.column for col in spec.pk_columns]
loader = GenericODSLoader(
self.db,
spec.table_name,
@@ -79,6 +89,7 @@ class BaseOdsTask(BaseTask):
source_file = self._resolve_source_file_hint(spec)
try:
global_index = 0
for page_no, page_records, _, _ in self.api.iter_paginated(
endpoint=spec.endpoint,
params=params,
@@ -97,11 +108,13 @@ class BaseOdsTask(BaseTask):
if spec.include_page_size
else None,
source_file=source_file,
record_index=global_index if spec.include_record_index else None,
)
if row is None:
counts["skipped"] += 1
continue
rows.append(row)
global_index += 1
inserted, updated, _ = loader.upsert_rows(rows)
counts["inserted"] += inserted
@@ -133,13 +146,19 @@ class BaseOdsTask(BaseTask):
return params
def _resolve_columns(self, spec: OdsTaskSpec) -> List[str]:
columns: List[str] = ["site_id"]
columns: List[str] = []
if spec.include_site_column:
columns.append("site_id")
seen = set(columns)
for col_spec in list(spec.pk_columns) + list(spec.extra_columns):
if col_spec.column not in seen:
columns.append(col_spec.column)
seen.add(col_spec.column)
if spec.include_record_index and "record_index" not in seen:
columns.append("record_index")
seen.add("record_index")
if spec.include_page_no and "page_no" not in seen:
columns.append("page_no")
seen.add("page_no")
@@ -156,7 +175,7 @@ class BaseOdsTask(BaseTask):
columns.append("source_endpoint")
seen.add("source_endpoint")
if "fetched_at" not in seen:
if spec.include_fetched_at and "fetched_at" not in seen:
columns.append("fetched_at")
seen.add("fetched_at")
if "payload" not in seen:
@@ -172,8 +191,11 @@ class BaseOdsTask(BaseTask):
page_no: int | None,
page_size_value: int | None,
source_file: str | None,
record_index: int | None = None,
) -> dict | None:
row: dict[str, Any] = {"site_id": store_id}
row: dict[str, Any] = {}
if spec.include_site_column:
row["site_id"] = store_id
for col_spec in spec.pk_columns + spec.extra_columns:
value = self._extract_value(record, col_spec)
@@ -191,12 +213,15 @@ class BaseOdsTask(BaseTask):
row["page_no"] = page_no
if spec.include_page_size:
row["page_size"] = page_size_value
if spec.include_record_index:
row["record_index"] = record_index
if spec.include_source_file:
row["source_file"] = source_file
if spec.include_source_endpoint:
row["source_endpoint"] = spec.endpoint
row["fetched_at"] = datetime.now(self.tz)
if spec.include_fetched_at:
row["fetched_at"] = datetime.now(self.tz)
row["payload"] = record
return row
@@ -241,178 +266,629 @@ def _int_col(name: str, *sources: str, required: bool = False) -> ColumnSpec:
ODS_TASK_SPECS: Tuple[OdsTaskSpec, ...] = (
OdsTaskSpec(
code="ODS_ASSISTANT_ACCOUNTS",
class_name="OdsAssistantAccountsTask",
table_name="billiards_ods.assistant_accounts_master",
endpoint="/PersonnelManagement/SearchAssistantInfo",
data_path=("data",),
list_key="assistantInfos",
pk_columns=(_int_col("id", "id", required=True),),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
description="助教账号档案 ODS：SearchAssistantInfo -> assistantInfos 原始 JSON",
),
OdsTaskSpec(
code="ODS_ORDER_SETTLE",
class_name="OdsOrderSettleTask",
table_name="billiards_ods.ods_order_settle",
table_name="billiards_ods.settlement_records",
endpoint="/Site/GetAllOrderSettleList",
data_path=("data",),
list_key="settleList",
pk_columns=(
_int_col(
"order_settle_id",
"orderSettleId",
"order_settle_id",
"settleList.id",
"id",
required=True,
),
),
extra_columns=(
_int_col("order_trade_no", "orderTradeNo", "order_trade_no", "settleList.orderTradeNo"),
),
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
time_fields=("rangeStartTime", "rangeEndTime"),
description="订单/结算 ODS 原始记录",
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="结账记录 ODS：GetAllOrderSettleList -> settleList 原始 JSON",
),
OdsTaskSpec(
code="ODS_TABLE_USE",
class_name="OdsTableUseTask",
table_name="billiards_ods.ods_table_use_log",
table_name="billiards_ods.table_fee_transactions",
endpoint="/Site/GetSiteTableOrderDetails",
data_path=("data",),
list_key="siteTableUseDetailsList",
pk_columns=(_int_col("ledger_id", "id", required=True),),
extra_columns=(
_int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
_int_col("order_settle_id", "order_settle_id", "orderSettleId"),
),
description="台费/开台流水 ODS",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="台费计费流水 ODS：GetSiteTableOrderDetails -> siteTableUseDetailsList 原始 JSON",
),
OdsTaskSpec(
code="ODS_ASSISTANT_LEDGER",
class_name="OdsAssistantLedgerTask",
table_name="billiards_ods.ods_assistant_service_log",
table_name="billiards_ods.assistant_service_records",
endpoint="/AssistantPerformance/GetOrderAssistantDetails",
data_path=("data",),
list_key="orderAssistantDetails",
pk_columns=(_int_col("ledger_id", "id", required=True),),
extra_columns=(
_int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
_int_col("order_settle_id", "order_settle_id", "orderSettleId"),
),
description="助教流水 ODS",
pk_columns=(_int_col("id", "id", required=True),),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
description="助教服务流水 ODS：GetOrderAssistantDetails -> orderAssistantDetails 原始 JSON",
),
OdsTaskSpec(
code="ODS_ASSISTANT_ABOLISH",
class_name="OdsAssistantAbolishTask",
table_name="billiards_ods.ods_assistant_cancel_log",
table_name="billiards_ods.assistant_cancellation_records",
endpoint="/AssistantPerformance/GetAbolitionAssistant",
data_path=("data",),
list_key="abolitionAssistants",
pk_columns=(_int_col("abolish_id", "id", required=True),),
description="助教作废记录 ODS",
pk_columns=(_int_col("id", "id", required=True),),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
description="助教废除记录 ODS：GetAbolitionAssistant -> abolitionAssistants 原始 JSON",
),
OdsTaskSpec(
code="ODS_GOODS_LEDGER",
class_name="OdsGoodsLedgerTask",
table_name="billiards_ods.ods_store_sale_item",
table_name="billiards_ods.store_goods_sales_records",
endpoint="/TenantGoods/GetGoodsSalesList",
data_path=("data",),
list_key="orderGoodsLedgers",
pk_columns=(_int_col("order_goods_id", "orderGoodsId", "id", required=True),),
extra_columns=(
_int_col("order_trade_no", "order_trade_no", "orderTradeNo"),
_int_col("order_settle_id", "order_settle_id", "orderSettleId"),
),
description="商品销售流水 ODS",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="门店商品销售流水 ODS：GetGoodsSalesList -> orderGoodsLedgers 原始 JSON",
),
OdsTaskSpec(
code="ODS_PAYMENT",
class_name="OdsPaymentTask",
table_name="billiards_ods.ods_payment_record",
table_name="billiards_ods.payment_transactions",
endpoint="/PayLog/GetPayLogListPage",
data_path=("data",),
pk_columns=(_int_col("pay_id", "payId", "id", required=True),),
extra_columns=(
ColumnSpec(column="relate_type", sources=("relate_type", "relateType")),
_int_col("relate_id", "relate_id", "relateId"),
),
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
time_fields=("StartPayTime", "EndPayTime"),
description="支付流水 ODS",
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="支付流水 ODS：GetPayLogListPage 原始 JSON",
),
OdsTaskSpec(
code="ODS_REFUND",
class_name="OdsRefundTask",
table_name="billiards_ods.ods_refund_record",
table_name="billiards_ods.refund_transactions",
endpoint="/Order/GetRefundPayLogList",
data_path=("data",),
pk_columns=(_int_col("refund_id", "id", required=True),),
extra_columns=(
ColumnSpec(column="relate_type", sources=("relate_type", "relateType")),
_int_col("relate_id", "relate_id", "relateId"),
),
description="退款流水 ODS",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="退款流水 ODS：GetRefundPayLogList 原始 JSON",
),
OdsTaskSpec(
code="ODS_COUPON_VERIFY",
class_name="OdsCouponVerifyTask",
table_name="billiards_ods.ods_platform_coupon_log",
table_name="billiards_ods.platform_coupon_redemption_records",
endpoint="/Promotion/GetOfflineCouponConsumePageList",
data_path=("data",),
pk_columns=(_int_col("coupon_id", "id", "couponId", required=True),),
description="平台验券/团购流水 ODS",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="平台/团购券核销 ODS：GetOfflineCouponConsumePageList 原始 JSON",
),
OdsTaskSpec(
code="ODS_MEMBER",
class_name="OdsMemberTask",
table_name="billiards_ods.ods_member_profile",
table_name="billiards_ods.member_profiles",
endpoint="/MemberProfile/GetTenantMemberList",
data_path=("data",),
list_key="tenantMemberInfos",
pk_columns=(_int_col("member_id", "memberId", required=True),),
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="会员档案 ODS",
description="会员档案 ODS：GetTenantMemberList -> tenantMemberInfos 原始 JSON",
),
OdsTaskSpec(
code="ODS_MEMBER_CARD",
class_name="OdsMemberCardTask",
table_name="billiards_ods.ods_member_card",
table_name="billiards_ods.member_stored_value_cards",
endpoint="/MemberProfile/GetTenantMemberCardList",
data_path=("data",),
list_key="tenantMemberCards",
pk_columns=(_int_col("card_id", "tenantMemberCardId", "cardId", required=True),),
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="会员卡/储值卡 ODS",
description="会员储值卡 ODS：GetTenantMemberCardList -> tenantMemberCards 原始 JSON",
),
OdsTaskSpec(
code="ODS_MEMBER_BALANCE",
class_name="OdsMemberBalanceTask",
table_name="billiards_ods.member_balance_changes",
endpoint="/MemberProfile/GetMemberCardBalanceChange",
data_path=("data",),
list_key="tenantMemberCardLogs",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="会员余额变动 ODS：GetMemberCardBalanceChange -> tenantMemberCardLogs 原始 JSON",
),
OdsTaskSpec(
code="ODS_RECHARGE_SETTLE",
class_name="OdsRechargeSettleTask",
table_name="billiards_ods.recharge_settlements",
endpoint="/Site/GetRechargeSettleList",
data_path=("data",),
list_key="settleList",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="会员充值结算 ODS：GetRechargeSettleList -> settleList 原始 JSON",
),
OdsTaskSpec(
code="ODS_PACKAGE",
class_name="OdsPackageTask",
table_name="billiards_ods.ods_group_package",
table_name="billiards_ods.group_buy_packages",
endpoint="/PackageCoupon/QueryPackageCouponList",
data_path=("data",),
list_key="packageCouponList",
pk_columns=(_int_col("package_id", "id", "packageId", required=True),),
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="团购/套餐定义 ODS",
description="团购套餐定义 ODS：QueryPackageCouponList -> packageCouponList 原始 JSON",
),
OdsTaskSpec(
code="ODS_GROUP_BUY_REDEMPTION",
class_name="OdsGroupBuyRedemptionTask",
table_name="billiards_ods.group_buy_redemption_records",
endpoint="/Site/GetSiteTableUseDetails",
data_path=("data",),
list_key="siteTableUseDetailsList",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="团购套餐核销 ODS：GetSiteTableUseDetails -> siteTableUseDetailsList 原始 JSON",
),
OdsTaskSpec(
code="ODS_INVENTORY_STOCK",
class_name="OdsInventoryStockTask",
table_name="billiards_ods.ods_inventory_stock",
table_name="billiards_ods.goods_stock_summary",
endpoint="/TenantGoods/GetGoodsStockReport",
data_path=("data",),
pk_columns=(
_int_col("site_goods_id", "siteGoodsId", required=True),
ColumnSpec(column="snapshot_key", default="default", required=True),
),
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="库存汇总 ODS",
description="库存汇总 ODS：GetGoodsStockReport 原始 JSON",
),
OdsTaskSpec(
code="ODS_INVENTORY_CHANGE",
class_name="OdsInventoryChangeTask",
table_name="billiards_ods.ods_inventory_change",
table_name="billiards_ods.goods_stock_movements",
endpoint="/GoodsStockManage/QueryGoodsOutboundReceipt",
data_path=("data",),
list_key="queryDeliveryRecordsList",
pk_columns=(_int_col("change_id", "siteGoodsStockId", "id", required=True),),
description="库存变动 ODS",
pk_columns=(_int_col("sitegoodsstockid", "siteGoodsStockId", required=True),),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
description="库存变化记录 ODS：QueryGoodsOutboundReceipt -> queryDeliveryRecordsList 原始 JSON",
),
OdsTaskSpec(
code="ODS_TABLES",
class_name="OdsTablesTask",
table_name="billiards_ods.site_tables_master",
endpoint="/Table/GetSiteTables",
data_path=("data",),
list_key="siteTables",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="台桌维表 ODS：GetSiteTables -> siteTables 原始 JSON",
),
OdsTaskSpec(
code="ODS_GOODS_CATEGORY",
class_name="OdsGoodsCategoryTask",
table_name="billiards_ods.stock_goods_category_tree",
endpoint="/TenantGoodsCategory/QueryPrimarySecondaryCategory",
data_path=("data",),
list_key="goodsCategoryList",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="库存商品分类树 ODS：QueryPrimarySecondaryCategory -> goodsCategoryList 原始 JSON",
),
OdsTaskSpec(
code="ODS_STORE_GOODS",
class_name="OdsStoreGoodsTask",
table_name="billiards_ods.store_goods_master",
endpoint="/TenantGoods/GetGoodsInventoryList",
data_path=("data",),
list_key="orderGoodsList",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="门店商品档案 ODS：GetGoodsInventoryList -> orderGoodsList 原始 JSON",
),
OdsTaskSpec(
code="ODS_TABLE_DISCOUNT",
class_name="OdsTableDiscountTask",
table_name="billiards_ods.table_fee_discount_records",
endpoint="/Site/GetTaiFeeAdjustList",
data_path=("data",),
list_key="taiFeeAdjustInfos",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="台费折扣/调账 ODS：GetTaiFeeAdjustList -> taiFeeAdjustInfos 原始 JSON",
),
OdsTaskSpec(
code="ODS_TENANT_GOODS",
class_name="OdsTenantGoodsTask",
table_name="billiards_ods.tenant_goods_master",
endpoint="/TenantGoods/QueryTenantGoods",
data_path=("data",),
list_key="tenantGoodsList",
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=False,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
description="租户商品档案 ODS：QueryTenantGoods -> tenantGoodsList 原始 JSON",
),
OdsTaskSpec(
code="ODS_SETTLEMENT_TICKET",
class_name="OdsSettlementTicketTask",
table_name="billiards_ods.settlement_ticket_details",
endpoint="/Order/GetOrderSettleTicketNew",
data_path=(),
list_key=None,
pk_columns=(),
include_site_column=False,
include_source_endpoint=False,
include_page_no=False,
include_page_size=False,
include_fetched_at=True,
include_record_index=True,
conflict_columns_override=("source_file", "record_index"),
requires_window=False,
include_site_id=False,
description="结账小票详情 ODS：GetOrderSettleTicketNew 原始 JSON",
),
)
def _get_spec(code: str) -> OdsTaskSpec:
for spec in ODS_TASK_SPECS:
if spec.code == code:
return spec
raise KeyError(f"Spec not found for code {code}")
_SETTLEMENT_TICKET_SPEC = _get_spec("ODS_SETTLEMENT_TICKET")
class OdsSettlementTicketTask(BaseOdsTask):
"""Special handling: fetch ticket details per payment relate_id/orderSettleId."""
SPEC = _SETTLEMENT_TICKET_SPEC
def extract(self, context) -> dict:
"""Fetch ticket payloads only (used by fetch-only pipeline)."""
existing_ids = self._fetch_existing_ticket_ids()
candidates = self._collect_settlement_ids(
context.store_id or 0, existing_ids, context.window_start, context.window_end
)
candidates = [cid for cid in candidates if cid and cid not in existing_ids]
payloads, skipped = self._fetch_ticket_payloads(candidates)
return {"records": payloads, "skipped": skipped, "fetched": len(candidates)}
def execute(self, cursor_data: dict | None = None) -> dict:
spec = self.SPEC
context = self._build_context(cursor_data)
store_id = TypeParser.parse_int(self.config.get("app.store_id")) or 0
counts = {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}
loader = GenericODSLoader(
self.db,
spec.table_name,
self._resolve_columns(spec),
list(spec.conflict_columns_override or ("source_file", "record_index")),
)
source_file = self._resolve_source_file_hint(spec)
try:
existing_ids = self._fetch_existing_ticket_ids()
candidates = self._collect_settlement_ids(
store_id, existing_ids, context.window_start, context.window_end
)
candidates = [cid for cid in candidates if cid and cid not in existing_ids]
counts["fetched"] = len(candidates)
if not candidates:
self.logger.info(
"%s: 窗口[%s ~ %s] 未发现需要抓取的小票",
spec.code,
context.window_start,
context.window_end,
)
return self._build_result("SUCCESS", counts)
payloads, skipped = self._fetch_ticket_payloads(candidates)
counts["skipped"] += skipped
rows: list[dict] = []
for idx, payload in enumerate(payloads):
row = self._build_row(
spec=spec,
store_id=store_id,
record=payload,
page_no=None,
page_size_value=None,
source_file=source_file,
record_index=idx if spec.include_record_index else None,
)
if row is None:
counts["skipped"] += 1
continue
rows.append(row)
inserted, updated, _ = loader.upsert_rows(rows)
counts["inserted"] += inserted
counts["updated"] += updated
self.db.commit()
self.logger.info(
"%s: 小票抓取完成，候选=%s 插入=%s 更新=%s 跳过=%s",
spec.code,
len(candidates),
inserted,
updated,
counts["skipped"],
)
return self._build_result("SUCCESS", counts)
except Exception:
counts["errors"] += 1
self.db.rollback()
self.logger.error("%s: 小票抓取失败", spec.code, exc_info=True)
raise
# ------------------------------------------------------------------ helpers
def _fetch_existing_ticket_ids(self) -> set[int]:
sql = """
SELECT DISTINCT
CASE WHEN (payload ->> 'orderSettleId') ~ '^[0-9]+$'
THEN (payload ->> 'orderSettleId')::bigint
END AS order_settle_id
FROM billiards_ods.settlement_ticket_details
"""
try:
rows = self.db.query(sql)
except Exception:
self.logger.warning("查询已有小票失败，按空集处理", exc_info=True)
return set()
return {
TypeParser.parse_int(row.get("order_settle_id"))
for row in rows
if row.get("order_settle_id") is not None
}
def _collect_settlement_ids(
self, store_id: int, existing_ids: set[int], window_start, window_end
) -> list[int]:
ids = self._fetch_from_payment_table(store_id)
if not ids:
ids = self._fetch_from_payment_api(store_id, window_start, window_end)
return sorted(i for i in ids if i is not None and i not in existing_ids)
def _fetch_from_payment_table(self, store_id: int) -> set[int]:
sql = """
SELECT DISTINCT COALESCE(
CASE WHEN (payload ->> 'orderSettleId') ~ '^[0-9]+$'
THEN (payload ->> 'orderSettleId')::bigint END,
CASE WHEN (payload ->> 'relateId') ~ '^[0-9]+$'
THEN (payload ->> 'relateId')::bigint END
) AS order_settle_id
FROM billiards_ods.payment_transactions
WHERE (payload ->> 'orderSettleId') ~ '^[0-9]+$'
OR (payload ->> 'relateId') ~ '^[0-9]+$'
"""
params = None
if store_id:
sql += " AND COALESCE((payload ->> 'siteId')::bigint, %s) = %s"
params = (store_id, store_id)
try:
rows = self.db.query(sql, params)
except Exception:
self.logger.warning("读取支付流水以获取结算单ID失败，将尝试调用支付接口回退", exc_info=True)
return set()
return {
TypeParser.parse_int(row.get("order_settle_id"))
for row in rows
if row.get("order_settle_id") is not None
}
def _fetch_from_payment_api(self, store_id: int, window_start, window_end) -> set[int]:
params = self._merge_common_params(
{
"siteId": store_id,
"StartPayTime": TypeParser.format_timestamp(window_start, self.tz),
"EndPayTime": TypeParser.format_timestamp(window_end, self.tz),
}
)
candidate_ids: set[int] = set()
try:
for _, records, _, _ in self.api.iter_paginated(
endpoint="/PayLog/GetPayLogListPage",
params=params,
page_size=self.config.get("api.page_size", 200),
data_path=("data",),
):
for rec in records:
relate_id = TypeParser.parse_int(
(rec or {}).get("relateId")
or (rec or {}).get("orderSettleId")
or (rec or {}).get("order_settle_id")
)
if relate_id:
candidate_ids.add(relate_id)
except Exception:
self.logger.warning("调用支付接口获取结算单ID失败，当前批次将跳过回退来源", exc_info=True)
return candidate_ids
def _fetch_ticket_payload(self, order_settle_id: int):
payload = None
try:
for _, _, _, response in self.api.iter_paginated(
endpoint=self.SPEC.endpoint,
params={"orderSettleId": order_settle_id},
page_size=None,
data_path=self.SPEC.data_path,
list_key=self.SPEC.list_key,
):
payload = response
except Exception:
self.logger.warning(
"调用小票接口失败 orderSettleId=%s", order_settle_id, exc_info=True
)
if isinstance(payload, dict) and isinstance(payload.get("data"), list) and len(payload["data"]) == 1:
# 本地桩/回放可能把响应包装成单元素 list，这里展开以贴近真实结构
payload = payload["data"][0]
return payload
def _fetch_ticket_payloads(self, candidates: list[int]) -> tuple[list, int]:
"""Fetch ticket payloads for a set of orderSettleIds; returns (payloads, skipped)."""
payloads: list = []
skipped = 0
for order_settle_id in candidates:
payload = self._fetch_ticket_payload(order_settle_id)
if payload:
payloads.append(payload)
else:
skipped += 1
return payloads, skipped
def _build_task_class(spec: OdsTaskSpec) -> Type[BaseOdsTask]:
attrs = {
"SPEC": spec,
@@ -422,8 +898,36 @@ def _build_task_class(spec: OdsTaskSpec) -> Type[BaseOdsTask]:
return type(spec.class_name, (BaseOdsTask,), attrs)
ODS_TASK_CLASSES: Dict[str, Type[BaseOdsTask]] = {
spec.code: _build_task_class(spec) for spec in ODS_TASK_SPECS
ENABLED_ODS_CODES = {
"ODS_ASSISTANT_ACCOUNTS",
"ODS_ASSISTANT_LEDGER",
"ODS_ASSISTANT_ABOLISH",
"ODS_INVENTORY_CHANGE",
"ODS_INVENTORY_STOCK",
"ODS_PACKAGE",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_MEMBER",
"ODS_MEMBER_BALANCE",
"ODS_MEMBER_CARD",
"ODS_PAYMENT",
"ODS_REFUND",
"ODS_COUPON_VERIFY",
"ODS_RECHARGE_SETTLE",
"ODS_TABLES",
"ODS_GOODS_CATEGORY",
"ODS_STORE_GOODS",
"ODS_TABLE_DISCOUNT",
"ODS_TENANT_GOODS",
"ODS_SETTLEMENT_TICKET",
"ODS_ORDER_SETTLE",
}
__all__ = ["ODS_TASK_CLASSES", "ODS_TASK_SPECS", "BaseOdsTask"]
ODS_TASK_CLASSES: Dict[str, Type[BaseOdsTask]] = {
spec.code: _build_task_class(spec)
for spec in ODS_TASK_SPECS
if spec.code in ENABLED_ODS_CODES
}
# Override with specialized settlement ticket implementation
ODS_TASK_CLASSES["ODS_SETTLEMENT_TICKET"] = OdsSettlementTicketTask
__all__ = ["ODS_TASK_CLASSES", "ODS_TASK_SPECS", "BaseOdsTask", "ENABLED_ODS_CODES"]

38
etl_billiards/tasks/payments_dwd_task.py
View File

@@ -20,16 +20,17 @@ class PaymentsDwdTask(BaseDwdTask):
window_start, window_end, _ = self._get_time_window()
self.logger.info(f"Processing window: {window_start} to {window_end}")
loader = PaymentLoader(self.db)
loader = PaymentLoader(self.db, logger=self.logger)
store_id = self.config.get("app.store_id")
total_inserted = 0
total_errors = 0
total_updated = 0
total_skipped = 0
# Iterate ODS Data
batches = self.iter_ods_rows(
table_name="billiards_ods.ods_payment",
columns=["store_id", "pay_id", "payload", "fetched_at"],
table_name="billiards_ods.ods_payment_record",
columns=["site_id", "pay_id", "payload", "fetched_at"],
start_time=window_start,
end_time=window_end
)
@@ -49,20 +50,30 @@ class PaymentsDwdTask(BaseDwdTask):
parsed_rows.append(parsed)
if parsed_rows:
inserted, errors = loader.upsert_payments(parsed_rows, store_id)
inserted, updated, skipped = loader.upsert_payments(parsed_rows, store_id)
total_inserted += inserted
total_errors += errors
total_updated += updated
total_skipped += skipped
self.db.commit()
self.logger.info(f"Task {self.get_task_code()} completed. Inserted: {total_inserted}, Errors: {total_errors}")
self.logger.info(
"Task %s completed. inserted=%s updated=%s skipped=%s",
self.get_task_code(),
total_inserted,
total_updated,
total_skipped,
)
return {
"status": "success",
"inserted": total_inserted,
"errors": total_errors,
"window_start": window_start.isoformat(),
"window_end": window_end.isoformat()
"status": "SUCCESS",
"counts": {
"inserted": total_inserted,
"updated": total_updated,
"skipped": total_skipped,
},
"window_start": window_start,
"window_end": window_end,
}
def _parse_payment(self, raw: dict, store_id: int) -> dict:
@@ -89,6 +100,7 @@ class PaymentsDwdTask(BaseDwdTask):
return {
"store_id": store_id,
"pay_id": pay_id,
"order_id": TypeParser.parse_int(raw.get("orderId") or raw.get("order_id")),
"order_settle_id": order_settle_id,
"order_trade_no": order_trade_no,
"relate_type": relate_type,
@@ -114,9 +126,11 @@ class PaymentsDwdTask(BaseDwdTask):
or raw.get("discount_amount")
),
"payment_method": str(raw.get("paymentMethod") or raw.get("payment_method") or ""),
"pay_type": raw.get("payType") or raw.get("pay_type"),
"online_pay_channel": raw.get("onlinePayChannel") or raw.get("online_pay_channel"),
"pay_terminal": raw.get("payTerminal") or raw.get("pay_terminal"),
"pay_status": str(raw.get("payStatus") or raw.get("pay_status") or ""),
"remark": raw.get("remark"),
"raw_data": json.dumps(raw, ensure_ascii=False)
}
except Exception as e:

6
etl_billiards/tasks/ticket_dwd_task.py
View File

@@ -24,7 +24,7 @@ class TicketDwdTask(BaseDwdTask):
self.logger.info(f"Processing window: {window_start} to {window_end}")
# 2. Initialize Loader
loader = TicketLoader(self.db)
loader = TicketLoader(self.db, logger=self.logger)
store_id = self.config.get("app.store_id")
total_inserted = 0
@@ -33,8 +33,8 @@ class TicketDwdTask(BaseDwdTask):
# 3. Iterate ODS Data
# We query ods_ticket_detail based on fetched_at
batches = self.iter_ods_rows(
table_name="billiards_ods.ods_ticket_detail",
columns=["store_id", "order_settle_id", "payload", "fetched_at"],
table_name="billiards_ods.settlement_ticket_details",
columns=["payload", "fetched_at", "source_file", "record_index"],
start_time=window_start,
end_time=window_end
)