ODS 完成
This commit is contained in:
@@ -1,176 +1,719 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
"""Manual ingestion task that replays archived JSON into ODS tables."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Iterable, Iterator
|
||||
|
||||
from .base_task import BaseTask
|
||||
from loaders.ods.generic import GenericODSLoader
|
||||
|
||||
|
||||
class ManualIngestTask(BaseTask):
|
||||
"""
|
||||
Task to ingest manually fetched JSON files from a directory into ODS tables.
|
||||
Load archived API responses (tests/source-data-doc) into billiards_ods.* tables.
|
||||
Used when upstream API is unavailable and we need to replay captured payloads.
|
||||
"""
|
||||
|
||||
FILE_MAPPING = {
|
||||
"小票详情": "billiards_ods.ods_ticket_detail",
|
||||
"结账记录": "billiards_ods.ods_order_settle",
|
||||
"支付记录": "billiards_ods.ods_payment",
|
||||
"助教流水": "billiards_ods.ods_assistant_ledger",
|
||||
"助教废除": "billiards_ods.ods_assistant_abolish",
|
||||
"商品档案": "billiards_ods.ods_goods_ledger", # Note: This might be dim_product source, but mapping to ledger for now if it's sales
|
||||
"库存变化": "billiards_ods.ods_inventory_change",
|
||||
"会员档案": "billiards_ods.ods_member",
|
||||
"充值记录": "billiards_ods.ods_member_card", # Approx
|
||||
"团购套餐": "billiards_ods.ods_package_coupon",
|
||||
"库存汇总": "billiards_ods.ods_inventory_stock"
|
||||
}
|
||||
FILE_MAPPING: list[tuple[tuple[str, ...], str]] = [
|
||||
(("会员档案",), "billiards_ods.ods_member_profile"),
|
||||
(("储值卡列表", "储值卡"), "billiards_ods.ods_member_card"),
|
||||
(("充值记录",), "billiards_ods.ods_recharge_record"),
|
||||
(("余额变动",), "billiards_ods.ods_balance_change"),
|
||||
(("助教账号",), "billiards_ods.ods_assistant_account"),
|
||||
(("助教流水",), "billiards_ods.ods_assistant_service_log"),
|
||||
(("助教废除", "助教作废"), "billiards_ods.ods_assistant_cancel_log"),
|
||||
(("台桌列表",), "billiards_ods.ods_table_info"),
|
||||
(("台费流水",), "billiards_ods.ods_table_use_log"),
|
||||
(("台费打折",), "billiards_ods.ods_table_fee_adjust"),
|
||||
(("商品档案",), "billiards_ods.ods_store_product"),
|
||||
(("门店商品销售", "销售记录"), "billiards_ods.ods_store_sale_item"),
|
||||
(("团购套餐定义", "套餐定义"), "billiards_ods.ods_group_package"),
|
||||
(("团购套餐使用", "套餐使用"), "billiards_ods.ods_group_package_log"),
|
||||
(("平台验券", "验券记录"), "billiards_ods.ods_platform_coupon_log"),
|
||||
(("库存汇总",), "billiards_ods.ods_inventory_stock"),
|
||||
(("库存变化记录1",), "billiards_ods.ods_inventory_change"),
|
||||
(("库存变化记录2", "分类配置"), "billiards_ods.ods_goods_category"),
|
||||
(("结账记录",), "billiards_ods.ods_order_settle"),
|
||||
(("小票详情", "小票明细", "票详"), "billiards_ods.ods_order_receipt_detail"),
|
||||
(("支付记录",), "billiards_ods.ods_payment_record"),
|
||||
(("退款记录",), "billiards_ods.ods_refund_record"),
|
||||
]
|
||||
WRAPPER_META_KEYS = {"code", "message", "msg", "success", "error", "status"}
|
||||
|
||||
def get_task_code(self) -> str:
|
||||
return "MANUAL_INGEST"
|
||||
|
||||
def execute(self) -> dict:
|
||||
self.logger.info("Starting Manual Ingest Task")
|
||||
|
||||
# Configurable directory, default to tests/testdata_json for now
|
||||
data_dir = self.config.get("manual.data_dir", r"c:\dev\LLTQ\ETL\feiqiu-ETL\etl_billiards\tests\testdata_json")
|
||||
|
||||
|
||||
data_dir = self.config.get(
|
||||
"manual.data_dir",
|
||||
r"c:\dev\LLTQ\ETL\feiqiu-ETL\etl_billiards\tests\testdata_json",
|
||||
)
|
||||
if not os.path.exists(data_dir):
|
||||
self.logger.error(f"Data directory not found: {data_dir}")
|
||||
self.logger.error("Data directory not found: %s", data_dir)
|
||||
return {"status": "error", "message": "Directory not found"}
|
||||
|
||||
total_files = 0
|
||||
total_rows = 0
|
||||
counts = {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}
|
||||
|
||||
for filename in os.listdir(data_dir):
|
||||
for filename in sorted(os.listdir(data_dir)):
|
||||
if not filename.endswith(".json"):
|
||||
continue
|
||||
|
||||
# Determine target table
|
||||
target_table = None
|
||||
for key, table in self.FILE_MAPPING.items():
|
||||
if key in filename:
|
||||
target_table = table
|
||||
break
|
||||
|
||||
if not target_table:
|
||||
self.logger.warning(f"No mapping found for file: {filename}, skipping.")
|
||||
filepath = os.path.join(data_dir, filename)
|
||||
try:
|
||||
with open(filepath, "r", encoding="utf-8") as fh:
|
||||
raw_entries = json.load(fh)
|
||||
except Exception:
|
||||
counts["errors"] += 1
|
||||
self.logger.exception("Failed to read %s", filename)
|
||||
continue
|
||||
|
||||
self.logger.info(f"Ingesting {filename} into {target_table}")
|
||||
|
||||
if not isinstance(raw_entries, list):
|
||||
raw_entries = [raw_entries]
|
||||
|
||||
records = self._normalize_records(raw_entries)
|
||||
if not records:
|
||||
counts["skipped"] += 1
|
||||
continue
|
||||
|
||||
target_table = self._match_by_filename(filename) or self._match_by_content(
|
||||
records, raw_entries
|
||||
)
|
||||
if not target_table:
|
||||
self.logger.warning("No mapping found for file: %s", filename)
|
||||
counts["skipped"] += 1
|
||||
continue
|
||||
|
||||
self.logger.info("Ingesting %s into %s", filename, target_table)
|
||||
|
||||
try:
|
||||
with open(os.path.join(data_dir, filename), 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
if not isinstance(data, list):
|
||||
data = [data]
|
||||
|
||||
# Prepare rows for GenericODSLoader
|
||||
# We need to adapt the data to what GenericODSLoader expects (or update it)
|
||||
# GenericODSLoader expects dicts. It handles normalization.
|
||||
# But we need to ensure the primary keys are present in the payload or extracted.
|
||||
# The GenericODSLoader might need configuration for PK extraction if it's not standard.
|
||||
# For now, let's assume the payload IS the row, and we wrap it.
|
||||
|
||||
# Actually, GenericODSLoader.upsert_rows expects the raw API result list.
|
||||
# It calls _normalize_row.
|
||||
# We need to make sure _normalize_row works for these files.
|
||||
# Most files have 'id' or similar.
|
||||
|
||||
# Let's instantiate a loader for this table
|
||||
# We need to know the PK for the table.
|
||||
# This is usually defined in ODS_TASK_CLASSES but here we are dynamic.
|
||||
# We might need a simpler loader or reuse GenericODSLoader with specific PK config.
|
||||
|
||||
# For simplicity, let's use a custom ingestion here that mimics GenericODSLoader but is file-aware.
|
||||
rows_to_insert = []
|
||||
for item in data:
|
||||
# Extract Store ID (usually in siteProfile or data root)
|
||||
store_id = self._extract_store_id(item) or self.config.get("app.store_id")
|
||||
|
||||
# Extract PK (id, orderSettleId, etc.)
|
||||
pk_val = self._extract_pk(item, target_table)
|
||||
|
||||
if not pk_val:
|
||||
# Try to find 'id' in the item
|
||||
pk_val = item.get("id")
|
||||
|
||||
if not pk_val:
|
||||
# Special case for Ticket Detail
|
||||
if "ods_ticket_detail" in target_table:
|
||||
pk_val = item.get("orderSettleId")
|
||||
|
||||
if not pk_val:
|
||||
rows = []
|
||||
for record in records:
|
||||
site_id = self._extract_store_id(record) or self.config.get(
|
||||
"app.store_id"
|
||||
)
|
||||
pk_value = self._extract_pk(record, target_table)
|
||||
pk_tuple = self._ensure_tuple(pk_value)
|
||||
if not all(value not in (None, "") for value in pk_tuple):
|
||||
continue
|
||||
|
||||
row = {
|
||||
"store_id": store_id,
|
||||
"payload": json.dumps(item, ensure_ascii=False),
|
||||
"site_id": site_id,
|
||||
"payload": json.dumps(record, ensure_ascii=False),
|
||||
"source_file": filename,
|
||||
"fetched_at": datetime.now()
|
||||
"fetched_at": datetime.now(),
|
||||
}
|
||||
|
||||
# Add specific PK column
|
||||
pk_col = self._get_pk_column(target_table)
|
||||
row[pk_col] = pk_val
|
||||
|
||||
rows_to_insert.append(row)
|
||||
for column, value in zip(
|
||||
self._get_conflict_columns(target_table), pk_tuple
|
||||
):
|
||||
row[column] = value
|
||||
self._enrich_row(row, record, target_table)
|
||||
rows.append(row)
|
||||
|
||||
if rows_to_insert:
|
||||
self._bulk_insert(target_table, rows_to_insert)
|
||||
total_rows += len(rows_to_insert)
|
||||
total_files += 1
|
||||
if rows:
|
||||
self._bulk_insert(target_table, rows)
|
||||
counts["inserted"] += len(rows)
|
||||
else:
|
||||
counts["skipped"] += 1
|
||||
counts["fetched"] += 1
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error processing {filename}: {e}", exc_info=True)
|
||||
except Exception:
|
||||
counts["errors"] += 1
|
||||
self.logger.exception("Error processing %s", filename)
|
||||
self.db.rollback()
|
||||
|
||||
return {"status": "success", "files_processed": total_files, "rows_inserted": total_rows}
|
||||
try:
|
||||
self.db.commit()
|
||||
except Exception:
|
||||
self.db.rollback()
|
||||
raise
|
||||
|
||||
def _extract_store_id(self, item):
|
||||
# Try common paths
|
||||
if "store_id" in item: return item["store_id"]
|
||||
if "siteProfile" in item and "id" in item["siteProfile"]: return item["siteProfile"]["id"]
|
||||
if "data" in item and "data" in item["data"] and "siteId" in item["data"]["data"]: return item["data"]["data"]["siteId"]
|
||||
return self._build_result("SUCCESS", counts)
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
def _match_by_filename(self, filename: str) -> str | None:
|
||||
for keywords, table in self.FILE_MAPPING:
|
||||
if any(keyword and keyword in filename for keyword in keywords):
|
||||
return table
|
||||
return None
|
||||
|
||||
def _extract_pk(self, item, table):
|
||||
# Helper to find PK based on table
|
||||
def _match_by_content(
|
||||
self, records: list[dict], raw_entries: list[dict]
|
||||
) -> str | None:
|
||||
"""
|
||||
Map content to PRD ODS tables.
|
||||
"""
|
||||
sample_record = records[0] if records else None
|
||||
wrapper = self._extract_sample(raw_entries)
|
||||
data_node = wrapper.get("data") if isinstance(wrapper, dict) else None
|
||||
data_keys = set(data_node.keys()) if isinstance(data_node, dict) else set()
|
||||
record_keys = set(sample_record.keys()) if isinstance(sample_record, dict) else set()
|
||||
|
||||
# Data node based hints
|
||||
if "tenantMemberInfos" in data_keys:
|
||||
return "billiards_ods.ods_member_profile"
|
||||
if "tenantMemberCards" in data_keys:
|
||||
return "billiards_ods.ods_member_card"
|
||||
if "queryDeliveryRecordsList" in data_keys:
|
||||
return "billiards_ods.ods_inventory_change"
|
||||
if "goodsStockA" in data_keys or "rangeStartStock" in data_keys:
|
||||
return "billiards_ods.ods_inventory_stock"
|
||||
if "goodsCategoryList" in data_keys:
|
||||
return "billiards_ods.ods_goods_category"
|
||||
if "orderAssistantDetails" in data_keys:
|
||||
return "billiards_ods.ods_assistant_service_log"
|
||||
if "abolitionAssistants" in data_keys:
|
||||
return "billiards_ods.ods_assistant_cancel_log"
|
||||
if "siteTableUseDetailsList" in data_keys:
|
||||
return "billiards_ods.ods_table_use_log"
|
||||
if "taiFeeAdjustInfos" in data_keys:
|
||||
return "billiards_ods.ods_table_fee_adjust"
|
||||
if "orderGoodsLedgers" in data_keys or "orderGoodsList" in data_keys:
|
||||
return "billiards_ods.ods_store_sale_item"
|
||||
if "tenantGoodsList" in data_keys:
|
||||
return "billiards_ods.ods_store_product"
|
||||
if "packageCouponList" in data_keys:
|
||||
return "billiards_ods.ods_group_package"
|
||||
if "settleList" in data_keys and "total" in data_keys:
|
||||
return "billiards_ods.ods_order_settle"
|
||||
|
||||
# Record key based hints
|
||||
if sample_record:
|
||||
if {"pay_amount", "pay_status"} <= record_keys or {"payAmount", "payStatus"} <= record_keys:
|
||||
return "billiards_ods.ods_payment_record"
|
||||
if "refundAmount" in record_keys or "refund_amount" in record_keys:
|
||||
return "billiards_ods.ods_refund_record"
|
||||
if "orderSettleId" in record_keys or "order_settle_id" in record_keys:
|
||||
return "billiards_ods.ods_order_receipt_detail"
|
||||
if "coupon_channel" in record_keys or "groupPackageId" in record_keys:
|
||||
return "billiards_ods.ods_platform_coupon_log"
|
||||
if "packageId" in record_keys or "package_id" in record_keys:
|
||||
return "billiards_ods.ods_group_package_log"
|
||||
if "memberCardId" in record_keys or "cardId" in record_keys:
|
||||
return "billiards_ods.ods_member_card"
|
||||
if "memberId" in record_keys:
|
||||
return "billiards_ods.ods_member_profile"
|
||||
if "siteGoodsId" in record_keys and "currentStock" in record_keys:
|
||||
return "billiards_ods.ods_inventory_stock"
|
||||
if "goodsId" in record_keys:
|
||||
return "billiards_ods.ods_product"
|
||||
|
||||
return None
|
||||
|
||||
def _extract_sample(self, payloads: Iterable[dict]) -> dict:
|
||||
for item in payloads:
|
||||
if isinstance(item, dict):
|
||||
return item
|
||||
return {}
|
||||
|
||||
def _normalize_records(self, payloads: list[dict]) -> list[dict]:
|
||||
records: list[dict] = []
|
||||
for payload in payloads:
|
||||
records.extend(self._unwrap_payload(payload))
|
||||
return records
|
||||
|
||||
def _unwrap_payload(self, payload) -> list[dict]:
|
||||
if isinstance(payload, dict):
|
||||
data_node = payload.get("data")
|
||||
extra_keys = set(payload.keys()) - {"data"} - self.WRAPPER_META_KEYS
|
||||
if isinstance(data_node, dict) and not extra_keys:
|
||||
flattened: list[dict] = []
|
||||
found_list = False
|
||||
for value in data_node.values():
|
||||
if isinstance(value, list):
|
||||
flattened.extend(value)
|
||||
found_list = True
|
||||
if found_list:
|
||||
return flattened
|
||||
return [data_node]
|
||||
return [payload]
|
||||
|
||||
if isinstance(payload, list):
|
||||
flattened: list[dict] = []
|
||||
for item in payload:
|
||||
flattened.extend(self._unwrap_payload(item))
|
||||
return flattened
|
||||
|
||||
return []
|
||||
|
||||
def _extract_store_id(self, item: dict):
|
||||
"""Extract site_id from record/siteProfile wrappers."""
|
||||
site_profile = item.get("siteProfile") or item.get("site_profile")
|
||||
if isinstance(site_profile, dict) and site_profile.get("id"):
|
||||
return site_profile["id"]
|
||||
|
||||
for key in ("site_id", "siteId", "register_site_id"):
|
||||
if item.get(key):
|
||||
return item[key]
|
||||
|
||||
data_node = item.get("data")
|
||||
if isinstance(data_node, dict):
|
||||
return data_node.get("siteId") or data_node.get("site_id")
|
||||
|
||||
return None
|
||||
|
||||
def _extract_pk(self, item: dict, table: str):
|
||||
if "ods_order_receipt_detail" in table:
|
||||
return item.get("orderSettleId") or item.get("order_settle_id") or item.get("id")
|
||||
if "ods_order_settle" in table:
|
||||
# Check for nested structure in some files
|
||||
if "settleList" in item and "settleList" in item["settleList"]:
|
||||
return item["settleList"]["settleList"].get("id")
|
||||
settle = item.get("settleList") or item.get("settle") or item
|
||||
if isinstance(settle, dict):
|
||||
return settle.get("id") or settle.get("settleId") or item.get("id")
|
||||
return item.get("id")
|
||||
|
||||
if "ods_payment_record" in table:
|
||||
return item.get("payId") or item.get("id")
|
||||
|
||||
if "ods_refund_record" in table:
|
||||
return item.get("refundId") or item.get("id")
|
||||
|
||||
if "ods_platform_coupon_log" in table:
|
||||
return item.get("couponId") or item.get("id")
|
||||
|
||||
if "ods_assistant_service_log" in table or "ods_table_use_log" in table:
|
||||
return item.get("ledgerId") or item.get("ledger_id") or item.get("id")
|
||||
|
||||
if "ods_assistant_cancel_log" in table:
|
||||
return item.get("cancel_id") or item.get("cancelId") or item.get("abolishId") or item.get("id")
|
||||
|
||||
if "ods_store_sale_item" in table:
|
||||
return (
|
||||
item.get("sale_item_id")
|
||||
or item.get("saleItemId")
|
||||
or item.get("orderGoodsId")
|
||||
or item.get("order_goods_id")
|
||||
or item.get("id")
|
||||
)
|
||||
|
||||
if "ods_inventory_change" in table:
|
||||
return item.get("siteGoodsStockId") or item.get("id")
|
||||
|
||||
if "ods_inventory_stock" in table:
|
||||
return (
|
||||
item.get("siteGoodsId")
|
||||
or item.get("id"),
|
||||
item.get("snapshotKey") or item.get("snapshot_key") or "default",
|
||||
)
|
||||
|
||||
if "ods_member_card" in table:
|
||||
return item.get("cardId") or item.get("memberCardId") or item.get("id")
|
||||
|
||||
if "ods_member_profile" in table:
|
||||
return item.get("memberId") or item.get("id")
|
||||
|
||||
if "ods_group_package_log" in table:
|
||||
return item.get("usage_id") or item.get("usageId") or item.get("couponId") or item.get("id")
|
||||
|
||||
if "ods_group_package" in table:
|
||||
return item.get("package_id") or item.get("packageId") or item.get("groupPackageId") or item.get("id")
|
||||
|
||||
if "ods_goods_category" in table:
|
||||
return item.get("category_id") or item.get("categoryId") or item.get("id")
|
||||
|
||||
if "ods_table_fee_adjust" in table:
|
||||
return item.get("adjust_id") or item.get("adjustId") or item.get("id")
|
||||
|
||||
if "ods_table_info" in table:
|
||||
return item.get("table_id") or item.get("tableId") or item.get("id")
|
||||
|
||||
if "ods_assistant_account" in table:
|
||||
return item.get("assistantId") or item.get("assistant_id") or item.get("id")
|
||||
|
||||
if "ods_store_product" in table:
|
||||
return item.get("siteGoodsId") or item.get("site_goods_id") or item.get("id")
|
||||
|
||||
if "ods_product" in table:
|
||||
return item.get("goodsId") or item.get("goods_id") or item.get("id")
|
||||
|
||||
if "ods_balance_change" in table:
|
||||
return item.get("change_id") or item.get("changeId") or item.get("id")
|
||||
|
||||
if "ods_recharge_record" in table:
|
||||
return item.get("recharge_id") or item.get("rechargeId") or item.get("id")
|
||||
|
||||
return item.get("id")
|
||||
|
||||
def _get_pk_column(self, table):
|
||||
if "ods_ticket_detail" in table: return "order_settle_id"
|
||||
if "ods_order_settle" in table: return "order_settle_id"
|
||||
if "ods_payment" in table: return "pay_id"
|
||||
if "ods_member" in table: return "member_id"
|
||||
if "ods_assistant_ledger" in table: return "ledger_id"
|
||||
if "ods_goods_ledger" in table: return "order_goods_id"
|
||||
if "ods_inventory_change" in table: return "change_id"
|
||||
if "ods_assistant_abolish" in table: return "abolish_id"
|
||||
if "ods_coupon_verify" in table: return "coupon_id"
|
||||
if "ods_member_card" in table: return "card_id"
|
||||
if "ods_package_coupon" in table: return "package_id"
|
||||
return "id" # Fallback
|
||||
def _get_conflict_columns(self, table: str) -> list[str]:
|
||||
if "ods_order_receipt_detail" in table:
|
||||
return ["order_settle_id"]
|
||||
if "ods_payment_record" in table:
|
||||
return ["pay_id"]
|
||||
if "ods_refund_record" in table:
|
||||
return ["refund_id"]
|
||||
if "ods_platform_coupon_log" in table:
|
||||
return ["coupon_id"]
|
||||
if "ods_assistant_service_log" in table or "ods_table_use_log" in table:
|
||||
return ["ledger_id"]
|
||||
if "ods_assistant_cancel_log" in table:
|
||||
return ["cancel_id"]
|
||||
if "ods_store_sale_item" in table:
|
||||
return ["sale_item_id"]
|
||||
if "ods_order_settle" in table:
|
||||
return ["order_settle_id"]
|
||||
if "ods_inventory_change" in table:
|
||||
return ["change_id"]
|
||||
if "ods_inventory_stock" in table:
|
||||
return ["site_goods_id", "snapshot_key"]
|
||||
if "ods_member_card" in table:
|
||||
return ["card_id"]
|
||||
if "ods_member_profile" in table:
|
||||
return ["member_id"]
|
||||
if "ods_group_package_log" in table:
|
||||
return ["usage_id"]
|
||||
if "ods_group_package" in table:
|
||||
return ["package_id"]
|
||||
if "ods_goods_category" in table:
|
||||
return ["category_id"]
|
||||
if "ods_table_info" in table:
|
||||
return ["table_id"]
|
||||
if "ods_table_fee_adjust" in table:
|
||||
return ["adjust_id"]
|
||||
if "ods_assistant_account" in table:
|
||||
return ["assistant_id"]
|
||||
if "ods_store_product" in table:
|
||||
return ["site_goods_id"]
|
||||
if "ods_product" in table:
|
||||
return ["goods_id"]
|
||||
if "ods_balance_change" in table:
|
||||
return ["change_id"]
|
||||
if "ods_recharge_record" in table:
|
||||
return ["recharge_id"]
|
||||
return ["id"]
|
||||
|
||||
def _enrich_row(self, row: dict, record: dict, table: str):
|
||||
"""Best-effort populate important columns from payload for PRD ODS schema."""
|
||||
def pick(obj, *keys):
|
||||
for k in keys:
|
||||
if isinstance(obj, dict) and obj.get(k) not in (None, ""):
|
||||
return obj.get(k)
|
||||
return None
|
||||
|
||||
if "ods_member_profile" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["member_name"] = pick(record, "name", "memberName")
|
||||
row["nickname"] = record.get("nickname")
|
||||
row["mobile"] = record.get("mobile")
|
||||
row["gender"] = record.get("sex")
|
||||
row["birthday"] = record.get("birthday")
|
||||
row["register_time"] = record.get("register_time") or record.get("registerTime")
|
||||
row["member_type_id"] = pick(record, "cardTypeId", "member_type_id")
|
||||
row["member_type_name"] = record.get("cardTypeName")
|
||||
row["status"] = pick(record, "status", "state")
|
||||
row["balance"] = record.get("balance")
|
||||
row["points"] = record.get("points") or record.get("point")
|
||||
row["last_visit_time"] = record.get("lastVisitTime")
|
||||
row["wechat_id"] = record.get("wechatId")
|
||||
row["alipay_id"] = record.get("alipayId")
|
||||
row["member_card_no"] = record.get("cardNo")
|
||||
row["remarks"] = record.get("remark")
|
||||
|
||||
if "ods_member_card" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["member_id"] = pick(record, "memberId", "member_id")
|
||||
row["card_type_id"] = record.get("cardTypeId")
|
||||
row["card_type_name"] = record.get("cardTypeName")
|
||||
row["card_balance"] = record.get("balance")
|
||||
row["discount_rate"] = record.get("discount") or record.get("discount_rate")
|
||||
row["valid_start_date"] = record.get("validStart")
|
||||
row["valid_end_date"] = record.get("validEnd")
|
||||
row["last_consume_time"] = record.get("lastConsumeTime")
|
||||
row["status"] = record.get("status")
|
||||
row["activate_time"] = record.get("activateTime")
|
||||
row["deactivate_time"] = record.get("cancelTime")
|
||||
row["issuer_id"] = record.get("issuerId")
|
||||
row["issuer_name"] = record.get("issuerName")
|
||||
|
||||
if "ods_recharge_record" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["member_id"] = pick(record, "memberId", "member_id")
|
||||
row["recharge_amount"] = record.get("amount") or record.get("rechargeAmount")
|
||||
row["gift_amount"] = record.get("giftAmount")
|
||||
row["pay_method"] = record.get("payType") or record.get("pay_method")
|
||||
row["pay_trade_no"] = record.get("payTradeNo")
|
||||
row["order_trade_no"] = record.get("orderTradeNo")
|
||||
row["recharge_time"] = record.get("createTime") or record.get("rechargeTime")
|
||||
row["status"] = record.get("status")
|
||||
row["operator_id"] = record.get("operatorId")
|
||||
row["operator_name"] = record.get("operatorName")
|
||||
|
||||
if "ods_balance_change" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["site_id"] = row.get("site_id") or pick(record, "siteId", "site_id")
|
||||
row["member_id"] = pick(record, "memberId", "member_id")
|
||||
row["change_amount"] = record.get("change_amount")
|
||||
row["balance_before"] = record.get("before_balance")
|
||||
row["balance_after"] = record.get("after_balance")
|
||||
row["change_type"] = record.get("from_type") or record.get("type")
|
||||
row["relate_id"] = record.get("relate_id")
|
||||
row["pay_method"] = record.get("pay_type")
|
||||
row["remark"] = record.get("remark")
|
||||
row["operator_id"] = record.get("operatorId")
|
||||
row["operator_name"] = record.get("operatorName")
|
||||
row["change_time"] = record.get("create_time") or record.get("changeTime")
|
||||
row["is_deleted"] = record.get("is_delete") or record.get("is_deleted")
|
||||
row["source_file"] = row.get("source_file")
|
||||
row["fetched_at"] = row.get("fetched_at")
|
||||
|
||||
if "ods_assistant_account" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["assistant_name"] = record.get("assistantName") or record.get("name")
|
||||
row["mobile"] = record.get("mobile")
|
||||
row["team_id"] = record.get("teamId")
|
||||
row["team_name"] = record.get("teamName")
|
||||
row["status"] = record.get("status")
|
||||
row["hired_date"] = record.get("hireDate")
|
||||
row["left_date"] = record.get("leaveDate")
|
||||
|
||||
if "ods_assistant_service_log" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["assistant_id"] = record.get("assistantId")
|
||||
row["service_type"] = record.get("serviceType")
|
||||
row["order_trade_no"] = record.get("orderTradeNo")
|
||||
row["order_settle_id"] = record.get("orderSettleId")
|
||||
row["start_time"] = record.get("startTime")
|
||||
row["end_time"] = record.get("endTime")
|
||||
row["duration_minutes"] = record.get("duration")
|
||||
row["original_fee"] = record.get("originFee") or record.get("original_fee")
|
||||
row["discount_amount"] = record.get("discountAmount")
|
||||
row["final_fee"] = record.get("finalFee") or record.get("final_fee")
|
||||
row["member_id"] = record.get("memberId")
|
||||
row["status"] = record.get("status")
|
||||
|
||||
if "ods_assistant_cancel_log" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["ledger_id"] = record.get("ledgerId")
|
||||
row["assistant_id"] = record.get("assistantId")
|
||||
row["order_trade_no"] = record.get("orderTradeNo")
|
||||
row["reason"] = record.get("reason")
|
||||
row["cancel_time"] = record.get("cancel_time") or record.get("cancelTime")
|
||||
row["operator_id"] = record.get("operatorId")
|
||||
row["operator_name"] = record.get("operatorName")
|
||||
|
||||
if "ods_table_info" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["table_code"] = record.get("tableCode")
|
||||
row["table_name"] = record.get("tableName")
|
||||
row["table_type"] = record.get("tableType")
|
||||
row["area_name"] = record.get("areaName")
|
||||
row["status"] = record.get("status")
|
||||
row["created_time"] = record.get("createTime")
|
||||
row["updated_time"] = record.get("updateTime")
|
||||
|
||||
if "ods_table_use_log" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["table_id"] = record.get("tableId")
|
||||
row["order_trade_no"] = record.get("orderTradeNo")
|
||||
row["order_settle_id"] = record.get("orderSettleId")
|
||||
row["start_time"] = record.get("startTime")
|
||||
row["end_time"] = record.get("endTime")
|
||||
row["duration_minutes"] = record.get("duration")
|
||||
row["original_table_fee"] = record.get("originFee") or record.get("original_table_fee")
|
||||
row["discount_amount"] = record.get("discountAmount")
|
||||
row["final_table_fee"] = record.get("finalFee") or record.get("final_table_fee")
|
||||
row["member_id"] = record.get("memberId")
|
||||
row["status"] = record.get("status")
|
||||
|
||||
if "ods_table_fee_adjust" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["ledger_id"] = record.get("ledgerId")
|
||||
row["order_trade_no"] = record.get("orderTradeNo")
|
||||
row["discount_amount"] = record.get("discountAmount")
|
||||
row["reason"] = record.get("reason")
|
||||
row["operator_id"] = record.get("operatorId")
|
||||
row["operator_name"] = record.get("operatorName")
|
||||
row["created_at"] = record.get("created_at") or record.get("createTime")
|
||||
|
||||
if "ods_store_product" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["goods_id"] = record.get("goodsId")
|
||||
row["goods_name"] = record.get("goodsName")
|
||||
row["category_id"] = record.get("categoryId")
|
||||
row["category_name"] = record.get("categoryName")
|
||||
row["sale_price"] = record.get("salePrice")
|
||||
row["cost_price"] = record.get("costPrice")
|
||||
row["status"] = record.get("status")
|
||||
|
||||
if "ods_store_sale_item" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["order_trade_no"] = record.get("orderTradeNo")
|
||||
row["order_settle_id"] = record.get("orderSettleId")
|
||||
row["goods_id"] = record.get("goodsId")
|
||||
row["goods_name"] = record.get("goodsName")
|
||||
row["category_id"] = record.get("categoryId")
|
||||
row["quantity"] = record.get("quantity")
|
||||
row["original_amount"] = record.get("originalAmount")
|
||||
row["discount_amount"] = record.get("discountAmount")
|
||||
row["final_amount"] = record.get("finalAmount")
|
||||
row["is_gift"] = record.get("isGift")
|
||||
row["sale_time"] = record.get("saleTime")
|
||||
|
||||
if "ods_group_package_log" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["package_id"] = record.get("packageId")
|
||||
row["coupon_id"] = record.get("couponId")
|
||||
row["order_trade_no"] = record.get("orderTradeNo")
|
||||
row["order_settle_id"] = record.get("orderSettleId")
|
||||
row["member_id"] = record.get("memberId")
|
||||
row["status"] = record.get("status")
|
||||
row["used_time"] = record.get("usedTime")
|
||||
row["deduct_amount"] = record.get("deductAmount")
|
||||
row["settle_price"] = record.get("settlePrice")
|
||||
|
||||
if "ods_group_package" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["package_name"] = record.get("packageName")
|
||||
row["platform_code"] = record.get("platformCode")
|
||||
row["status"] = record.get("status")
|
||||
row["face_price"] = record.get("facePrice")
|
||||
row["settle_price"] = record.get("settlePrice")
|
||||
row["valid_from"] = record.get("validFrom")
|
||||
row["valid_to"] = record.get("validTo")
|
||||
|
||||
if "ods_platform_coupon_log" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["platform_code"] = record.get("platformCode")
|
||||
row["verify_code"] = record.get("verifyCode")
|
||||
row["order_trade_no"] = record.get("orderTradeNo")
|
||||
row["order_settle_id"] = record.get("orderSettleId")
|
||||
row["member_id"] = record.get("memberId")
|
||||
row["status"] = record.get("status")
|
||||
row["used_time"] = record.get("usedTime")
|
||||
row["deduct_amount"] = record.get("deductAmount")
|
||||
row["settle_price"] = record.get("settlePrice")
|
||||
|
||||
if "ods_payment_record" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["order_trade_no"] = record.get("orderTradeNo")
|
||||
row["order_settle_id"] = record.get("orderSettleId")
|
||||
row["member_id"] = record.get("memberId")
|
||||
row["pay_method_code"] = record.get("payMethodCode") or record.get("pay_type")
|
||||
row["pay_method_name"] = record.get("payMethodName")
|
||||
row["pay_amount"] = record.get("payAmount")
|
||||
row["pay_time"] = record.get("payTime")
|
||||
row["relate_type"] = record.get("relateType")
|
||||
row["relate_id"] = record.get("relateId")
|
||||
|
||||
if "ods_refund_record" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["order_trade_no"] = record.get("orderTradeNo")
|
||||
row["order_settle_id"] = record.get("orderSettleId")
|
||||
row["member_id"] = record.get("memberId")
|
||||
row["pay_method_code"] = record.get("payMethodCode")
|
||||
row["refund_amount"] = record.get("refundAmount")
|
||||
row["refund_time"] = record.get("refundTime")
|
||||
row["status"] = record.get("status")
|
||||
|
||||
if "ods_inventory_change" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["site_goods_id"] = record.get("siteGoodsId")
|
||||
row["goods_id"] = record.get("goodsId")
|
||||
row["change_amount"] = record.get("changeAmount")
|
||||
row["before_stock"] = record.get("beforeStock")
|
||||
row["after_stock"] = record.get("afterStock")
|
||||
row["change_type"] = record.get("changeType")
|
||||
row["relate_id"] = record.get("relateId")
|
||||
row["remark"] = record.get("remark")
|
||||
row["operator_id"] = record.get("operatorId")
|
||||
row["operator_name"] = record.get("operatorName")
|
||||
row["change_time"] = record.get("changeTime")
|
||||
|
||||
if "ods_inventory_stock" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["goods_id"] = record.get("goodsId")
|
||||
row["current_stock"] = record.get("currentStock")
|
||||
row["cost_price"] = record.get("costPrice")
|
||||
|
||||
if "ods_goods_category" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["category_name"] = record.get("categoryName")
|
||||
row["parent_id"] = record.get("parentId")
|
||||
row["level_no"] = record.get("levelNo")
|
||||
row["status"] = record.get("status")
|
||||
row["remark"] = record.get("remark")
|
||||
|
||||
if "ods_order_receipt_detail" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["order_trade_no"] = record.get("orderTradeNo")
|
||||
row["receipt_no"] = record.get("receiptNo")
|
||||
row["receipt_time"] = record.get("receiptTime")
|
||||
row["total_amount"] = record.get("totalAmount")
|
||||
row["discount_amount"] = record.get("discountAmount")
|
||||
row["final_amount"] = record.get("finalAmount")
|
||||
row["member_id"] = record.get("memberId")
|
||||
row["snapshot_raw"] = record.get("siteProfile") or record.get("site_profile")
|
||||
|
||||
if "ods_order_settle" in table:
|
||||
settle = record.get("settleList") if isinstance(record.get("settleList"), dict) else record
|
||||
if isinstance(settle, dict):
|
||||
row["tenant_id"] = pick(settle, "tenantId", "tenant_id")
|
||||
row["settle_relate_id"] = settle.get("settleRelateId")
|
||||
row["settle_name"] = settle.get("settleName")
|
||||
row["settle_type"] = settle.get("settleType")
|
||||
row["settle_status"] = settle.get("settleStatus")
|
||||
row["member_id"] = settle.get("memberId")
|
||||
row["member_phone"] = settle.get("memberPhone")
|
||||
row["table_id"] = settle.get("tableId")
|
||||
row["consume_money"] = settle.get("consumeMoney")
|
||||
row["table_charge_money"] = settle.get("tableChargeMoney")
|
||||
row["goods_money"] = settle.get("goodsMoney")
|
||||
row["service_money"] = settle.get("serviceMoney")
|
||||
row["assistant_pd_money"] = settle.get("assistantPdMoney")
|
||||
row["assistant_cx_money"] = settle.get("assistantCxMoney")
|
||||
row["pay_amount"] = settle.get("payAmount")
|
||||
row["coupon_amount"] = settle.get("couponAmount")
|
||||
row["card_amount"] = settle.get("cardAmount")
|
||||
row["balance_amount"] = settle.get("balanceAmount")
|
||||
row["refund_amount"] = settle.get("refundAmount")
|
||||
row["prepay_money"] = settle.get("prepayMoney")
|
||||
row["adjust_amount"] = settle.get("adjustAmount")
|
||||
row["rounding_amount"] = settle.get("roundingAmount")
|
||||
row["payment_method"] = settle.get("paymentMethod")
|
||||
row["create_time"] = settle.get("createTime")
|
||||
row["pay_time"] = settle.get("payTime")
|
||||
row["operator_id"] = settle.get("operatorId")
|
||||
row["operator_name"] = settle.get("operatorName")
|
||||
|
||||
if "ods_product" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
row["goods_id"] = record.get("goodsId")
|
||||
row["goods_name"] = record.get("goodsName")
|
||||
row["goods_code"] = record.get("goodsCode")
|
||||
row["category_id"] = record.get("categoryId")
|
||||
row["category_name"] = record.get("categoryName")
|
||||
row["unit"] = record.get("unit")
|
||||
row["price"] = record.get("price")
|
||||
row["status"] = record.get("status")
|
||||
|
||||
if "ods_platform_coupon_log" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
|
||||
if "ods_table_use_log" in table:
|
||||
row["tenant_id"] = pick(record, "tenantId", "tenant_id")
|
||||
|
||||
def _ensure_tuple(self, value):
|
||||
if isinstance(value, tuple):
|
||||
return value
|
||||
return (value,)
|
||||
|
||||
def _bulk_insert(self, table: str, rows: list[dict]):
|
||||
if not rows:
|
||||
return
|
||||
|
||||
columns = list(rows[0].keys())
|
||||
col_clause = ", ".join(columns)
|
||||
val_clause = ", ".join(f"%({col})s" for col in columns)
|
||||
conflict_cols = ["site_id"] + self._get_conflict_columns(table)
|
||||
conflict_clause = ", ".join(conflict_cols)
|
||||
|
||||
def _bulk_insert(self, table, rows):
|
||||
if not rows: return
|
||||
|
||||
keys = list(rows[0].keys())
|
||||
cols = ", ".join(keys)
|
||||
vals = ", ".join([f"%({k})s" for k in keys])
|
||||
|
||||
# Determine PK col for conflict
|
||||
pk_col = self._get_pk_column(table)
|
||||
|
||||
sql = f"""
|
||||
INSERT INTO {table} ({cols})
|
||||
VALUES ({vals})
|
||||
ON CONFLICT (store_id, {pk_col}) DO UPDATE SET
|
||||
INSERT INTO {table} ({col_clause})
|
||||
VALUES ({val_clause})
|
||||
ON CONFLICT ({conflict_clause}) DO UPDATE SET
|
||||
payload = EXCLUDED.payload,
|
||||
fetched_at = EXCLUDED.fetched_at,
|
||||
source_file = EXCLUDED.source_file;
|
||||
source_file = EXCLUDED.source_file
|
||||
"""
|
||||
self.db.batch_execute(sql, rows)
|
||||
|
||||
Reference in New Issue
Block a user