1311 lines
52 KiB
Python
1311 lines
52 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""DWD 层批量校验器
|
||
|
||
校验逻辑：对比 ODS 源数据与 DWD 表数据
|
||
- 维度表：SCD2 模式，对比当前版本
|
||
- 事实表：主键对比，批量 UPSERT 补齐
|
||
"""
|
||
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import time
|
||
from datetime import datetime, timezone
|
||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||
|
||
from psycopg2.extras import Json, execute_values
|
||
|
||
from .base_verifier import BaseVerifier, VerificationFetchError
|
||
from tasks.dwd.dwd_load_task import DwdLoadTask
|
||
|
||
|
||
class DwdVerifier(BaseVerifier):
|
||
"""DWD 层校验器"""
|
||
|
||
def __init__(
|
||
self,
|
||
db_connection: Any,
|
||
logger: Optional[logging.Logger] = None,
|
||
config: Any = None,
|
||
):
|
||
"""
|
||
初始化 DWD 校验器
|
||
|
||
Args:
|
||
db_connection: 数据库连接
|
||
logger: 日志器
|
||
"""
|
||
super().__init__(db_connection, logger)
|
||
self._table_config = self._load_table_config()
|
||
self.config = config
|
||
|
||
@property
|
||
def layer_name(self) -> str:
|
||
return "DWD"
|
||
|
||
def _load_table_config(self) -> Dict[str, dict]:
|
||
"""加载 DWD 表配置"""
|
||
# ODS 表主键列名映射（ODS 列名通常都是 id，特殊情况单独配置）
|
||
# 格式：ods_table -> ods_pk_column
|
||
ODS_PK_MAP = {
|
||
"table_fee_transactions": "id",
|
||
"site_tables_master": "id",
|
||
"assistant_accounts_master": "id",
|
||
"member_profiles": "id",
|
||
"member_stored_value_cards": "id",
|
||
"tenant_goods_master": "id",
|
||
"store_goods_master": "id",
|
||
"stock_goods_category_tree": "id",
|
||
"group_buy_packages": "id",
|
||
"settlement_records": "id",
|
||
"table_fee_discount_records": "id",
|
||
"store_goods_sales_records": "id",
|
||
"assistant_service_records": "id",
|
||
"assistant_cancellation_records": "id",
|
||
"member_balance_changes": "id",
|
||
"group_buy_redemption_records": "id",
|
||
"platform_coupon_redemption_records": "id",
|
||
"recharge_settlements": "id", # 注意：这里 ODS 列是 id，但 DWD 列是 recharge_order_id
|
||
"payment_transactions": "id",
|
||
"refund_transactions": "id",
|
||
"goods_stock_summary": "sitegoodsid", # 特殊：主键不是 id
|
||
"settlement_ticket_details": "ordersettleid", # 特殊：主键不是 id
|
||
}
|
||
|
||
# ODS 主键特殊覆盖（按 DWD 表名）
|
||
# 格式：dwd_table -> ods_pk_columns
|
||
ODS_PK_OVERRIDE = {
|
||
"dim_site": ["site_id"],
|
||
"dim_site_ex": ["site_id"],
|
||
}
|
||
|
||
# ODS 到 DWD 主键列名映射（ODS 的 id 对应 DWD 的语义化列名）
|
||
# 格式：dwd_table -> {ods_column: dwd_column}
|
||
ODS_TO_DWD_PK_MAP = {
|
||
# 维度表（复杂映射的表设为空字典，跳过 backfill）
|
||
"dim_site": {"site_id": "site_id"},
|
||
"dim_site_ex": {"site_id": "site_id"},
|
||
"dim_table": {"id": "table_id"},
|
||
"dim_table_ex": {"id": "table_id"},
|
||
"dim_assistant": {"id": "assistant_id"},
|
||
"dim_assistant_ex": {"id": "assistant_id"},
|
||
"dim_member": {"id": "member_id"},
|
||
"dim_member_ex": {"id": "member_id"},
|
||
"dim_member_card_account": {"id": "member_card_id"},
|
||
"dim_member_card_account_ex": {"id": "member_card_id"},
|
||
"dim_tenant_goods": {"id": "tenant_goods_id"},
|
||
"dim_tenant_goods_ex": {"id": "tenant_goods_id"},
|
||
"dim_store_goods": {"id": "site_goods_id"},
|
||
"dim_store_goods_ex": {"id": "site_goods_id"},
|
||
"dim_goods_category": {"id": "category_id"},
|
||
"dim_groupbuy_package": {"id": "groupbuy_package_id"},
|
||
"dim_groupbuy_package_ex": {"id": "groupbuy_package_id"},
|
||
# 事实表
|
||
"dwd_settlement_head": {"id": "order_settle_id"},
|
||
"dwd_settlement_head_ex": {"id": "order_settle_id"},
|
||
"dwd_table_fee_log": {"id": "table_fee_log_id"},
|
||
"dwd_table_fee_log_ex": {"id": "table_fee_log_id"},
|
||
"dwd_table_fee_adjust": {"id": "table_fee_adjust_id"},
|
||
"dwd_table_fee_adjust_ex": {"id": "table_fee_adjust_id"},
|
||
"dwd_store_goods_sale": {"id": "store_goods_sale_id"},
|
||
"dwd_store_goods_sale_ex": {"id": "store_goods_sale_id"},
|
||
"dwd_assistant_service_log": {"id": "assistant_service_id"},
|
||
"dwd_assistant_service_log_ex": {"id": "assistant_service_id"},
|
||
"dwd_assistant_trash_event": {"id": "assistant_trash_event_id"},
|
||
"dwd_assistant_trash_event_ex": {"id": "assistant_trash_event_id"},
|
||
"dwd_member_balance_change": {"id": "balance_change_id"},
|
||
"dwd_member_balance_change_ex": {"id": "balance_change_id"},
|
||
"dwd_groupbuy_redemption": {"id": "redemption_id"},
|
||
"dwd_groupbuy_redemption_ex": {"id": "redemption_id"},
|
||
"dwd_platform_coupon_redemption": {"id": "platform_coupon_redemption_id"},
|
||
"dwd_platform_coupon_redemption_ex": {"id": "platform_coupon_redemption_id"},
|
||
"dwd_recharge_order": {"id": "recharge_order_id"},
|
||
"dwd_recharge_order_ex": {"id": "recharge_order_id"},
|
||
"dwd_payment": {"id": "payment_id"},
|
||
"dwd_refund": {"id": "refund_id"},
|
||
"dwd_refund_ex": {"id": "refund_id"},
|
||
}
|
||
|
||
# DWD 事实表的业务时间列映射（用于时间窗口过滤）
|
||
DWD_TIME_COL_MAP = {
|
||
"dwd_settlement_head": "pay_time",
|
||
"dwd_settlement_head_ex": "pay_time",
|
||
"dwd_table_fee_log": "start_use_time",
|
||
"dwd_table_fee_log_ex": "start_use_time",
|
||
"dwd_table_fee_adjust": "create_time",
|
||
"dwd_table_fee_adjust_ex": "create_time",
|
||
"dwd_store_goods_sale": "create_time",
|
||
"dwd_store_goods_sale_ex": "create_time",
|
||
"dwd_assistant_service_log": "start_use_time",
|
||
"dwd_assistant_service_log_ex": "start_use_time",
|
||
"dwd_assistant_trash_event": "create_time",
|
||
"dwd_assistant_trash_event_ex": "create_time",
|
||
"dwd_member_balance_change": "create_time",
|
||
"dwd_member_balance_change_ex": "create_time",
|
||
"dwd_groupbuy_redemption": "create_time",
|
||
"dwd_groupbuy_redemption_ex": "create_time",
|
||
"dwd_platform_coupon_redemption": "create_time",
|
||
"dwd_platform_coupon_redemption_ex": "create_time",
|
||
"dwd_recharge_order": "pay_time",
|
||
"dwd_recharge_order_ex": "pay_time",
|
||
"dwd_payment": "pay_time",
|
||
"dwd_refund": "create_time",
|
||
"dwd_refund_ex": "create_time",
|
||
}
|
||
|
||
scd2_cols = {"scd2_start_time", "scd2_end_time", "scd2_is_current", "scd2_version"}
|
||
|
||
try:
|
||
# 尝试多种导入路径以兼容不同运行环境
|
||
from tasks.dwd.dwd_load_task import DwdLoadTask
|
||
config = {}
|
||
for full_dwd_table, full_ods_table in DwdLoadTask.TABLE_MAP.items():
|
||
# 提取不带 schema 前缀的表名
|
||
if "." in full_dwd_table:
|
||
dwd_table = full_dwd_table.split(".")[-1]
|
||
else:
|
||
dwd_table = full_dwd_table
|
||
|
||
if "." in full_ods_table:
|
||
ods_table = full_ods_table.split(".")[-1]
|
||
else:
|
||
ods_table = full_ods_table
|
||
|
||
is_dimension = dwd_table.startswith("dim_")
|
||
|
||
# 获取 ODS 表的主键列名（用于查询 ODS）
|
||
ods_pk_column = ODS_PK_MAP.get(ods_table, "id")
|
||
ods_pk_columns = ODS_PK_OVERRIDE.get(dwd_table)
|
||
if not ods_pk_columns:
|
||
ods_pk_columns = [ods_pk_column]
|
||
|
||
# 获取 DWD 表的时间列（用于时间窗口过滤）
|
||
time_column = DWD_TIME_COL_MAP.get(dwd_table, "fetched_at")
|
||
# 维度表使用 scd2_start_time
|
||
if is_dimension:
|
||
time_column = "scd2_start_time"
|
||
|
||
# 若未配置主键映射，且业务主键与 ODS 主键同名，则自动推断映射
|
||
pk_columns = self._get_pk_from_db(dwd_table)
|
||
business_pk_cols = [c for c in pk_columns if c.lower() not in scd2_cols]
|
||
ods_to_dwd_map = ODS_TO_DWD_PK_MAP.get(dwd_table, {})
|
||
if not ods_to_dwd_map and business_pk_cols:
|
||
if all(pk in ods_pk_columns for pk in business_pk_cols):
|
||
ods_to_dwd_map = {pk: pk for pk in business_pk_cols}
|
||
|
||
config[dwd_table] = {
|
||
"full_dwd_table": full_dwd_table,
|
||
"ods_table": ods_table,
|
||
"full_ods_table": full_ods_table,
|
||
"is_dimension": is_dimension,
|
||
"pk_columns": pk_columns, # DWD 表的主键
|
||
"ods_pk_columns": ods_pk_columns, # ODS 表的主键（用于查询 ODS）
|
||
"ods_to_dwd_pk_map": ods_to_dwd_map, # ODS 到 DWD 主键映射
|
||
"time_column": time_column, # DWD 时间列
|
||
"ods_time_column": "fetched_at", # ODS 时间列
|
||
}
|
||
return config
|
||
except (ImportError, AttributeError) as e:
|
||
self.logger.warning("无法加载 DWD 表映射，使用数据库查询: %s", e)
|
||
return {}
|
||
|
||
def _get_pk_from_db(self, table: str) -> List[str]:
|
||
"""从数据库获取表的主键"""
|
||
sql = """
|
||
SELECT kcu.column_name
|
||
FROM information_schema.table_constraints tc
|
||
JOIN information_schema.key_column_usage kcu
|
||
ON tc.constraint_name = kcu.constraint_name
|
||
AND tc.table_schema = kcu.table_schema
|
||
WHERE tc.constraint_type = 'PRIMARY KEY'
|
||
AND tc.table_schema = 'dwd'
|
||
AND tc.table_name = %s
|
||
ORDER BY kcu.ordinal_position
|
||
"""
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(sql, (table,))
|
||
result = [row[0] for row in cur.fetchall()]
|
||
return result if result else ["id"]
|
||
except Exception as e:
|
||
self.logger.warning("获取 DWD 主键失败: %s, error=%s", table, e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
return ["id"]
|
||
|
||
def get_tables(self) -> List[str]:
|
||
"""获取需要校验的 DWD 表列表"""
|
||
if self._table_config:
|
||
return list(self._table_config.keys())
|
||
|
||
sql = """
|
||
SELECT table_name
|
||
FROM information_schema.tables
|
||
WHERE table_schema = 'dwd'
|
||
AND table_type = 'BASE TABLE'
|
||
ORDER BY table_name
|
||
"""
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(sql)
|
||
return [row[0] for row in cur.fetchall()]
|
||
except Exception as e:
|
||
self.logger.warning("获取 DWD 表列表失败: %s", e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
return []
|
||
|
||
def get_dimension_tables(self) -> List[str]:
|
||
"""获取维度表列表"""
|
||
return [t for t in self.get_tables() if t.startswith("dim_")]
|
||
|
||
def get_fact_tables(self) -> List[str]:
|
||
"""获取事实表列表"""
|
||
return [t for t in self.get_tables() if t.startswith("dwd_") or t.startswith("fact_")]
|
||
|
||
def get_primary_keys(self, table: str) -> List[str]:
|
||
"""获取表的主键列"""
|
||
if table in self._table_config:
|
||
pk_cols = self._table_config[table].get("pk_columns", [])
|
||
if pk_cols:
|
||
return pk_cols
|
||
# 尝试从数据库获取，如果配置中没有或为空
|
||
return self._get_pk_from_db(table)
|
||
|
||
def get_time_column(self, table: str) -> Optional[str]:
|
||
"""获取表的时间列"""
|
||
if table in self._table_config:
|
||
return self._table_config[table].get("time_column", "create_time")
|
||
|
||
# 尝试从表结构中查找常见的时间列
|
||
common_time_cols = ["create_time", "pay_time", "start_time", "modify_time", "fetched_at"]
|
||
try:
|
||
sql = """
|
||
SELECT column_name
|
||
FROM information_schema.columns
|
||
WHERE table_schema = 'dwd'
|
||
AND table_name = %s
|
||
AND column_name = ANY(%s)
|
||
"""
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(sql, (table, common_time_cols))
|
||
rows = cur.fetchall()
|
||
if rows:
|
||
return rows[0][0]
|
||
except Exception:
|
||
pass
|
||
|
||
return "create_time"
|
||
|
||
def get_ods_table(self, dwd_table: str) -> Optional[str]:
|
||
"""获取 DWD 表对应的 ODS 源表"""
|
||
if dwd_table in self._table_config:
|
||
return self._table_config[dwd_table].get("ods_table")
|
||
|
||
# 推断 ODS 表名
|
||
if dwd_table.startswith("dim_"):
|
||
ods_name = dwd_table.replace("dim_", "ods_")
|
||
elif dwd_table.startswith("dwd_"):
|
||
ods_name = dwd_table.replace("dwd_", "ods_")
|
||
else:
|
||
ods_name = f"ods_{dwd_table}"
|
||
|
||
return ods_name
|
||
|
||
def is_dimension_table(self, table: str) -> bool:
|
||
"""判断是否为维度表"""
|
||
if table in self._table_config:
|
||
return self._table_config[table].get("is_dimension", False)
|
||
return table.startswith("dim_")
|
||
|
||
def get_ods_pk_columns(self, table: str) -> List[str]:
|
||
"""获取 ODS 表的主键列名（用于查询 ODS）"""
|
||
if table in self._table_config:
|
||
return self._table_config[table].get("ods_pk_columns", ["id"])
|
||
return ["id"]
|
||
|
||
def get_ods_time_column(self, table: str) -> str:
|
||
"""获取 ODS 表的时间列名"""
|
||
if table in self._table_config:
|
||
return self._table_config[table].get("ods_time_column", "fetched_at")
|
||
return "fetched_at"
|
||
|
||
def get_ods_to_dwd_pk_map(self, table: str) -> Dict[str, str]:
|
||
"""获取 ODS 到 DWD 主键列名映射
|
||
|
||
返回 {ods_column: dwd_column} 映射字典
|
||
"""
|
||
if table in self._table_config:
|
||
mapping = self._table_config[table].get("ods_to_dwd_pk_map", {})
|
||
if mapping:
|
||
return mapping
|
||
# 若未显式配置映射，尝试用同名业务主键兜底
|
||
scd2_cols = {"scd2_start_time", "scd2_end_time", "scd2_is_current", "scd2_version"}
|
||
pk_cols = self.get_primary_keys(table)
|
||
business_pk_cols = [c for c in pk_cols if c.lower() not in scd2_cols]
|
||
ods_pk_cols = self.get_ods_pk_columns(table)
|
||
if business_pk_cols and all(pk in ods_pk_cols for pk in business_pk_cols):
|
||
return {pk: pk for pk in business_pk_cols}
|
||
return {}
|
||
return {}
|
||
|
||
def fetch_source_keys(
|
||
self,
|
||
table: str,
|
||
window_start: datetime,
|
||
window_end: datetime,
|
||
) -> Set[Tuple]:
|
||
"""从 ODS 源表获取主键集合
|
||
|
||
注意：使用 fetched_at 过滤 ODS 数据。这意味着只检查最近获取的 ODS 记录
|
||
是否正确同步到 DWD 表。历史数据不在校验范围内。
|
||
"""
|
||
ods_table = self.get_ods_table(table)
|
||
if not ods_table:
|
||
return set()
|
||
|
||
# 使用 ODS 表的主键列名（不是 DWD 的）
|
||
ods_pk_cols = self.get_ods_pk_columns(table)
|
||
|
||
# 如果没有主键定义，跳过查询
|
||
if not ods_pk_cols:
|
||
self.logger.debug("表 %s 没有 ODS 主键配置，跳过获取源主键", table)
|
||
return set()
|
||
|
||
# 使用 ODS 的时间列
|
||
ods_time_col = self.get_ods_time_column(table)
|
||
|
||
pk_select = ", ".join(ods_pk_cols)
|
||
sql = f"""
|
||
SELECT DISTINCT {pk_select}
|
||
FROM ods.{ods_table}
|
||
WHERE {ods_time_col} >= %s AND {ods_time_col} < %s
|
||
"""
|
||
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(sql, (window_start, window_end))
|
||
return {tuple(row) for row in cur.fetchall()}
|
||
except Exception as e:
|
||
self.logger.warning("获取 ODS 主键失败: %s, error=%s", ods_table, e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
raise VerificationFetchError(f"获取 ODS 主键失败: {ods_table}") from e
|
||
|
||
def fetch_target_keys(
|
||
self,
|
||
table: str,
|
||
window_start: datetime,
|
||
window_end: datetime,
|
||
) -> Set[Tuple]:
|
||
"""从 DWD 表获取主键集合
|
||
|
||
注意：为了与 fetch_source_keys 返回的 ODS 主键进行比较，
|
||
这里返回的是业务主键（映射后的 DWD 列，与 ODS 主键数量相同）。
|
||
对于维度表，不包含 scd2_start_time。
|
||
"""
|
||
# 获取 ODS 到 DWD 的主键映射
|
||
ods_to_dwd_map = self.get_ods_to_dwd_pk_map(table)
|
||
|
||
# 确定要查询的主键列
|
||
if ods_to_dwd_map:
|
||
# 使用映射的 DWD 业务主键列（与 ODS 主键数量相同）
|
||
dwd_pk_cols = list(ods_to_dwd_map.values())
|
||
else:
|
||
# 没有映射，使用原始主键（可能无法与 ODS 正确比较）
|
||
dwd_pk_cols = self.get_primary_keys(table)
|
||
if not dwd_pk_cols:
|
||
self.logger.debug("表 %s 没有主键配置，跳过获取目标主键", table)
|
||
return set()
|
||
|
||
pk_select = ", ".join(dwd_pk_cols)
|
||
|
||
# 构建查询
|
||
if self.is_dimension_table(table):
|
||
# 维度表：查询当前版本
|
||
sql = f"""
|
||
SELECT DISTINCT {pk_select}
|
||
FROM dwd.{table}
|
||
WHERE scd2_is_current = 1
|
||
"""
|
||
params = ()
|
||
else:
|
||
# 事实表：使用时间窗口过滤
|
||
time_col = self.get_time_column(table)
|
||
|
||
# 检查时间列是否存在
|
||
time_col_exists = False
|
||
try:
|
||
check_sql = """
|
||
SELECT 1 FROM information_schema.columns
|
||
WHERE table_schema = 'dwd'
|
||
AND table_name = %s AND column_name = %s
|
||
"""
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(check_sql, (table, time_col))
|
||
if cur.fetchone():
|
||
time_col_exists = True
|
||
else:
|
||
# 尝试其他时间列
|
||
fallback_cols = ["create_time", "pay_time", "start_use_time"]
|
||
for fc in fallback_cols:
|
||
cur.execute(check_sql, (table, fc))
|
||
if cur.fetchone():
|
||
time_col = fc
|
||
time_col_exists = True
|
||
break
|
||
except Exception:
|
||
pass
|
||
|
||
if time_col_exists:
|
||
sql = f"""
|
||
SELECT DISTINCT {pk_select}
|
||
FROM dwd.{table}
|
||
WHERE {time_col} >= %s AND {time_col} < %s
|
||
"""
|
||
params = (window_start, window_end)
|
||
else:
|
||
# 没有时间列，获取全部数据
|
||
sql = f"""
|
||
SELECT DISTINCT {pk_select}
|
||
FROM dwd.{table}
|
||
"""
|
||
params = ()
|
||
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(sql, params)
|
||
return {tuple(row) for row in cur.fetchall()}
|
||
except Exception as e:
|
||
self.logger.warning("获取 DWD 主键失败: %s, error=%s", table, e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
raise VerificationFetchError(f"获取 DWD 主键失败: {table}") from e
|
||
|
||
def fetch_source_hashes(
|
||
self,
|
||
table: str,
|
||
window_start: datetime,
|
||
window_end: datetime,
|
||
) -> Dict[Tuple, str]:
|
||
"""从 ODS 源表获取主键->content_hash 映射"""
|
||
ods_table = self.get_ods_table(table)
|
||
if not ods_table:
|
||
return {}
|
||
|
||
# 使用 ODS 表的主键列名（不是 DWD 的）
|
||
ods_pk_cols = self.get_ods_pk_columns(table)
|
||
|
||
# 如果没有主键定义，跳过查询
|
||
if not ods_pk_cols:
|
||
self.logger.debug("表 %s 没有 ODS 主键配置，跳过获取源哈希", table)
|
||
return {}
|
||
|
||
# 使用 ODS 的时间列
|
||
ods_time_col = self.get_ods_time_column(table)
|
||
|
||
pk_select = ", ".join(ods_pk_cols)
|
||
sql = f"""
|
||
SELECT {pk_select}, content_hash
|
||
FROM ods.{ods_table}
|
||
WHERE {ods_time_col} >= %s AND {ods_time_col} < %s
|
||
"""
|
||
|
||
result = {}
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(sql, (window_start, window_end))
|
||
for row in cur.fetchall():
|
||
pk = tuple(row[:-1])
|
||
content_hash = row[-1]
|
||
result[pk] = content_hash or ""
|
||
except Exception as e:
|
||
self.logger.warning("获取 ODS hash 失败: %s, error=%s", ods_table, e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
raise VerificationFetchError(f"获取 ODS hash 失败: {ods_table}") from e
|
||
|
||
return result
|
||
|
||
def fetch_target_hashes(
|
||
self,
|
||
table: str,
|
||
window_start: datetime,
|
||
window_end: datetime,
|
||
) -> Dict[Tuple, str]:
|
||
"""从 DWD 表获取主键->计算的哈希 映射"""
|
||
pk_cols = self.get_primary_keys(table)
|
||
|
||
# 如果没有主键定义，跳过查询
|
||
if not pk_cols:
|
||
self.logger.debug("表 %s 没有主键配置，跳过获取目标哈希", table)
|
||
return {}
|
||
|
||
# DWD 表可能没有 content_hash，需要计算
|
||
# 获取所有非系统列
|
||
exclude_cols = {
|
||
"scd2_start_time", "scd2_end_time", "scd2_is_current", "scd2_version",
|
||
"dwd_insert_time", "dwd_update_time"
|
||
}
|
||
|
||
sql = f"""
|
||
SELECT column_name
|
||
FROM information_schema.columns
|
||
WHERE table_schema = 'dwd'
|
||
AND table_name = %s
|
||
ORDER BY ordinal_position
|
||
"""
|
||
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(sql, (table,))
|
||
all_cols = [row[0] for row in cur.fetchall()]
|
||
except Exception as e:
|
||
self.logger.warning("获取 DWD 表列信息失败: %s, error=%s", table, e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
all_cols = pk_cols
|
||
|
||
data_cols = [c for c in all_cols if c not in exclude_cols]
|
||
col_select = ", ".join(data_cols)
|
||
pk_indices = [data_cols.index(c) for c in pk_cols if c in data_cols]
|
||
|
||
if self.is_dimension_table(table):
|
||
sql = f"""
|
||
SELECT {col_select}
|
||
FROM dwd.{table}
|
||
WHERE scd2_is_current = 1
|
||
"""
|
||
params = ()
|
||
else:
|
||
# 事实表使用 DWD 的业务时间列
|
||
time_col = self.get_time_column(table)
|
||
# 检查时间列是否在数据列中
|
||
if time_col not in data_cols:
|
||
# 时间列不存在，使用备选方案
|
||
fallback_cols = ["create_time", "pay_time", "start_use_time"]
|
||
time_col = None
|
||
for fc in fallback_cols:
|
||
if fc in data_cols:
|
||
time_col = fc
|
||
break
|
||
|
||
if not time_col:
|
||
# 没有找到时间列，查询全部数据
|
||
sql = f"""
|
||
SELECT {col_select}
|
||
FROM dwd.{table}
|
||
"""
|
||
params = ()
|
||
else:
|
||
sql = f"""
|
||
SELECT {col_select}
|
||
FROM dwd.{table}
|
||
WHERE {time_col} >= %s AND {time_col} < %s
|
||
"""
|
||
params = (window_start, window_end)
|
||
else:
|
||
sql = f"""
|
||
SELECT {col_select}
|
||
FROM dwd.{table}
|
||
WHERE {time_col} >= %s AND {time_col} < %s
|
||
"""
|
||
params = (window_start, window_end)
|
||
|
||
result = {}
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(sql, params)
|
||
for row in cur.fetchall():
|
||
pk = tuple(row[i] for i in pk_indices)
|
||
# 计算整行数据的哈希
|
||
row_dict = dict(zip(data_cols, row))
|
||
content_str = json.dumps(row_dict, sort_keys=True, default=str)
|
||
content_hash = hashlib.md5(content_str.encode()).hexdigest()
|
||
result[pk] = content_hash
|
||
except Exception as e:
|
||
self.logger.warning("获取 DWD hash 失败: %s, error=%s", table, e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
raise VerificationFetchError(f"获取 DWD hash 失败: {table}") from e
|
||
|
||
return result
|
||
|
||
def backfill_missing(
|
||
self,
|
||
table: str,
|
||
missing_keys: Set[Tuple],
|
||
window_start: datetime,
|
||
window_end: datetime,
|
||
) -> int:
|
||
"""批量补齐缺失数据"""
|
||
if not missing_keys:
|
||
return 0
|
||
|
||
ods_table = self.get_ods_table(table)
|
||
if not ods_table:
|
||
return 0
|
||
|
||
# 检查是否有主键映射（用于判断是否可以 backfill）
|
||
ods_to_dwd_map = self.get_ods_to_dwd_pk_map(table)
|
||
if not ods_to_dwd_map and self.is_dimension_table(table):
|
||
# 维度表没有主键映射，可能是复杂映射（如从嵌套 JSON 提取）
|
||
# 无法自动 backfill，跳过
|
||
self.logger.warning(
|
||
"DWD 表 %s 没有主键映射配置，跳过 backfill（需要完整 ETL 同步）",
|
||
table
|
||
)
|
||
return 0
|
||
|
||
pk_cols = self.get_primary_keys(table) # DWD 主键列名
|
||
ods_pk_cols = self.get_ods_pk_columns(table) # ODS 主键列名（通常是 id）
|
||
ods_time_col = self.get_ods_time_column(table)
|
||
|
||
self.logger.info(
|
||
"DWD 补齐缺失: 表=%s, 数量=%d",
|
||
table, len(missing_keys)
|
||
)
|
||
|
||
# 在执行之前确保事务状态干净
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
|
||
# 过滤主键列数不匹配的数据
|
||
valid_keys = [pk for pk in missing_keys if len(pk) == len(ods_pk_cols)]
|
||
if not valid_keys:
|
||
return 0
|
||
|
||
# 分批通过 VALUES + JOIN 回查 ODS，避免超长 OR 条件导致 SQL 解析/执行变慢
|
||
batch_size = 1000
|
||
records: List[dict] = []
|
||
key_cols_sql = ", ".join(ods_pk_cols)
|
||
join_sql = " AND ".join(f"o.{col} = k.{col}" for col in ods_pk_cols)
|
||
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
for i in range(0, len(valid_keys), batch_size):
|
||
batch_keys = valid_keys[i:i + batch_size]
|
||
row_placeholder = "(" + ", ".join(["%s"] * len(ods_pk_cols)) + ")"
|
||
values_sql = ", ".join([row_placeholder] * len(batch_keys))
|
||
params = [v for pk in batch_keys for v in pk]
|
||
sql = f"""
|
||
WITH k ({key_cols_sql}) AS (
|
||
VALUES {values_sql}
|
||
)
|
||
SELECT o.*
|
||
FROM ods.{ods_table} o
|
||
JOIN k ON {join_sql}
|
||
WHERE o.{ods_time_col} >= %s AND o.{ods_time_col} < %s
|
||
"""
|
||
cur.execute(sql, params + [window_start, window_end])
|
||
columns = [desc[0] for desc in cur.description]
|
||
records.extend(dict(zip(columns, row)) for row in cur.fetchall())
|
||
except Exception as e:
|
||
self.logger.error("获取 ODS 记录失败: %s", e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
return 0
|
||
|
||
if not records:
|
||
return 0
|
||
|
||
# 执行 DWD 装载
|
||
return self._load_to_dwd(table, records, pk_cols)
|
||
|
||
def backfill_mismatch(
|
||
self,
|
||
table: str,
|
||
mismatch_keys: Set[Tuple],
|
||
window_start: datetime,
|
||
window_end: datetime,
|
||
) -> int:
|
||
"""批量更新不一致数据"""
|
||
# 对于维度表，使用 SCD2 逻辑
|
||
# 对于事实表，直接 UPSERT
|
||
return self.backfill_missing(table, mismatch_keys, window_start, window_end)
|
||
|
||
def _get_fact_column_map(self, table: str) -> Dict[str, Tuple[str, str | None]]:
|
||
"""获取事实表 DWD->ODS 列映射（用于 backfill）。"""
|
||
mapping_entries = DwdLoadTask.FACT_MAPPINGS.get(f"dwd.{table}") or []
|
||
result: Dict[str, Tuple[str, str | None]] = {}
|
||
for dwd_col, src, cast_type in mapping_entries:
|
||
if isinstance(src, str) and src.isidentifier():
|
||
result[dwd_col.lower()] = (src.lower(), cast_type)
|
||
return result
|
||
|
||
@staticmethod
|
||
def _coerce_bool(value: Any) -> bool | None:
|
||
if value is None:
|
||
return None
|
||
if isinstance(value, bool):
|
||
return value
|
||
if isinstance(value, (int, float)):
|
||
return bool(value)
|
||
if isinstance(value, str):
|
||
lowered = value.strip().lower()
|
||
if lowered in {"true", "1", "yes", "y", "t"}:
|
||
return True
|
||
if lowered in {"false", "0", "no", "n", "f"}:
|
||
return False
|
||
return bool(value)
|
||
|
||
@classmethod
|
||
def _adapt_fact_value(cls, value: Any, cast_type: str | None = None) -> Any:
|
||
"""适配事实表 UPSERT 值，处理 JSON 字段。"""
|
||
if cast_type == "boolean":
|
||
return cls._coerce_bool(value)
|
||
if isinstance(value, (dict, list)):
|
||
return Json(value, dumps=lambda v: json.dumps(v, ensure_ascii=False, default=str))
|
||
return value
|
||
|
||
def _load_to_dwd(self, table: str, records: List[dict], pk_cols: List[str]) -> int:
|
||
"""装载记录到 DWD 表"""
|
||
if not records:
|
||
return 0
|
||
|
||
is_dim = self.is_dimension_table(table)
|
||
|
||
if is_dim:
|
||
# 获取 ODS 主键列名和 ODS 到 DWD 的映射
|
||
ods_pk_cols = self.get_ods_pk_columns(table)
|
||
ods_to_dwd_map = self.get_ods_to_dwd_pk_map(table)
|
||
|
||
# 过滤掉 SCD2 列，只保留业务主键
|
||
# 因为 ODS 记录中没有 scd2_start_time 等字段
|
||
scd2_cols = {"scd2_start_time", "scd2_end_time", "scd2_is_current", "scd2_version"}
|
||
business_pk_cols = [c for c in pk_cols if c not in scd2_cols]
|
||
|
||
# DEBUG: 记录主键过滤情况
|
||
self.logger.debug(
|
||
"维度表 %s: 原始 pk_cols=%s, 过滤后 business_pk_cols=%s, ods_pk_cols=%s",
|
||
table, pk_cols, business_pk_cols, ods_pk_cols
|
||
)
|
||
|
||
if not business_pk_cols:
|
||
self.logger.warning(
|
||
"维度表 %s: 过滤 SCD2 列后业务主键为空，原始 pk_cols=%s",
|
||
table, pk_cols
|
||
)
|
||
return 0
|
||
|
||
return self._merge_dimension(table, records, business_pk_cols, ods_pk_cols, ods_to_dwd_map)
|
||
else:
|
||
return self._merge_fact(table, records, pk_cols)
|
||
|
||
def _merge_dimension(
|
||
self,
|
||
table: str,
|
||
records: List[dict],
|
||
pk_cols: List[str],
|
||
ods_pk_cols: List[str],
|
||
ods_to_dwd_map: Dict[str, str]
|
||
) -> int:
|
||
"""合并维度表（SCD2）
|
||
|
||
Args:
|
||
table: DWD 表名
|
||
records: ODS 记录列表
|
||
pk_cols: DWD 主键列名（排除 scd2_start_time）
|
||
ods_pk_cols: ODS 主键列名
|
||
ods_to_dwd_map: ODS 到 DWD 列名映射 {ods_col: dwd_col}
|
||
"""
|
||
# 获取 DWD 表列
|
||
sql = """
|
||
SELECT column_name
|
||
FROM information_schema.columns
|
||
WHERE table_schema = 'dwd'
|
||
AND table_name = %s
|
||
ORDER BY ordinal_position
|
||
"""
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(sql, (table,))
|
||
dwd_cols = [row[0] for row in cur.fetchall()]
|
||
except Exception as e:
|
||
self.logger.error("获取 DWD 表列失败: %s", e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
return 0
|
||
|
||
# 过滤出可映射的列
|
||
scd2_cols = {"scd2_start_time", "scd2_end_time", "scd2_is_current", "scd2_version"}
|
||
data_cols = [c for c in dwd_cols if c not in scd2_cols]
|
||
|
||
# 构建 ODS 到 DWD 列名映射（包含主键映射和其他同名列）
|
||
# 反向映射：dwd_col -> ods_col
|
||
dwd_to_ods_map = {v: k for k, v in ods_to_dwd_map.items()}
|
||
|
||
# 按业务主键去重，只保留最后一条记录
|
||
# 这避免了 ODS 中同一业务实体多次出现导致 SCD2 主键冲突
|
||
unique_records = {}
|
||
for record in records:
|
||
# 提取业务主键值
|
||
pk_values = []
|
||
skip = False
|
||
for dwd_pk_col in pk_cols:
|
||
ods_col = dwd_to_ods_map.get(dwd_pk_col, dwd_pk_col)
|
||
value = record.get(ods_col)
|
||
if value is None:
|
||
value = record.get(dwd_pk_col)
|
||
if value is None:
|
||
skip = True
|
||
break
|
||
pk_values.append(value)
|
||
if not skip:
|
||
pk_key = tuple(pk_values)
|
||
unique_records[pk_key] = record # 后面的覆盖前面的
|
||
|
||
self.logger.debug(
|
||
"维度表 %s: 原始记录数=%d, 去重后=%d",
|
||
table, len(records), len(unique_records)
|
||
)
|
||
|
||
count = 0
|
||
|
||
for pk_key, record in unique_records.items():
|
||
# pk_key 已经是去重时提取的主键元组
|
||
pk_values = pk_key
|
||
record_time = datetime.now(timezone.utc).replace(tzinfo=None)
|
||
|
||
# 1. 关闭旧版本
|
||
pk_where = " AND ".join(f"{c} = %s" for c in pk_cols)
|
||
update_sql = f"""
|
||
UPDATE dwd.{table}
|
||
SET scd2_is_current = 0, scd2_end_time = %s
|
||
WHERE {pk_where} AND scd2_is_current = 1
|
||
"""
|
||
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(update_sql, (record_time,) + pk_values)
|
||
except Exception as e:
|
||
self.logger.warning("关闭旧版本失败: %s", e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
continue
|
||
|
||
# 2. 准备插入数据（考虑列名映射）
|
||
insert_cols = []
|
||
values = []
|
||
|
||
for dwd_col in data_cols:
|
||
# 获取对应的 ODS 列名
|
||
ods_col = dwd_to_ods_map.get(dwd_col, dwd_col)
|
||
|
||
# 优先从 ODS 列名获取值，然后尝试 DWD 列名
|
||
if ods_col in record:
|
||
insert_cols.append(dwd_col)
|
||
values.append(record[ods_col])
|
||
elif dwd_col in record:
|
||
insert_cols.append(dwd_col)
|
||
values.append(record[dwd_col])
|
||
|
||
# 添加 SCD2 列
|
||
insert_cols.extend(["scd2_start_time", "scd2_end_time", "scd2_is_current", "scd2_version"])
|
||
values.extend([record_time, None, 1, 1])
|
||
|
||
col_list = ", ".join(insert_cols)
|
||
placeholders = ", ".join(["%s"] * len(values))
|
||
|
||
insert_sql = f"""
|
||
INSERT INTO dwd.{table} ({col_list})
|
||
VALUES ({placeholders})
|
||
"""
|
||
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(insert_sql, values)
|
||
count += 1
|
||
except Exception as e:
|
||
self.logger.warning("插入新版本失败: %s, error=%s", table, e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
|
||
try:
|
||
self.db.commit()
|
||
except Exception as e:
|
||
self.logger.error("提交事务失败: %s", e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
return count
|
||
|
||
def _merge_fact(self, table: str, records: List[dict], pk_cols: List[str]) -> int:
|
||
"""合并事实表（UPSERT）
|
||
|
||
注意：事实表的 backfill 有限制：
|
||
- ODS 记录列名与 DWD 列名可能不同
|
||
- 当前实现只处理主键映射，其他列需要名称相同
|
||
- 如果列名完全不匹配，会跳过 backfill
|
||
"""
|
||
if not records:
|
||
return 0
|
||
|
||
# 获取 ODS 到 DWD 主键映射
|
||
ods_to_dwd_map = self.get_ods_to_dwd_pk_map(table)
|
||
dwd_to_ods_pk_map = {v.lower(): k.lower() for k, v in ods_to_dwd_map.items()}
|
||
fact_col_map = self._get_fact_column_map(table)
|
||
|
||
# 获取 DWD 表列
|
||
sql = """
|
||
SELECT column_name
|
||
FROM information_schema.columns
|
||
WHERE table_schema = 'dwd'
|
||
AND table_name = %s
|
||
ORDER BY ordinal_position
|
||
"""
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
cur.execute(sql, (table,))
|
||
dwd_cols = [row[0] for row in cur.fetchall()]
|
||
except Exception as e:
|
||
self.logger.error("获取 DWD 表列失败: %s", e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
return 0
|
||
|
||
if not records:
|
||
return 0
|
||
|
||
# 统一字段名为小写，避免大小写影响匹配
|
||
records_lower = [{k.lower(): v for k, v in record.items()} for record in records]
|
||
sample_record = records_lower[0]
|
||
|
||
# 找出可映射的列（考虑列名映射）
|
||
mappable_cols = []
|
||
col_source_map = {} # dwd_col -> (source_key, cast_type)
|
||
|
||
for dwd_col in dwd_cols:
|
||
dwd_key = dwd_col.lower()
|
||
ods_col = fact_col_map.get(dwd_key)
|
||
if ods_col and ods_col[0] in sample_record:
|
||
# 优先使用事实表映射
|
||
mappable_cols.append(dwd_col)
|
||
col_source_map[dwd_col] = ods_col
|
||
continue
|
||
ods_col = dwd_to_ods_pk_map.get(dwd_key)
|
||
if ods_col and ods_col in sample_record:
|
||
# 有映射且 ODS 记录中有该列
|
||
mappable_cols.append(dwd_col)
|
||
col_source_map[dwd_col] = (ods_col, None)
|
||
elif dwd_key in sample_record:
|
||
# ODS 记录中有同名列
|
||
mappable_cols.append(dwd_col)
|
||
col_source_map[dwd_col] = (dwd_key, None)
|
||
|
||
if not mappable_cols:
|
||
self.logger.warning(
|
||
"事实表 %s: 无可映射列，跳过 backfill。ODS 列=%s, DWD 列=%s",
|
||
table, list(sample_record.keys())[:10], dwd_cols[:10]
|
||
)
|
||
return 0
|
||
|
||
# 确保主键列在可映射列中
|
||
for pk_col in pk_cols:
|
||
if pk_col not in mappable_cols:
|
||
pk_key = pk_col.lower()
|
||
ods_pk = fact_col_map.get(pk_key) or dwd_to_ods_pk_map.get(pk_key)
|
||
if ods_pk:
|
||
src_key = ods_pk[0] if isinstance(ods_pk, tuple) else ods_pk
|
||
else:
|
||
src_key = None
|
||
if src_key and src_key in sample_record:
|
||
mappable_cols.append(pk_col)
|
||
col_source_map[pk_col] = ods_pk if isinstance(ods_pk, tuple) else (src_key, None)
|
||
else:
|
||
self.logger.warning(
|
||
"事实表 %s: 主键列 %s 无法映射，跳过 backfill",
|
||
table, pk_col
|
||
)
|
||
return 0
|
||
|
||
# 按业务主键去重，避免批量 UPSERT 出现同主键重复
|
||
unique_records = {}
|
||
for record in records_lower:
|
||
pk_values = []
|
||
missing_pk = False
|
||
for pk_col in pk_cols:
|
||
src_key, _ = col_source_map[pk_col]
|
||
value = record.get(src_key)
|
||
if value is None:
|
||
missing_pk = True
|
||
break
|
||
pk_values.append(value)
|
||
if missing_pk:
|
||
continue
|
||
unique_records[tuple(pk_values)] = record
|
||
if len(unique_records) != len(records_lower):
|
||
self.logger.info(
|
||
"事实表 %s: 去重记录 %d -> %d",
|
||
table,
|
||
len(records_lower),
|
||
len(unique_records),
|
||
)
|
||
records_lower = list(unique_records.values())
|
||
|
||
col_list = ", ".join(mappable_cols)
|
||
pk_list = ", ".join(pk_cols)
|
||
|
||
update_cols = [c for c in mappable_cols if c not in pk_cols]
|
||
if update_cols:
|
||
update_set = ", ".join(f"{c} = EXCLUDED.{c}" for c in update_cols)
|
||
update_where = " OR ".join(
|
||
f"dwd.{table}.{c} IS DISTINCT FROM EXCLUDED.{c}"
|
||
for c in update_cols
|
||
)
|
||
upsert_sql = (
|
||
f"INSERT INTO dwd.{table} ({col_list}) "
|
||
f"VALUES ({', '.join(['%s'] * len(mappable_cols))}) "
|
||
f"ON CONFLICT ({pk_list}) DO UPDATE SET {update_set} "
|
||
f"WHERE {update_where}"
|
||
)
|
||
upsert_values_sql = (
|
||
f"INSERT INTO dwd.{table} ({col_list}) "
|
||
f"VALUES %s "
|
||
f"ON CONFLICT ({pk_list}) DO UPDATE SET {update_set} "
|
||
f"WHERE {update_where}"
|
||
)
|
||
else:
|
||
# 只有主键列，使用 DO NOTHING
|
||
upsert_sql = (
|
||
f"INSERT INTO dwd.{table} ({col_list}) "
|
||
f"VALUES ({', '.join(['%s'] * len(mappable_cols))}) "
|
||
f"ON CONFLICT ({pk_list}) DO NOTHING"
|
||
)
|
||
upsert_values_sql = (
|
||
f"INSERT INTO dwd.{table} ({col_list}) "
|
||
f"VALUES %s "
|
||
f"ON CONFLICT ({pk_list}) DO NOTHING"
|
||
)
|
||
|
||
all_values: List[List[Any]] = []
|
||
for record in records_lower:
|
||
row_values = []
|
||
for col in mappable_cols:
|
||
src_key, cast_type = col_source_map[col]
|
||
row_values.append(self._adapt_fact_value(record.get(src_key), cast_type))
|
||
all_values.append(row_values)
|
||
|
||
count = 0
|
||
# 可配置批量参数，降低锁等待与回退成本
|
||
batch_size = self._get_fact_upsert_batch_size()
|
||
min_batch_size = self._get_fact_upsert_min_batch_size()
|
||
if min_batch_size > batch_size:
|
||
min_batch_size = batch_size
|
||
max_retries = self._get_fact_upsert_max_retries()
|
||
backoff_sec = self._get_fact_upsert_backoff()
|
||
lock_timeout_ms = self._get_fact_upsert_lock_timeout_ms()
|
||
|
||
def _sleep_with_backoff(attempt: int):
|
||
if not backoff_sec:
|
||
return
|
||
idx = min(attempt, len(backoff_sec) - 1)
|
||
wait_sec = backoff_sec[idx]
|
||
if wait_sec > 0:
|
||
time.sleep(wait_sec)
|
||
|
||
def _iter_batches(items: List[List[Any]], size: int):
|
||
for idx in range(0, len(items), size):
|
||
yield items[idx:idx + size]
|
||
|
||
def _commit_batch():
|
||
"""批次级提交，缩短锁持有时间。"""
|
||
try:
|
||
self.db.commit()
|
||
except Exception as commit_error:
|
||
self.logger.error("提交事务失败: %s", commit_error)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
def _execute_batch(cur, batch_values: List[List[Any]]):
|
||
cur.execute("SAVEPOINT dwd_fact_batch_sp")
|
||
try:
|
||
execute_values(
|
||
cur,
|
||
upsert_values_sql,
|
||
batch_values,
|
||
page_size=len(batch_values),
|
||
)
|
||
cur.execute("RELEASE SAVEPOINT dwd_fact_batch_sp")
|
||
affected = int(cur.rowcount or 0)
|
||
if affected < 0:
|
||
affected = 0
|
||
return affected, None
|
||
except Exception as batch_error:
|
||
cur.execute("ROLLBACK TO SAVEPOINT dwd_fact_batch_sp")
|
||
cur.execute("RELEASE SAVEPOINT dwd_fact_batch_sp")
|
||
return 0, batch_error
|
||
|
||
def _fallback_rows(cur, batch_values: List[List[Any]]):
|
||
affected_total = 0
|
||
# 批量失败时退化到逐行，尽量跳过坏数据并继续处理
|
||
for values in batch_values:
|
||
cur.execute("SAVEPOINT dwd_fact_row_sp")
|
||
try:
|
||
cur.execute(upsert_sql, values)
|
||
cur.execute("RELEASE SAVEPOINT dwd_fact_row_sp")
|
||
affected = int(cur.rowcount or 0)
|
||
if affected < 0:
|
||
affected = 0
|
||
affected_total += affected
|
||
except Exception as row_error:
|
||
cur.execute("ROLLBACK TO SAVEPOINT dwd_fact_row_sp")
|
||
cur.execute("RELEASE SAVEPOINT dwd_fact_row_sp")
|
||
self.logger.warning(
|
||
"UPSERT 失败: %s, error=%s",
|
||
table,
|
||
row_error,
|
||
)
|
||
return affected_total
|
||
|
||
def _process_batch(cur, batch_values: List[List[Any]], current_size: int) -> int:
|
||
if not batch_values:
|
||
return 0
|
||
if len(batch_values) > current_size:
|
||
# 继续拆分为当前批次大小
|
||
total = 0
|
||
for sub_batch in _iter_batches(batch_values, current_size):
|
||
total += _process_batch(cur, sub_batch, current_size)
|
||
return total
|
||
|
||
for attempt in range(max_retries + 1):
|
||
affected, batch_error = _execute_batch(cur, batch_values)
|
||
if batch_error is None:
|
||
_commit_batch()
|
||
return affected
|
||
|
||
if self._is_lock_timeout_error(batch_error):
|
||
if current_size > min_batch_size:
|
||
new_size = max(min_batch_size, current_size // 2)
|
||
self.logger.warning(
|
||
"批量 UPSERT 锁超时，缩小批次: table=%s, %d -> %d",
|
||
table,
|
||
current_size,
|
||
new_size,
|
||
)
|
||
total = 0
|
||
for sub_batch in _iter_batches(batch_values, new_size):
|
||
total += _process_batch(cur, sub_batch, new_size)
|
||
return total
|
||
if attempt < max_retries:
|
||
self.logger.warning(
|
||
"批量 UPSERT 锁超时，重试: table=%s, attempt=%d/%d",
|
||
table,
|
||
attempt + 1,
|
||
max_retries,
|
||
)
|
||
_sleep_with_backoff(attempt)
|
||
continue
|
||
|
||
# 非锁超时或重试耗尽：回退逐行
|
||
self.logger.warning(
|
||
"批量 UPSERT 失败，回退逐行: table=%s, batch_size=%d, error=%s",
|
||
table,
|
||
len(batch_values),
|
||
batch_error,
|
||
)
|
||
affected_rows = _fallback_rows(cur, batch_values)
|
||
_commit_batch()
|
||
return affected_rows
|
||
|
||
return 0
|
||
|
||
try:
|
||
with self.db.conn.cursor() as cur:
|
||
if lock_timeout_ms is not None:
|
||
# 设置当前事务的锁等待上限，避免长时间阻塞
|
||
cur.execute("SET LOCAL lock_timeout = %s", (int(lock_timeout_ms),))
|
||
|
||
for batch_values in _iter_batches(all_values, batch_size):
|
||
count += _process_batch(cur, batch_values, batch_size)
|
||
except Exception as e:
|
||
self.logger.error("事实表 backfill 失败: %s", e)
|
||
try:
|
||
self.db.conn.rollback()
|
||
except Exception:
|
||
pass
|
||
|
||
return count
|
||
|
||
def _get_fact_upsert_batch_size(self) -> int:
|
||
"""读取事实表 UPSERT 批次大小（可配置）。"""
|
||
return self._get_int_config("dwd.fact_upsert_batch_size", 1000, 10, 5000)
|
||
|
||
def _get_fact_upsert_min_batch_size(self) -> int:
|
||
"""读取事实表 UPSERT 最小批次大小（可配置）。"""
|
||
return self._get_int_config("dwd.fact_upsert_min_batch_size", 100, 1, 2000)
|
||
|
||
def _get_fact_upsert_max_retries(self) -> int:
|
||
"""读取事实表 UPSERT 最大重试次数（可配置）。"""
|
||
return self._get_int_config("dwd.fact_upsert_max_retries", 2, 0, 10)
|
||
|
||
def _get_fact_upsert_lock_timeout_ms(self) -> Optional[int]:
|
||
"""读取事实表 UPSERT 锁等待超时（毫秒，可为空）。"""
|
||
if not self.config:
|
||
return None
|
||
value = self.config.get("dwd.fact_upsert_lock_timeout_ms")
|
||
try:
|
||
return int(value) if value is not None else None
|
||
except Exception:
|
||
return None
|
||
|
||
def _get_fact_upsert_backoff(self) -> List[int]:
|
||
"""读取事实表 UPSERT 重试退避（秒）。"""
|
||
if not self.config:
|
||
return [1, 2, 4]
|
||
value = self.config.get("dwd.fact_upsert_retry_backoff_sec", [1, 2, 4])
|
||
if not isinstance(value, list):
|
||
return [1, 2, 4]
|
||
return [int(v) for v in value if isinstance(v, (int, float)) and v >= 0]
|
||
|
||
def _get_int_config(self, key: str, default: int, min_value: int, max_value: int) -> int:
|
||
"""读取整数配置并裁剪到合理范围。"""
|
||
value = default
|
||
if self.config:
|
||
value = self.config.get(key, default)
|
||
try:
|
||
value = int(value)
|
||
except Exception:
|
||
value = default
|
||
value = max(min_value, min(value, max_value))
|
||
return value
|
||
|
||
@staticmethod
|
||
def _is_lock_timeout_error(error: Exception) -> bool:
|
||
"""判断是否为锁超时/锁冲突错误。"""
|
||
pgcode = getattr(error, "pgcode", None)
|
||
if pgcode in ("55P03", "57014"):
|
||
return True
|
||
message = str(error).lower()
|
||
return "lock timeout" in message or "锁超时" in message or "canceling statement due to lock timeout" in message
|