1169 lines
44 KiB
Python
1169 lines
44 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
ETL 全链路数据一致性黑盒检查
|
||
|
||
黑盒手段：获取最近一次成功的 ETL 任务，对 API→ODS→DWD→DWS/INDEX
|
||
逐表逐字段进行实际数据（JSON + DB）比对，输出详细差异报告。
|
||
|
||
用法:
|
||
cd C:\\NeoZQYY
|
||
uv run python scripts/ops/etl_consistency_check.py
|
||
|
||
输出: ETL_REPORT_ROOT / consistency_check_<timestamp>.md
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from datetime import datetime
|
||
from decimal import Decimal
|
||
from pathlib import Path
|
||
from zoneinfo import ZoneInfo
|
||
|
||
import psycopg2
|
||
import psycopg2.extras
|
||
from dotenv import load_dotenv
|
||
|
||
_ROOT = Path(__file__).resolve().parents[2]
|
||
load_dotenv(_ROOT / ".env", override=False)
|
||
|
||
# ── 环境变量（缺失即报错） ──
|
||
def _env(var: str) -> Path:
|
||
val = os.environ.get(var)
|
||
if not val:
|
||
raise KeyError(f"环境变量 {var} 未定义")
|
||
p = Path(val)
|
||
p.mkdir(parents=True, exist_ok=True)
|
||
return p
|
||
|
||
REPORT_ROOT = _env("ETL_REPORT_ROOT")
|
||
JSON_ROOT = _env("FETCH_ROOT")
|
||
LOG_ROOT = _env("LOG_ROOT")
|
||
|
||
PG_DSN = os.environ.get("TEST_DB_DSN")
|
||
if not PG_DSN:
|
||
raise RuntimeError("TEST_DB_DSN 未在 .env 中定义")
|
||
|
||
TZ = ZoneInfo("Asia/Shanghai")
|
||
NOW = datetime.now(TZ)
|
||
TS = NOW.strftime("%Y%m%d_%H%M%S")
|
||
|
||
# 采样条数上限
|
||
SAMPLE_LIMIT = 5
|
||
|
||
# 字段统计采样上限（用于 per-field 统计的表行数阈值，超过此值跳过 COUNT DISTINCT）
|
||
FIELD_STATS_DISTINCT_THRESHOLD = 3000
|
||
|
||
# ── ODS 任务名 → ODS 表名映射（与 blackbox_test_report.py 保持一致） ──
|
||
# CHANGE 2026-03-04 | 补充 ODS_STAFF_INFO，此前遗漏导致员工表不参与一致性检查
|
||
ODS_TASK_TO_TABLE = {
|
||
"ODS_ASSISTANT_ACCOUNT": "assistant_accounts_master",
|
||
"ODS_ASSISTANT_LEDGER": "assistant_service_records",
|
||
"ODS_SETTLEMENT_RECORDS": "settlement_records",
|
||
"ODS_TABLE_USE": "table_fee_transactions",
|
||
"ODS_TABLE_FEE_DISCOUNT": "table_fee_discount_records",
|
||
"ODS_TABLES": "site_tables_master",
|
||
"ODS_PAYMENT": "payment_transactions",
|
||
"ODS_REFUND": "refund_transactions",
|
||
"ODS_PLATFORM_COUPON": "platform_coupon_redemption_records",
|
||
"ODS_MEMBER": "member_profiles",
|
||
"ODS_MEMBER_CARD": "member_stored_value_cards",
|
||
"ODS_MEMBER_BALANCE": "member_balance_changes",
|
||
"ODS_RECHARGE_SETTLE": "recharge_settlements",
|
||
"ODS_GROUP_PACKAGE": "group_buy_packages",
|
||
"ODS_GROUP_BUY_REDEMPTION": "group_buy_redemption_records",
|
||
"ODS_INVENTORY_STOCK": "goods_stock_summary",
|
||
"ODS_INVENTORY_CHANGE": "goods_stock_movements",
|
||
"ODS_GOODS_CATEGORY": "stock_goods_category_tree",
|
||
"ODS_STORE_GOODS": "store_goods_master",
|
||
"ODS_STORE_GOODS_SALES": "store_goods_sales_records",
|
||
"ODS_TENANT_GOODS": "tenant_goods_master",
|
||
"ODS_STAFF_INFO": "staff_info_master",
|
||
}
|
||
|
||
# DWD 表 → ODS 表映射
|
||
# CHANGE 2026-03-04 | 补充 dim_staff、dim_site、
|
||
# dwd_goods_stock_movement、dwd_goods_stock_summary，此前遗漏
|
||
# CHANGE 2026-03-04 | 移除 dwd_assistant_trash_event（表已于 2026-02-22 DROP，禁止复活）
|
||
DWD_TO_ODS = {
|
||
"dwd.dim_assistant": "ods.assistant_accounts_master",
|
||
"dwd.dim_member": "ods.member_profiles",
|
||
"dwd.dim_member_card_account": "ods.member_stored_value_cards",
|
||
"dwd.dim_table": "ods.site_tables_master",
|
||
"dwd.dim_groupbuy_package": "ods.group_buy_packages",
|
||
"dwd.dim_store_goods": "ods.store_goods_master",
|
||
"dwd.dim_tenant_goods": "ods.tenant_goods_master",
|
||
"dwd.dim_goods_category": "ods.stock_goods_category_tree",
|
||
"dwd.dim_staff": "ods.staff_info_master",
|
||
"dwd.dim_site": "ods.site_tables_master",
|
||
"dwd.dwd_assistant_service_log": "ods.assistant_service_records",
|
||
"dwd.dwd_member_balance_change": "ods.member_balance_changes",
|
||
"dwd.dwd_recharge_order": "ods.recharge_settlements",
|
||
"dwd.dwd_settlement_head": "ods.settlement_records",
|
||
"dwd.dwd_table_fee_log": "ods.table_fee_transactions",
|
||
"dwd.dwd_table_fee_adjust": "ods.table_fee_discount_records",
|
||
"dwd.dwd_payment": "ods.payment_transactions",
|
||
"dwd.dwd_refund": "ods.refund_transactions",
|
||
"dwd.dwd_platform_coupon_redemption": "ods.platform_coupon_redemption_records",
|
||
"dwd.dwd_groupbuy_redemption": "ods.group_buy_redemption_records",
|
||
"dwd.dwd_store_goods_sale": "ods.store_goods_sales_records",
|
||
"dwd.dwd_goods_stock_movement": "ods.goods_stock_movements",
|
||
"dwd.dwd_goods_stock_summary": "ods.goods_stock_summary",
|
||
}
|
||
|
||
# ETL 元数据列（不参与值比对）
|
||
ETL_META_COLS = {"source_file", "source_endpoint", "fetched_at", "payload", "content_hash"}
|
||
# SCD2 管理列（不参与 ODS→DWD 值比对）
|
||
SCD2_COLS = {
|
||
"valid_from", "valid_to", "is_current", "etl_loaded_at", "etl_batch_id",
|
||
"scd2_start_time", "scd2_end_time", "scd2_is_current", "scd2_version",
|
||
}
|
||
|
||
|
||
# ── 数据库连接 ──
|
||
def get_conn():
|
||
conn = psycopg2.connect(PG_DSN, cursor_factory=psycopg2.extras.RealDictCursor)
|
||
conn.set_session(readonly=True)
|
||
return conn
|
||
|
||
|
||
# ── ETL 日志解析 ──
|
||
def find_latest_log() -> Path | None:
|
||
"""找到最新的 ETL 日志文件"""
|
||
logs = sorted(LOG_ROOT.glob("*.log"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||
return logs[0] if logs else None
|
||
|
||
|
||
def parse_etl_log(log_path: Path) -> dict:
|
||
"""解析 ETL 日志，提取任务执行结果"""
|
||
results = {}
|
||
task_start_times = {}
|
||
|
||
with open(log_path, "r", encoding="utf-8") as f:
|
||
for line in f:
|
||
m = re.match(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*开始执行(\w+) \(ODS\)", line)
|
||
if m:
|
||
task_start_times[m.group(2)] = m.group(1)
|
||
continue
|
||
|
||
m = re.match(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*?(\w+) ODS 任务完成: (\{.*\})", line)
|
||
if m:
|
||
task_name = m.group(2)
|
||
try:
|
||
counts = eval(m.group(3))
|
||
except Exception:
|
||
counts = {}
|
||
results[task_name] = {
|
||
"status": "SUCC", "layer": "ODS",
|
||
"start": task_start_times.get(task_name, ""),
|
||
"end": m.group(1), "counts": counts,
|
||
}
|
||
continue
|
||
|
||
m = re.match(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*DWD_LOAD_FROM_ODS.*完成.*?(\d+).*表", line)
|
||
if m:
|
||
results["DWD_LOAD_FROM_ODS"] = {"status": "SUCC", "layer": "DWD", "end": m.group(1)}
|
||
continue
|
||
|
||
m = re.match(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*?(\w+): 工具类任务执行成功", line)
|
||
if m:
|
||
results[m.group(2)] = {"status": "SUCC", "layer": "DWS/INDEX", "end": m.group(1)}
|
||
continue
|
||
|
||
m = re.match(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*?(\w+): 工具类任务执行失败: (.*)", line)
|
||
if m:
|
||
results[m.group(2)] = {
|
||
"status": "FAIL", "layer": "DWS/INDEX",
|
||
"end": m.group(1), "error": m.group(3)[:120],
|
||
}
|
||
continue
|
||
|
||
m = re.match(r".*任务 (\w+) 未启用或不存在", line)
|
||
if m:
|
||
results[m.group(1)] = {"status": "SKIP", "layer": "DWS", "error": "未注册"}
|
||
|
||
return results
|
||
|
||
|
||
# ── API JSON 读取 ──
|
||
def load_api_json_records(task_name: str) -> list[dict] | None:
|
||
"""读取某个 ODS 任务最新一次 JSON 落盘的全部记录。
|
||
|
||
JSON 结构为 ETL 框架包装格式：
|
||
{task_code, run_id, pages: [{page, request, response: {data: {<list_field>: [...]}}}]}
|
||
数据列表字段名因端点而异（如 tenantMemberInfos、list 等），
|
||
取 response.data 下第一个 list 类型的值。
|
||
"""
|
||
task_dir = JSON_ROOT / task_name
|
||
if not task_dir.exists():
|
||
return None
|
||
subdirs = sorted(task_dir.iterdir(), key=lambda p: p.name, reverse=True)
|
||
for sd in subdirs:
|
||
if not sd.is_dir():
|
||
continue
|
||
records = []
|
||
for jf in sorted(sd.glob("*.json")):
|
||
try:
|
||
data = json.loads(jf.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
|
||
# ETL 框架包装格式：pages[].response.data.<list_field>
|
||
if isinstance(data, dict) and "pages" in data:
|
||
for page in data["pages"]:
|
||
resp = page.get("response", {})
|
||
if not isinstance(resp, dict):
|
||
continue
|
||
resp_data = resp.get("data", {})
|
||
if isinstance(resp_data, dict):
|
||
# 找 response.data 下第一个 list 类型的值
|
||
for v in resp_data.values():
|
||
if isinstance(v, list):
|
||
records.extend(v)
|
||
break
|
||
elif isinstance(resp_data, list):
|
||
records.extend(resp_data)
|
||
# 兼容旧格式：直接是 list 或 {data: {list: [...]}}
|
||
elif isinstance(data, list):
|
||
records.extend(data)
|
||
elif isinstance(data, dict):
|
||
lst = data.get("data", {}).get("list", data.get("data", []))
|
||
if isinstance(lst, list):
|
||
records.extend(lst)
|
||
|
||
if records:
|
||
return records
|
||
return None
|
||
|
||
|
||
# ── 表结构查询 ──
|
||
def get_table_columns(conn, schema: str, table: str) -> list[str]:
|
||
"""获取表的列名列表"""
|
||
with conn.cursor() as cur:
|
||
cur.execute("""
|
||
SELECT column_name FROM information_schema.columns
|
||
WHERE table_schema = %s AND table_name = %s
|
||
ORDER BY ordinal_position
|
||
""", (schema, table))
|
||
return [r["column_name"] for r in cur.fetchall()]
|
||
|
||
|
||
def get_table_row_count(conn, schema: str, table: str) -> int:
|
||
"""获取表行数"""
|
||
with conn.cursor() as cur:
|
||
try:
|
||
cur.execute(f"SELECT COUNT(*) AS cnt FROM {schema}.{table}")
|
||
return cur.fetchone()["cnt"]
|
||
except Exception:
|
||
conn.rollback()
|
||
return -1
|
||
|
||
|
||
def sample_rows(conn, schema: str, table: str, limit: int = SAMPLE_LIMIT) -> list[dict]:
|
||
"""随机采样若干行"""
|
||
with conn.cursor() as cur:
|
||
try:
|
||
cur.execute(f"""
|
||
SELECT * FROM {schema}.{table}
|
||
ORDER BY random() LIMIT {limit}
|
||
""")
|
||
rows = cur.fetchall()
|
||
# 转换为可序列化的 dict
|
||
result = []
|
||
for row in rows:
|
||
d = {}
|
||
for k, v in dict(row).items():
|
||
if isinstance(v, Decimal):
|
||
d[k] = float(v)
|
||
elif isinstance(v, datetime):
|
||
d[k] = str(v)
|
||
elif isinstance(v, (dict, list)):
|
||
d[k] = v # JSON 类型保持原样
|
||
else:
|
||
d[k] = v
|
||
result.append(d)
|
||
return result
|
||
except Exception:
|
||
conn.rollback()
|
||
return []
|
||
|
||
|
||
# ── 字段级统计（类似 field_level_report 的 per-field 统计） ──
|
||
def get_field_stats(conn, schema: str, table: str) -> list[dict]:
|
||
"""批量获取表中每个字段的统计信息（NULL率、数值 min/max/avg、唯一值等）"""
|
||
cols_meta = []
|
||
with conn.cursor() as cur:
|
||
cur.execute("""
|
||
SELECT column_name, udt_name
|
||
FROM information_schema.columns
|
||
WHERE table_schema = %s AND table_name = %s
|
||
ORDER BY ordinal_position
|
||
""", (schema, table))
|
||
cols_meta = [(r["column_name"], r["udt_name"]) for r in cur.fetchall()]
|
||
|
||
if not cols_meta:
|
||
return []
|
||
|
||
row_count = get_table_row_count(conn, schema, table)
|
||
if row_count <= 0:
|
||
return [{"column": c, "type": t, "total": row_count, "null_pct": "—"} for c, t in cols_meta]
|
||
|
||
# 构造批量聚合 SQL（与 field_level_report.py 类似的策略）
|
||
select_parts = [f"{row_count} AS _total"]
|
||
skip_stats = {"payload", "content_hash", "record_index", "source_file", "source_endpoint"}
|
||
col_plan = []
|
||
|
||
for cname, udt in cols_meta:
|
||
safe = f'"{cname}"'
|
||
alias = cname.replace(" ", "_").replace("-", "_")
|
||
plan = {"column": cname, "type": udt, "stats": []}
|
||
|
||
select_parts.append(f"COUNT(*) FILTER (WHERE {safe} IS NULL) AS null_{alias}")
|
||
plan["stats"].append("null")
|
||
|
||
if udt in ("jsonb", "json", "bytea") or cname in skip_stats:
|
||
col_plan.append(plan)
|
||
continue
|
||
|
||
if udt in ("int2", "int4", "int8", "float4", "float8", "numeric"):
|
||
select_parts.append(f"MIN({safe}) AS min_{alias}")
|
||
select_parts.append(f"MAX({safe}) AS max_{alias}")
|
||
select_parts.append(f"ROUND(AVG({safe})::numeric, 2) AS avg_{alias}")
|
||
plan["stats"].extend(["min", "max", "avg"])
|
||
elif udt in ("date", "timestamp", "timestamptz"):
|
||
select_parts.append(f"MIN({safe}::text) FILTER (WHERE {safe}::text >= '0001') AS min_{alias}")
|
||
select_parts.append(f"MAX({safe}::text) FILTER (WHERE {safe}::text <= '9999') AS max_{alias}")
|
||
plan["stats"].extend(["earliest", "latest"])
|
||
elif udt in ("text", "varchar", "bpchar", "name"):
|
||
select_parts.append(f"MIN(LENGTH({safe})) AS minlen_{alias}")
|
||
select_parts.append(f"MAX(LENGTH({safe})) AS maxlen_{alias}")
|
||
plan["stats"].extend(["min_len", "max_len"])
|
||
elif udt == "bool":
|
||
select_parts.append(f"COUNT(*) FILTER (WHERE {safe} = TRUE) AS true_{alias}")
|
||
select_parts.append(f"COUNT(*) FILTER (WHERE {safe} = FALSE) AS false_{alias}")
|
||
plan["stats"].extend(["true_count", "false_count"])
|
||
|
||
col_plan.append(plan)
|
||
|
||
with conn.cursor() as cur:
|
||
try:
|
||
cur.execute(f"SELECT {', '.join(select_parts)} FROM {schema}.{table}")
|
||
agg = cur.fetchone()
|
||
except Exception:
|
||
conn.rollback()
|
||
return [{"column": c, "type": t, "total": row_count, "null_pct": "?", "error": True}
|
||
for c, t in cols_meta]
|
||
|
||
results = []
|
||
for plan in col_plan:
|
||
cname = plan["column"]
|
||
udt = plan["type"]
|
||
alias = cname.replace(" ", "_").replace("-", "_")
|
||
null_cnt = agg.get(f"null_{alias}", 0) or 0
|
||
null_pct = round(null_cnt / row_count * 100, 1) if row_count > 0 else 0
|
||
|
||
r = {"column": cname, "type": udt, "total": row_count,
|
||
"null_count": null_cnt, "null_pct": f"{null_pct}%"}
|
||
|
||
if udt in ("jsonb", "json", "bytea") or cname in skip_stats:
|
||
r["note"] = f"({udt.upper()})" if udt in ("jsonb", "json", "bytea") else "(ETL元数据)"
|
||
results.append(r)
|
||
continue
|
||
|
||
if "min" in plan["stats"]:
|
||
r["min"] = agg.get(f"min_{alias}")
|
||
r["max"] = agg.get(f"max_{alias}")
|
||
r["avg"] = agg.get(f"avg_{alias}")
|
||
if "earliest" in plan["stats"]:
|
||
r["earliest"] = str(agg.get(f"min_{alias}") or "")
|
||
r["latest"] = str(agg.get(f"max_{alias}") or "")
|
||
if "min_len" in plan["stats"]:
|
||
r["min_len"] = agg.get(f"minlen_{alias}")
|
||
r["max_len"] = agg.get(f"maxlen_{alias}")
|
||
if "true_count" in plan["stats"]:
|
||
r["true_count"] = agg.get(f"true_{alias}")
|
||
r["false_count"] = agg.get(f"false_{alias}")
|
||
|
||
results.append(r)
|
||
|
||
# 小表补充 distinct 计数
|
||
if row_count <= FIELD_STATS_DISTINCT_THRESHOLD:
|
||
for r in results:
|
||
cname = r["column"]
|
||
if r.get("note"):
|
||
r["distinct"] = "-"
|
||
continue
|
||
with conn.cursor() as cur:
|
||
try:
|
||
cur.execute(f'SELECT COUNT(DISTINCT "{cname}") AS d FROM {schema}.{table}')
|
||
r["distinct"] = cur.fetchone()["d"]
|
||
except Exception:
|
||
conn.rollback()
|
||
r["distinct"] = "?"
|
||
else:
|
||
for r in results:
|
||
r["distinct"] = "-"
|
||
|
||
return results
|
||
|
||
|
||
# ── 数据截止日期查询 ──
|
||
# 每张 ODS 表的截止日期字段映射：大部分用 create_time/createtime，两张维表用 fetched_at
|
||
_CUTOFF_DATE_COLUMN: dict[str, str] = {
|
||
"goods_stock_summary": "fetched_at",
|
||
"stock_goods_category_tree": "fetched_at",
|
||
# 以下表使用 createtime（无下划线）
|
||
"goods_stock_movements": "createtime",
|
||
"settlement_records": "createtime",
|
||
"recharge_settlements": "createtime",
|
||
}
|
||
|
||
|
||
def get_data_cutoff_date(conn, schema: str, table: str) -> str | None:
|
||
"""查询表中数据的最后截止日期（MAX of 时间字段）"""
|
||
col = _CUTOFF_DATE_COLUMN.get(table, "create_time")
|
||
with conn.cursor() as cur:
|
||
try:
|
||
cur.execute(
|
||
f'SELECT MAX("{col}")::date::text AS cutoff FROM {schema}.{table}'
|
||
)
|
||
row = cur.fetchone()
|
||
return row["cutoff"] if row and row["cutoff"] else None
|
||
except Exception:
|
||
conn.rollback()
|
||
return None
|
||
|
||
|
||
# ── API vs ODS 字段级比对 ──
|
||
def check_api_vs_ods(conn, task_name: str, ods_table: str) -> dict:
|
||
"""比对 API JSON 字段与 ODS 表列，并采样值比对"""
|
||
result = {
|
||
"task": task_name, "ods_table": ods_table,
|
||
"api_records": 0, "ods_rows": 0, "ods_distinct_ids": 0,
|
||
"field_check": {"api_fields": 0, "ods_cols": 0, "matched": 0,
|
||
"api_only": [], "ods_only": []},
|
||
"value_diffs": [], "status": "✅",
|
||
}
|
||
|
||
# 读取 API JSON
|
||
api_records = load_api_json_records(task_name)
|
||
if not api_records:
|
||
result["status"] = "⚠️ 无 API JSON"
|
||
return result
|
||
result["api_records"] = len(api_records)
|
||
|
||
# ODS 行数
|
||
ods_cols = get_table_columns(conn, "ods", ods_table)
|
||
result["ods_rows"] = get_table_row_count(conn, "ods", ods_table)
|
||
with conn.cursor() as cur:
|
||
try:
|
||
cur.execute(f"SELECT COUNT(DISTINCT id) AS cnt FROM ods.{ods_table}")
|
||
result["ods_distinct_ids"] = cur.fetchone()["cnt"]
|
||
except Exception:
|
||
conn.rollback()
|
||
result["ods_distinct_ids"] = -1
|
||
|
||
# 字段比对：API JSON 顶层 key vs ODS 列
|
||
api_fields = set()
|
||
for rec in api_records[:50]: # 取前 50 条的 key 并集
|
||
if isinstance(rec, dict):
|
||
api_fields.update(rec.keys())
|
||
|
||
ods_col_set = set(ods_cols) - ETL_META_COLS
|
||
api_lower = {f.lower(): f for f in api_fields}
|
||
ods_lower = {c.lower(): c for c in ods_col_set}
|
||
|
||
matched = set(api_lower.keys()) & set(ods_lower.keys())
|
||
api_only = sorted(set(api_lower.keys()) - set(ods_lower.keys()))
|
||
ods_only = sorted(set(ods_lower.keys()) - set(api_lower.keys()))
|
||
|
||
result["field_check"]["api_fields"] = len(api_fields)
|
||
result["field_check"]["ods_cols"] = len(ods_col_set)
|
||
result["field_check"]["matched"] = len(matched)
|
||
result["field_check"]["api_only"] = [api_lower[k] for k in api_only]
|
||
result["field_check"]["ods_only"] = [ods_lower[k] for k in ods_only]
|
||
|
||
# 值采样比对：取 API 中前 N 条记录的 id，查 ODS 对应行比较
|
||
value_diffs = []
|
||
sample_ids = []
|
||
for rec in api_records[:SAMPLE_LIMIT]:
|
||
if isinstance(rec, dict) and "id" in rec:
|
||
sample_ids.append(rec["id"])
|
||
|
||
if sample_ids and matched:
|
||
compare_cols = sorted(matched)[:10] # 最多比 10 个字段
|
||
with conn.cursor() as cur:
|
||
for sid in sample_ids:
|
||
try:
|
||
cur.execute(
|
||
f"SELECT * FROM ods.{ods_table} WHERE id = %s ORDER BY fetched_at DESC LIMIT 1",
|
||
(sid,)
|
||
)
|
||
ods_row = cur.fetchone()
|
||
except Exception:
|
||
conn.rollback()
|
||
ods_row = None
|
||
|
||
if not ods_row:
|
||
continue
|
||
|
||
api_rec = next((r for r in api_records if r.get("id") == sid), None)
|
||
if not api_rec:
|
||
continue
|
||
|
||
for col_lower in compare_cols:
|
||
api_key = api_lower[col_lower]
|
||
ods_key = ods_lower[col_lower]
|
||
api_val = api_rec.get(api_key)
|
||
ods_val = ods_row.get(ods_key)
|
||
|
||
is_diff, reason = _values_differ(api_val, ods_val)
|
||
if is_diff or reason == "whitelist":
|
||
value_diffs.append({
|
||
"id": sid, "field": col_lower,
|
||
"api_val": _fmt_val(api_val),
|
||
"ods_val": _fmt_val(ods_val),
|
||
"whitelist": reason == "whitelist",
|
||
})
|
||
|
||
result["value_diffs"] = value_diffs[:20] # 最多报 20 条差异
|
||
# 只有真正差异（非白名单）才标记为异常
|
||
real_diffs = [d for d in value_diffs if not d.get("whitelist")]
|
||
if api_only or ods_only or real_diffs:
|
||
result["status"] = "❌ 存在差异"
|
||
|
||
# 字段级统计（ODS 表）
|
||
result["ods_field_stats"] = get_field_stats(conn, "ods", ods_table)
|
||
|
||
# 数据截止日期
|
||
result["data_cutoff"] = get_data_cutoff_date(conn, "ods", ods_table)
|
||
|
||
return result
|
||
|
||
|
||
def _values_differ(api_val, ods_val) -> tuple[bool, str]:
|
||
"""比较两个值是否实质不同（容忍类型差异）。
|
||
|
||
返回 (is_different, reason):
|
||
- (False, "") — 值相同
|
||
- (False, "whitelist") — API 空字符串 vs DB None，视为等价（白名单）
|
||
- (True, "") — 值确实不同
|
||
注意：0 与 None 绝不相等！
|
||
"""
|
||
if api_val is None and ods_val is None:
|
||
return False, ""
|
||
|
||
# API 空字符串 "" vs DB None → 白名单（等价但标记）
|
||
if api_val is not None and ods_val is None:
|
||
if isinstance(api_val, str) and api_val.strip() == "":
|
||
return False, "whitelist"
|
||
return True, ""
|
||
if api_val is None and ods_val is not None:
|
||
if isinstance(ods_val, str) and str(ods_val).strip() == "":
|
||
return False, "whitelist"
|
||
return True, ""
|
||
|
||
# 都转字符串比较（容忍 int vs str、Decimal vs float 等）
|
||
a = str(api_val).strip()
|
||
b = str(ods_val).strip()
|
||
if a == b:
|
||
return False, ""
|
||
# 数值比较
|
||
try:
|
||
if float(a) == float(b):
|
||
return False, ""
|
||
except (ValueError, TypeError):
|
||
pass
|
||
return True, ""
|
||
|
||
|
||
def _fmt_val(val) -> str:
|
||
"""完整展示差异值，不截断"""
|
||
return str(val)
|
||
|
||
|
||
# ── ODS vs DWD 字段级比对 ──
|
||
def _find_ex_table(conn, dwd_schema: str, dwd_table: str) -> str | None:
|
||
"""查找 DWD 主表对应的 EX 表（如 dim_assistant → dim_assistant_ex）"""
|
||
ex_name = dwd_table + "_ex"
|
||
with conn.cursor() as cur:
|
||
cur.execute("""
|
||
SELECT table_name FROM information_schema.tables
|
||
WHERE table_schema = %s AND table_name = %s AND table_type = 'BASE TABLE'
|
||
""", (dwd_schema, ex_name))
|
||
row = cur.fetchone()
|
||
return ex_name if row else None
|
||
|
||
|
||
def _get_dwd_pk(conn, dwd_schema: str, dwd_table: str) -> str | None:
|
||
"""获取 DWD 主表的第一列（即 PK 列名）"""
|
||
with conn.cursor() as cur:
|
||
cur.execute("""
|
||
SELECT column_name FROM information_schema.columns
|
||
WHERE table_schema = %s AND table_name = %s
|
||
ORDER BY ordinal_position LIMIT 1
|
||
""", (dwd_schema, dwd_table))
|
||
row = cur.fetchone()
|
||
return row["column_name"] if row else None
|
||
|
||
|
||
def check_ods_vs_dwd(conn, dwd_full: str, ods_full: str) -> dict:
|
||
"""比对 ODS 与 DWD（主表 + EX 表合并）的行数和字段值"""
|
||
dwd_s, dwd_t = dwd_full.split(".")
|
||
ods_s, ods_t = ods_full.split(".")
|
||
is_dim = dwd_t.startswith("dim_")
|
||
|
||
result = {
|
||
"dwd_table": dwd_full, "ods_table": ods_full,
|
||
"type": "维度(SCD2)" if is_dim else "事实",
|
||
"ods_rows": 0, "ods_distinct_ids": 0, "dwd_rows": 0,
|
||
"dwd_cols": [], "ods_cols": [],
|
||
"common_cols": [], "dwd_only_cols": [], "ods_only_cols": [],
|
||
"value_diffs": [], "status": "✅",
|
||
"ex_table": None,
|
||
}
|
||
|
||
# 行数
|
||
result["ods_rows"] = get_table_row_count(conn, ods_s, ods_t)
|
||
result["dwd_rows"] = get_table_row_count(conn, dwd_s, dwd_t)
|
||
with conn.cursor() as cur:
|
||
try:
|
||
cur.execute(f"SELECT COUNT(DISTINCT id) AS cnt FROM {ods_s}.{ods_t}")
|
||
result["ods_distinct_ids"] = cur.fetchone()["cnt"]
|
||
except Exception:
|
||
conn.rollback()
|
||
|
||
# DWD PK 列名（第一列）
|
||
dwd_pk = _get_dwd_pk(conn, dwd_s, dwd_t)
|
||
if not dwd_pk:
|
||
return result
|
||
|
||
# 查找 EX 表
|
||
ex_table = _find_ex_table(conn, dwd_s, dwd_t)
|
||
result["ex_table"] = f"{dwd_s}.{ex_table}" if ex_table else None
|
||
|
||
# 列比对：合并主表 + EX 表的列集合
|
||
dwd_main_cols = set(get_table_columns(conn, dwd_s, dwd_t))
|
||
dwd_ex_cols = set()
|
||
if ex_table:
|
||
dwd_ex_cols = set(get_table_columns(conn, dwd_s, ex_table))
|
||
# 合并后去掉 SCD2 管理列和 PK 重复列
|
||
dwd_all_cols = dwd_main_cols | dwd_ex_cols
|
||
dwd_biz_cols = dwd_all_cols - SCD2_COLS
|
||
|
||
ods_cols = set(get_table_columns(conn, ods_s, ods_t)) - ETL_META_COLS
|
||
|
||
result["dwd_cols"] = sorted(dwd_all_cols)
|
||
result["ods_cols"] = sorted(ods_cols)
|
||
|
||
# 找共同列（忽略大小写）
|
||
dwd_lower = {c.lower(): c for c in dwd_biz_cols}
|
||
ods_lower = {c.lower(): c for c in ods_cols}
|
||
common = set(dwd_lower.keys()) & set(ods_lower.keys())
|
||
dwd_only = sorted(set(dwd_lower.keys()) - set(ods_lower.keys()))
|
||
ods_only = sorted(set(ods_lower.keys()) - set(dwd_lower.keys()))
|
||
|
||
result["common_cols"] = sorted(common)
|
||
result["dwd_only_cols"] = [dwd_lower[k] for k in dwd_only]
|
||
result["ods_only_cols"] = [ods_lower[k] for k in ods_only]
|
||
|
||
if not common:
|
||
return result
|
||
|
||
# 确定每个共同列来自主表还是 EX 表
|
||
main_lower = {c.lower(): c for c in dwd_main_cols}
|
||
ex_lower = {c.lower(): c for c in dwd_ex_cols} if ex_table else {}
|
||
|
||
compare_cols = sorted(common)[:15] # 最多比 15 个字段
|
||
value_diffs = []
|
||
|
||
with conn.cursor() as cur:
|
||
# 取 DWD 采样行（主表）
|
||
try:
|
||
if is_dim:
|
||
cur.execute(f"""
|
||
SELECT * FROM {dwd_s}.{dwd_t}
|
||
WHERE scd2_is_current = true
|
||
ORDER BY random() LIMIT {SAMPLE_LIMIT}
|
||
""")
|
||
else:
|
||
cur.execute(f"""
|
||
SELECT * FROM {dwd_s}.{dwd_t}
|
||
ORDER BY random() LIMIT {SAMPLE_LIMIT}
|
||
""")
|
||
dwd_samples = cur.fetchall()
|
||
except Exception:
|
||
conn.rollback()
|
||
dwd_samples = []
|
||
|
||
for dwd_row in dwd_samples:
|
||
pk_val = dwd_row.get(dwd_pk)
|
||
if pk_val is None:
|
||
continue
|
||
|
||
# 如果有 EX 表，查 EX 行并合并
|
||
dwd_merged = dict(dwd_row)
|
||
if ex_table:
|
||
try:
|
||
cur.execute(
|
||
f'SELECT * FROM {dwd_s}.{ex_table} WHERE "{dwd_pk}" = %s LIMIT 1',
|
||
(pk_val,)
|
||
)
|
||
ex_row = cur.fetchone()
|
||
if ex_row:
|
||
dwd_merged.update(dict(ex_row))
|
||
except Exception:
|
||
conn.rollback()
|
||
|
||
# 查 ODS 对应行（最新快照，ODS 用 id 列）
|
||
try:
|
||
cur.execute(
|
||
f"SELECT * FROM {ods_s}.{ods_t} WHERE id = %s ORDER BY fetched_at DESC LIMIT 1",
|
||
(pk_val,)
|
||
)
|
||
ods_row = cur.fetchone()
|
||
except Exception:
|
||
conn.rollback()
|
||
ods_row = None
|
||
|
||
if not ods_row:
|
||
value_diffs.append({
|
||
"id": pk_val, "field": "(全行)",
|
||
"dwd_val": "存在", "ods_val": "ODS 中未找到",
|
||
"whitelist": False,
|
||
})
|
||
continue
|
||
|
||
for col_lower in compare_cols:
|
||
dwd_key = dwd_lower.get(col_lower) or main_lower.get(col_lower) or ex_lower.get(col_lower)
|
||
ods_key = ods_lower[col_lower]
|
||
dwd_val = dwd_merged.get(dwd_key)
|
||
ods_val = ods_row.get(ods_key)
|
||
|
||
is_diff, reason = _values_differ(dwd_val, ods_val)
|
||
if is_diff or reason == "whitelist":
|
||
value_diffs.append({
|
||
"id": pk_val, "field": col_lower,
|
||
"dwd_val": _fmt_val(dwd_val),
|
||
"ods_val": _fmt_val(ods_val),
|
||
"whitelist": reason == "whitelist",
|
||
})
|
||
|
||
result["value_diffs"] = value_diffs[:30]
|
||
real_diffs = [d for d in value_diffs if not d.get("whitelist")]
|
||
if dwd_only or real_diffs:
|
||
result["status"] = "❌ 存在差异"
|
||
|
||
# 字段级统计（DWD 主表）
|
||
result["dwd_field_stats"] = get_field_stats(conn, dwd_s, dwd_t)
|
||
|
||
# 数据截止日期（从 ODS 源表查询）
|
||
result["data_cutoff"] = get_data_cutoff_date(conn, ods_s, ods_t)
|
||
|
||
return result
|
||
|
||
|
||
# ── DWD vs DWS 聚合逻辑验证 ──
|
||
def check_dwd_vs_dws(conn) -> list[dict]:
|
||
"""检查 DWS 表的数据是否与 DWD 源表一致（行数 + 关键指标抽查）"""
|
||
results = []
|
||
|
||
# 获取 DWS 层所有表
|
||
with conn.cursor() as cur:
|
||
cur.execute("""
|
||
SELECT table_name FROM information_schema.tables
|
||
WHERE table_schema = 'dws' AND table_type = 'BASE TABLE'
|
||
ORDER BY table_name
|
||
""")
|
||
dws_tables = [r["table_name"] for r in cur.fetchall()]
|
||
|
||
for dws_t in dws_tables:
|
||
row_count = get_table_row_count(conn, "dws", dws_t)
|
||
cols = get_table_columns(conn, "dws", dws_t)
|
||
|
||
entry = {
|
||
"table": f"dws.{dws_t}",
|
||
"rows": row_count,
|
||
"col_count": len(cols),
|
||
"status": "✅" if row_count > 0 else "⚠️ 空表",
|
||
"sample_checks": [],
|
||
}
|
||
|
||
if row_count > 0:
|
||
# 对数值列做基本健全性检查（负值、NULL 率）
|
||
numeric_cols = []
|
||
with conn.cursor() as cur:
|
||
cur.execute("""
|
||
SELECT column_name, udt_name FROM information_schema.columns
|
||
WHERE table_schema = 'dws' AND table_name = %s
|
||
AND udt_name IN ('int2','int4','int8','float4','float8','numeric')
|
||
ORDER BY ordinal_position
|
||
""", (dws_t,))
|
||
numeric_cols = [(r["column_name"], r["udt_name"]) for r in cur.fetchall()]
|
||
|
||
for col_name, col_type in numeric_cols[:8]: # 最多检查 8 个数值列
|
||
with conn.cursor() as cur:
|
||
try:
|
||
cur.execute(f"""
|
||
SELECT
|
||
COUNT(*) FILTER (WHERE "{col_name}" IS NULL) AS null_cnt,
|
||
COUNT(*) FILTER (WHERE "{col_name}" < 0) AS neg_cnt,
|
||
MIN("{col_name}") AS min_val,
|
||
MAX("{col_name}") AS max_val,
|
||
COUNT(*) AS total
|
||
FROM dws.{dws_t}
|
||
""")
|
||
stats = cur.fetchone()
|
||
total = stats["total"]
|
||
null_pct = round(stats["null_cnt"] / total * 100, 1) if total > 0 else 0
|
||
check = {
|
||
"col": col_name,
|
||
"null_pct": f"{null_pct}%",
|
||
"neg_count": stats["neg_cnt"],
|
||
"min": str(stats["min_val"]),
|
||
"max": str(stats["max_val"]),
|
||
"status": "✅",
|
||
}
|
||
# 金额列出现负值可能异常
|
||
if stats["neg_cnt"] > 0 and "amount" in col_name.lower():
|
||
check["status"] = "⚠️ 金额负值"
|
||
if null_pct > 50:
|
||
check["status"] = "⚠️ 高 NULL 率"
|
||
entry["sample_checks"].append(check)
|
||
except Exception:
|
||
conn.rollback()
|
||
|
||
results.append(entry)
|
||
|
||
return results
|
||
|
||
|
||
# ── 报告生成 ──
|
||
def generate_report(
|
||
log_path: Path,
|
||
log_results: dict,
|
||
api_ods_checks: list[dict],
|
||
ods_dwd_checks: list[dict],
|
||
dws_checks: list[dict],
|
||
) -> str:
|
||
"""生成 Markdown 报告"""
|
||
lines: list[str] = []
|
||
|
||
def w(s: str = ""):
|
||
lines.append(s)
|
||
|
||
w("# ETL 全链路数据一致性检查报告")
|
||
w()
|
||
w(f"生成时间: {NOW.strftime('%Y-%m-%d %H:%M:%S')} CST")
|
||
w(f"ETL 日志: `{log_path.name}`")
|
||
w()
|
||
|
||
def _fmt_field_stat(fs: dict) -> str:
|
||
"""格式化单个字段统计为表格行"""
|
||
col = fs.get("column", "?")
|
||
typ = fs.get("type", "?")
|
||
null_pct = fs.get("null_pct", "—")
|
||
distinct = fs.get("distinct", "-")
|
||
parts = []
|
||
if "min" in fs and fs["min"] is not None:
|
||
parts.append(f"min={fs['min']}, max={fs['max']}, avg={fs['avg']}")
|
||
if "earliest" in fs and fs["earliest"]:
|
||
parts.append(f"{fs['earliest']} ~ {fs['latest']}")
|
||
if "min_len" in fs and fs["min_len"] is not None:
|
||
parts.append(f"len={fs['min_len']}~{fs['max_len']}")
|
||
if "true_count" in fs:
|
||
parts.append(f"T={fs['true_count']}, F={fs['false_count']}")
|
||
if fs.get("note"):
|
||
parts.append(fs["note"])
|
||
stats = "; ".join(parts) if parts else "—"
|
||
return f"| `{col}` | {typ} | {null_pct} | {distinct} | {stats} |"
|
||
|
||
def _write_field_stats_table(w, stats: list[dict], table_label: str):
|
||
"""输出字段级统计表格（折叠展示）"""
|
||
if not stats:
|
||
return
|
||
w(f"<details><summary>📊 {table_label} 字段级统计（{len(stats)} 列）</summary>")
|
||
w()
|
||
w("| 字段 | 类型 | NULL率 | 唯一值 | 统计 |")
|
||
w("|------|------|--------|--------|------|")
|
||
for fs in stats:
|
||
w(_fmt_field_stat(fs))
|
||
w()
|
||
w("</details>")
|
||
w()
|
||
|
||
# ── 1. ETL 执行概览 ──
|
||
w("## 1. ETL 执行概览")
|
||
w()
|
||
succ = sum(1 for v in log_results.values() if v.get("status") == "SUCC")
|
||
fail = sum(1 for v in log_results.values() if v.get("status") == "FAIL")
|
||
skip = sum(1 for v in log_results.values() if v.get("status") == "SKIP")
|
||
w(f"| 指标 | 值 |")
|
||
w(f"|------|-----|")
|
||
w(f"| 成功任务 | {succ} |")
|
||
w(f"| 失败任务 | {fail} |")
|
||
w(f"| 跳过任务 | {skip} |")
|
||
w(f"| 总计 | {len(log_results)} |")
|
||
w()
|
||
|
||
if fail > 0:
|
||
w("### 失败任务")
|
||
w()
|
||
for name, info in sorted(log_results.items()):
|
||
if info.get("status") == "FAIL":
|
||
w(f"- **{name}**: {info.get('error', '未知错误')}")
|
||
w()
|
||
|
||
# ── 2. API↔ODS 数据一致性 ──
|
||
w("## 2. API↔ODS 数据一致性")
|
||
w()
|
||
w("### 2.1 汇总")
|
||
w()
|
||
w("| ODS 表 | API 记录数 | ODS 行数 | ODS 去重ID | 数据截止 | 字段匹配 | API独有 | ODS独有 | 值差异 | 白名单 | 状态 |")
|
||
w("|--------|-----------|---------|-----------|---------|---------|--------|--------|-------|--------|------|")
|
||
for c in api_ods_checks:
|
||
fc = c["field_check"]
|
||
real_diffs = [d for d in c["value_diffs"] if not d.get("whitelist")]
|
||
wl_diffs = [d for d in c["value_diffs"] if d.get("whitelist")]
|
||
cutoff = c.get("data_cutoff") or "—"
|
||
w(f"| `{c['ods_table']}` | {c['api_records']} | {c['ods_rows']} | {c['ods_distinct_ids']} "
|
||
f"| {cutoff} "
|
||
f"| {fc['matched']}/{fc['api_fields']} | {len(fc['api_only'])} | {len(fc['ods_only'])} "
|
||
f"| {len(real_diffs)} | {len(wl_diffs)} | {c['status']} |")
|
||
w()
|
||
|
||
# 逐表差异详情 + 字段统计
|
||
for c in api_ods_checks:
|
||
fc = c["field_check"]
|
||
has_diff = fc["api_only"] or fc["ods_only"] or c["value_diffs"]
|
||
has_stats = c.get("ods_field_stats")
|
||
|
||
if not has_diff and not has_stats:
|
||
continue
|
||
|
||
w(f"### 2.2 {c['ods_table']} 详情")
|
||
w()
|
||
|
||
if fc["api_only"]:
|
||
w(f"**API 独有字段** ({len(fc['api_only'])} 个): `{'`, `'.join(fc['api_only'][:15])}`")
|
||
w()
|
||
|
||
if fc["ods_only"]:
|
||
w(f"**ODS 独有字段** ({len(fc['ods_only'])} 个): `{'`, `'.join(fc['ods_only'][:15])}`")
|
||
w()
|
||
|
||
if c["value_diffs"]:
|
||
real = [d for d in c["value_diffs"] if not d.get("whitelist")]
|
||
wl = [d for d in c["value_diffs"] if d.get("whitelist")]
|
||
|
||
if real:
|
||
w(f"**值差异采样** ({len(real)} 条)")
|
||
w()
|
||
w("| ID | 字段 | API 值 | ODS 值 |")
|
||
w("|----|------|--------|--------|")
|
||
for d in real:
|
||
w(f"| {d['id']} | `{d['field']}` | {d['api_val']} | {d['ods_val']} |")
|
||
w()
|
||
|
||
if wl:
|
||
w(f"<details><summary>🔕 白名单差异（{len(wl)} 条）— API 空字符串 vs DB None，视为等价</summary>")
|
||
w()
|
||
w("| ID | 字段 | API 值 | ODS 值 |")
|
||
w("|----|------|--------|--------|")
|
||
for d in wl:
|
||
w(f"| {d['id']} | `{d['field']}` | {d['api_val']} | {d['ods_val']} |")
|
||
w()
|
||
w("</details>")
|
||
w()
|
||
|
||
# ODS 字段级统计
|
||
if has_stats:
|
||
_write_field_stats_table(w, c["ods_field_stats"], f"ods.{c['ods_table']}")
|
||
|
||
# ── 3. ODS↔DWD 数据一致性 ──
|
||
w("## 3. ODS↔DWD 数据一致性")
|
||
w()
|
||
w("### 3.1 汇总")
|
||
w()
|
||
w("| DWD 表 | EX 表 | ODS 表 | 类型 | ODS 行 | ODS 去重ID | DWD 行 | 数据截止 | 共同列 | DWD独有 | 值差异 | 白名单 | 状态 |")
|
||
w("|--------|-------|--------|------|-------|-----------|-------|---------|-------|--------|-------|--------|------|")
|
||
for c in ods_dwd_checks:
|
||
real_diffs = [d for d in c["value_diffs"] if not d.get("whitelist")]
|
||
wl_diffs = [d for d in c["value_diffs"] if d.get("whitelist")]
|
||
ex_label = c.get("ex_table", "—") or "—"
|
||
cutoff = c.get("data_cutoff") or "—"
|
||
w(f"| `{c['dwd_table']}` | `{ex_label}` | `{c['ods_table']}` | {c['type']} "
|
||
f"| {c['ods_rows']} | {c['ods_distinct_ids']} | {c['dwd_rows']} "
|
||
f"| {cutoff} "
|
||
f"| {len(c['common_cols'])} | {len(c['dwd_only_cols'])} "
|
||
f"| {len(real_diffs)} | {len(wl_diffs)} | {c['status']} |")
|
||
w()
|
||
|
||
# 逐表差异详情 + 字段统计
|
||
for c in ods_dwd_checks:
|
||
has_diff = c["dwd_only_cols"] or c["value_diffs"]
|
||
has_stats = c.get("dwd_field_stats")
|
||
|
||
if not has_diff and not has_stats:
|
||
continue
|
||
|
||
w(f"### 3.2 {c['dwd_table']} 详情")
|
||
w()
|
||
|
||
if c["dwd_only_cols"]:
|
||
w(f"**DWD 独有列** ({len(c['dwd_only_cols'])} 个): `{'`, `'.join(c['dwd_only_cols'][:15])}`")
|
||
w()
|
||
|
||
if c["ods_only_cols"]:
|
||
w(f"**ODS 独有列** ({len(c['ods_only_cols'])} 个): `{'`, `'.join(c['ods_only_cols'][:15])}`")
|
||
w()
|
||
|
||
if c["value_diffs"]:
|
||
real = [d for d in c["value_diffs"] if not d.get("whitelist")]
|
||
wl = [d for d in c["value_diffs"] if d.get("whitelist")]
|
||
|
||
if real:
|
||
w(f"**值差异采样** ({len(real)} 条)")
|
||
w()
|
||
w("| ID | 字段 | DWD 值 | ODS 值 |")
|
||
w("|----|------|--------|--------|")
|
||
for d in real:
|
||
w(f"| {d['id']} | `{d['field']}` | {d['dwd_val']} | {d['ods_val']} |")
|
||
w()
|
||
|
||
if wl:
|
||
w(f"<details><summary>🔕 白名单差异（{len(wl)} 条）— 空字符串 vs None，视为等价</summary>")
|
||
w()
|
||
w("| ID | 字段 | DWD 值 | ODS 值 |")
|
||
w("|----|------|--------|--------|")
|
||
for d in wl:
|
||
w(f"| {d['id']} | `{d['field']}` | {d['dwd_val']} | {d['ods_val']} |")
|
||
w()
|
||
w("</details>")
|
||
w()
|
||
|
||
# DWD 字段级统计
|
||
if has_stats:
|
||
_write_field_stats_table(w, c["dwd_field_stats"], c["dwd_table"])
|
||
|
||
# ── 4. DWD↔DWS 数据一致性 ──
|
||
w("## 4. DWD↔DWS 数据一致性")
|
||
w()
|
||
w("### 4.1 DWS 表概览")
|
||
w()
|
||
non_empty = sum(1 for c in dws_checks if c["rows"] > 0)
|
||
w(f"DWS 层共 {len(dws_checks)} 张表，{non_empty} 张有数据，{len(dws_checks) - non_empty} 张为空。")
|
||
w()
|
||
w("| DWS 表 | 行数 | 列数 | 状态 |")
|
||
w("|--------|------|------|------|")
|
||
for c in dws_checks:
|
||
w(f"| `{c['table']}` | {c['rows']} | {c['col_count']} | {c['status']} |")
|
||
w()
|
||
|
||
# DWS 数值列健全性检查
|
||
has_checks = [c for c in dws_checks if c["sample_checks"]]
|
||
if has_checks:
|
||
w("### 4.2 DWS 数值列健全性检查")
|
||
w()
|
||
for c in has_checks:
|
||
anomalies = [sc for sc in c["sample_checks"] if sc["status"] != "✅"]
|
||
if not anomalies:
|
||
continue
|
||
w(f"#### {c['table']}")
|
||
w()
|
||
w("| 列 | NULL率 | 负值数 | 最小值 | 最大值 | 状态 |")
|
||
w("|----|--------|--------|--------|--------|------|")
|
||
for sc in anomalies:
|
||
w(f"| `{sc['col']}` | {sc['null_pct']} | {sc['neg_count']} | {sc['min']} | {sc['max']} | {sc['status']} |")
|
||
w()
|
||
|
||
# ── 5. 异常汇总 ──
|
||
w("## 5. 异常汇总与建议")
|
||
w()
|
||
issues = []
|
||
for c in api_ods_checks:
|
||
if c["status"] != "✅":
|
||
issues.append(f"API↔ODS `{c['ods_table']}`: {c['status']}")
|
||
for c in ods_dwd_checks:
|
||
if c["status"] != "✅":
|
||
issues.append(f"ODS↔DWD `{c['dwd_table']}`: {c['status']}")
|
||
for c in dws_checks:
|
||
for sc in c.get("sample_checks", []):
|
||
if sc["status"] != "✅":
|
||
issues.append(f"DWS `{c['table']}.{sc['col']}`: {sc['status']}")
|
||
|
||
if issues:
|
||
w(f"共发现 {len(issues)} 项异常：")
|
||
w()
|
||
for i, issue in enumerate(issues, 1):
|
||
w(f"{i}. {issue}")
|
||
else:
|
||
w("✅ 未发现数据一致性异常。")
|
||
w()
|
||
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ── 主入口 ──
|
||
def main():
|
||
print(f"[{NOW.strftime('%H:%M:%S')}] ETL 全链路数据一致性检查开始...")
|
||
|
||
# 1. 找到最新 ETL 日志
|
||
log_path = find_latest_log()
|
||
if not log_path:
|
||
print("❌ 未找到 ETL 日志文件")
|
||
sys.exit(1)
|
||
print(f" 日志: {log_path.name}")
|
||
|
||
# 2. 解析日志
|
||
log_results = parse_etl_log(log_path)
|
||
succ_ods = {k: v for k, v in log_results.items()
|
||
if v.get("status") == "SUCC" and v.get("layer") == "ODS"}
|
||
print(f" 成功 ODS 任务: {len(succ_ods)}")
|
||
|
||
# 3. 连接数据库
|
||
conn = get_conn()
|
||
print(f" 数据库连接成功（只读模式）")
|
||
|
||
try:
|
||
# 4. API vs ODS 检查
|
||
print(f"\n[API↔ODS] 开始逐表检查...")
|
||
api_ods_checks = []
|
||
for task_name, ods_table in sorted(ODS_TASK_TO_TABLE.items()):
|
||
if task_name not in succ_ods:
|
||
continue
|
||
print(f" 检查 {task_name} → ods.{ods_table}...", end=" ")
|
||
result = check_api_vs_ods(conn, task_name, ods_table)
|
||
api_ods_checks.append(result)
|
||
print(result["status"])
|
||
|
||
# 5. ODS vs DWD 检查
|
||
print(f"\n[ODS↔DWD] 开始逐表检查...")
|
||
ods_dwd_checks = []
|
||
for dwd_full, ods_full in sorted(DWD_TO_ODS.items()):
|
||
print(f" 检查 {dwd_full} ← {ods_full}...", end=" ")
|
||
result = check_ods_vs_dwd(conn, dwd_full, ods_full)
|
||
ods_dwd_checks.append(result)
|
||
print(result["status"])
|
||
|
||
# 6. DWD vs DWS 检查
|
||
print(f"\n[DWD↔DWS] 开始检查...")
|
||
dws_checks = check_dwd_vs_dws(conn)
|
||
non_empty = sum(1 for c in dws_checks if c["rows"] > 0)
|
||
print(f" DWS 表: {len(dws_checks)} 张，{non_empty} 张有数据")
|
||
|
||
# 7. 生成报告
|
||
report = generate_report(log_path, log_results, api_ods_checks, ods_dwd_checks, dws_checks)
|
||
out_file = REPORT_ROOT / f"consistency_check_{TS}.md"
|
||
out_file.write_text(report, encoding="utf-8")
|
||
print(f"\n✅ 报告已生成: {out_file}")
|
||
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|