在前后端开发联调前 的提交20260223
This commit is contained in:
@@ -13,7 +13,7 @@
|
||||
|
||||
用法:
|
||||
python scripts/ops/gen_dataflow_report.py
|
||||
python scripts/ops/gen_dataflow_report.py --output-dir export/dataflow_analysis
|
||||
python scripts/ops/gen_dataflow_report.py --output-dir /path/to/output
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,7 +24,51 @@ import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from dotenv import load_dotenv # noqa: F401 — _env_paths 负责加载，此处保留以防其他模块间接引用
|
||||
|
||||
# ── 白名单定义 ──────────────────────────────────────────────────────────
|
||||
# 白名单字段仍然参与检查和统计，但在报告的 1.1 差异明细表格和 3. 逐表详情表格中
|
||||
# 折叠显示（不展开详细行），并注明白名单原因。
|
||||
# CHANGE 2026-02-21 | 重构白名单逻辑：统一术语为"白名单"，字段仍正常检查，仅报告展示折叠
|
||||
|
||||
# ODS 层 ETL 元数据列（非业务字段，ETL 流程自动生成）
|
||||
WHITELIST_ETL_META_COLS = {
|
||||
"source_file", "source_endpoint", "fetched_at", "payload", "content_hash",
|
||||
}
|
||||
|
||||
# DWD 维表 SCD2 管理列（ETL 框架自动维护，非业务映射）
|
||||
WHITELIST_DWD_SCD2_COLS = {
|
||||
"valid_from", "valid_to", "is_current", "etl_loaded_at", "etl_batch_id",
|
||||
}
|
||||
|
||||
# API 嵌套对象前缀（上游 API 的门店信息嵌套结构，已通过 site_id 关联，不逐字段映射）
|
||||
WHITELIST_API_NESTED_PREFIXES = ("siteProfile.",)
|
||||
|
||||
|
||||
def is_whitelist_etl_meta(col_name: str) -> bool:
|
||||
"""判断是否为 ETL 元数据白名单列"""
|
||||
return col_name in WHITELIST_ETL_META_COLS
|
||||
|
||||
|
||||
def is_whitelist_scd2(col_name: str) -> bool:
|
||||
"""判断是否为 DWD SCD2 管理白名单列"""
|
||||
return col_name in WHITELIST_DWD_SCD2_COLS
|
||||
|
||||
|
||||
def is_whitelist_api_nested(json_path: str) -> bool:
|
||||
"""判断是否为 API 嵌套对象白名单字段"""
|
||||
return any(json_path.startswith(p) for p in WHITELIST_API_NESTED_PREFIXES)
|
||||
|
||||
|
||||
def whitelist_reason(col_name: str, json_path: str = "", layer: str = "") -> str:
|
||||
"""返回白名单原因描述，非白名单返回空字符串"""
|
||||
if is_whitelist_etl_meta(col_name):
|
||||
return "ETL 元数据列"
|
||||
if is_whitelist_scd2(col_name):
|
||||
return "SCD2 管理列"
|
||||
if json_path and is_whitelist_api_nested(json_path):
|
||||
return "API 嵌套对象（siteProfile）"
|
||||
return ""
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict | list | None:
|
||||
@@ -37,17 +81,15 @@ def load_json(path: Path) -> dict | list | None:
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="生成数据流结构分析 Markdown 报告")
|
||||
parser.add_argument("--output-dir", type=str, default=None,
|
||||
help="输出目录（默认读取 SYSTEM_ANALYZE_ROOT 或 export/dataflow_analysis）")
|
||||
help="输出目录（默认读取 .env 中的 SYSTEM_ANALYZE_ROOT）")
|
||||
return parser
|
||||
|
||||
|
||||
def resolve_data_dir(override: str | None = None) -> Path:
|
||||
if override:
|
||||
return Path(override)
|
||||
env_root = os.environ.get("SYSTEM_ANALYZE_ROOT")
|
||||
if env_root:
|
||||
return Path(env_root)
|
||||
return Path("export/dataflow_analysis")
|
||||
from _env_paths import get_output_path
|
||||
return get_output_path("SYSTEM_ANALYZE_ROOT")
|
||||
|
||||
|
||||
def _esc(s: str) -> str:
|
||||
@@ -55,81 +97,6 @@ def _esc(s: str) -> str:
|
||||
return str(s).replace("|", "\\|").replace("\n", " ") if s else ""
|
||||
|
||||
|
||||
# ── 字段用途推测规则 ──
|
||||
# 基于字段名模式 + 表名上下文推断字段可能的业务含义
|
||||
# 置信度：高(≥80%) / 中(50-79%) / 低(<50%)
|
||||
import re as _re
|
||||
|
||||
_FIELD_GUESS_RULES: list[tuple[str, str, str]] = [
|
||||
# (字段名模式正则, 推测用途, 置信度)
|
||||
# ── SCD2 / ETL 元数据 ──
|
||||
(r"^scd2_", "SCD2 缓慢变化维度元数据", "高"),
|
||||
(r"^etl_", "ETL 流程元数据", "高"),
|
||||
(r"^dw_insert", "数仓装载时间戳", "高"),
|
||||
(r"^content_hash$", "数据变更检测哈希", "高"),
|
||||
(r"^source_file$", "ETL 来源文件标识", "高"),
|
||||
(r"^source_endpoint$", "ETL 来源接口标识", "高"),
|
||||
(r"^fetched_at$", "ETL 抓取时间", "高"),
|
||||
(r"^payload$", "原始 JSON 全量存储", "高"),
|
||||
# ── 主键 / 外键 ──
|
||||
(r"^id$", "主键标识", "高"),
|
||||
# ── 门店 / 组织（放在通用 _id$ 之前） ──
|
||||
(r"^(site_id|shop_id|store_id)$", "门店标识", "高"),
|
||||
(r"^(tenant_id|org_id)$", "租户/组织标识", "高"),
|
||||
(r"(shop_name|site_name|store_name)", "门店名称", "高"),
|
||||
# ── 时间类 ──
|
||||
(r"(^|_)(create|created)(_at|_time|_date)$", "记录创建时间", "高"),
|
||||
(r"(^|_)(update|updated|modify)(_at|_time|_date)$", "记录更新时间", "高"),
|
||||
(r"(^|_)(delete|deleted)(_at|_time|_date)$", "逻辑删除时间", "高"),
|
||||
(r"(^|_)(start|begin)(_at|_time|_date)$", "起始时间", "中"),
|
||||
(r"(^|_)(end|expire)(_at|_time|_date)$", "结束/过期时间", "中"),
|
||||
(r"(^|_)entry_time$", "入职/入场时间", "中"),
|
||||
(r"(^|_)resign_time$", "离职时间", "中"),
|
||||
(r"_time$", "时间戳字段", "中"),
|
||||
(r"_date$", "日期字段", "中"),
|
||||
# ── 通用派生（放在标志位之前，确保 derived_flag 等优先匹配派生） ──
|
||||
(r"^derived_", "ETL 派生计算列", "高"),
|
||||
(r"^calc_", "计算字段", "中"),
|
||||
# ── 状态 / 标志 ──
|
||||
(r"(^|_)is_delete$", "逻辑删除标志", "高"),
|
||||
(r"^is_", "布尔标志位", "中"),
|
||||
(r"(^|_)status$", "状态码", "中"),
|
||||
(r"_status$", "状态字段", "中"),
|
||||
(r"_enabled$", "启用/禁用开关", "中"),
|
||||
(r"_flag$", "标志位", "中"),
|
||||
# ── 金额 / 价格 ──
|
||||
(r"(price|amount|fee|cost|money|balance|total)", "金额/价格相关", "中"),
|
||||
(r"(discount|coupon|refund)", "优惠/退款相关", "中"),
|
||||
# ── 人员 ──
|
||||
(r"(real_name|nickname|^name$)", "姓名/昵称", "中"),
|
||||
(r"(mobile|phone|tel)", "联系电话", "中"),
|
||||
(r"(avatar|photo|image)", "头像/图片 URL", "中"),
|
||||
(r"(gender|sex)", "性别", "高"),
|
||||
(r"(birth|birthday)", "出生日期", "高"),
|
||||
(r"(height|weight)", "身高/体重", "高"),
|
||||
# ── 嵌套对象常见前缀 ──
|
||||
(r"^siteProfile\.", "门店档案嵌套属性", "高"),
|
||||
(r"^memberInfo\.", "会员信息嵌套属性", "中"),
|
||||
(r"^assistantInfo\.", "助教信息嵌套属性", "中"),
|
||||
(r"^tableInfo\.", "台桌信息嵌套属性", "中"),
|
||||
(r"^orderInfo\.", "订单信息嵌套属性", "中"),
|
||||
(r"^payInfo\.", "支付信息嵌套属性", "中"),
|
||||
# ── 排序 / 显示 ──
|
||||
(r"(sort|order|rank|seq)", "排序/序号", "低"),
|
||||
(r"(remark|memo|note|comment|introduce)", "备注/说明文本", "中"),
|
||||
(r"(url|link|qrcode|qr_code)", "链接/二维码", "中"),
|
||||
# ── 通用 ID 后缀（放在具体 ID 规则之后） ──
|
||||
(r"_id$", "关联实体 ID（外键）", "中"),
|
||||
]
|
||||
|
||||
|
||||
def _guess_field_purpose(field_name: str, table_name: str, layer: str) -> tuple[str, str]:
|
||||
"""根据字段名和表上下文推测用途，返回 (推测用途, 置信度)。"""
|
||||
fn_lower = field_name.lower()
|
||||
for pattern, purpose, confidence in _FIELD_GUESS_RULES:
|
||||
if _re.search(pattern, fn_lower):
|
||||
return purpose, confidence
|
||||
return f"待分析（{layer}层字段）", "低"
|
||||
|
||||
|
||||
def _format_samples(samples: list[str], max_show: int = 5) -> str:
|
||||
@@ -155,12 +122,71 @@ def _is_enum_like(samples: list[str], total_records: int) -> bool:
|
||||
return 1 < len(samples) <= 8
|
||||
|
||||
|
||||
def _write_source_file_manifest(w, data_dir: Path, tables: list[dict], fm_dir: Path | None = None):
|
||||
"""在报告开头输出本次分析用到的所有 JSON 数据源文件清单"""
|
||||
if fm_dir is None:
|
||||
fm_dir = data_dir / "field_mappings"
|
||||
w("## 数据源文件清单")
|
||||
w()
|
||||
w("本报告基于以下 JSON 数据文件生成：")
|
||||
w()
|
||||
|
||||
categories = [
|
||||
("collection_manifest.json", "采集元数据（表清单、日期范围、记录数）"),
|
||||
("json_trees/", "API JSON 字段结构（递归展开后的字段路径、类型、示例值）"),
|
||||
("field_mappings/", "三层字段映射（API→ODS→DWD 映射关系）"),
|
||||
("db_schemas/", "数据库表结构（ODS/DWD 列定义，来自 PostgreSQL）"),
|
||||
("bd_descriptions/", "业务描述（来自 BD_manual 文档）"),
|
||||
]
|
||||
|
||||
for cat_path, cat_desc in categories:
|
||||
if cat_path.endswith("/"):
|
||||
# 子目录：列出实际存在的文件
|
||||
# CHANGE 2026-02-21 | field_mappings 使用传入的 fm_dir（可能是 field_mappings_new）
|
||||
if cat_path.rstrip("/") == "field_mappings":
|
||||
sub_dir = fm_dir
|
||||
else:
|
||||
sub_dir = data_dir / cat_path.rstrip("/")
|
||||
if sub_dir.is_dir():
|
||||
try:
|
||||
files = sorted(f.name for f in sub_dir.iterdir() if f.suffix == ".json")
|
||||
except PermissionError:
|
||||
w(f"**{cat_path}** — {cat_desc}（目录权限拒绝）")
|
||||
w()
|
||||
continue
|
||||
if sub_dir.is_dir():
|
||||
files = sorted(f.name for f in sub_dir.iterdir() if f.suffix == ".json")
|
||||
w(f"**{cat_path}** — {cat_desc}（{len(files)} 个文件）")
|
||||
w()
|
||||
for fn in files:
|
||||
w(f"- `{cat_path}{fn}`")
|
||||
w()
|
||||
else:
|
||||
w(f"**{cat_path}** — {cat_desc}（目录不存在）")
|
||||
w()
|
||||
else:
|
||||
# 单文件
|
||||
fp = data_dir / cat_path
|
||||
status = "✓" if fp.exists() else "✗ 缺失"
|
||||
w(f"- `{cat_path}` — {cat_desc}（{status}）")
|
||||
w()
|
||||
|
||||
w("---")
|
||||
w()
|
||||
|
||||
|
||||
def generate_report(data_dir: Path) -> str:
|
||||
"""生成完整的 Markdown 报告"""
|
||||
manifest = load_json(data_dir / "collection_manifest.json")
|
||||
if not manifest:
|
||||
raise FileNotFoundError(f"找不到 collection_manifest.json: {data_dir}")
|
||||
|
||||
# CHANGE 2026-02-21 | Windows 文件锁 fallback：field_mappings_new 优先于被锁的 field_mappings
|
||||
_fm_dir = data_dir / "field_mappings"
|
||||
_fm_new = data_dir / "field_mappings_new"
|
||||
if _fm_new.exists() and any(_fm_new.iterdir()):
|
||||
_fm_dir = _fm_new
|
||||
|
||||
tables = manifest["tables"]
|
||||
now = datetime.now()
|
||||
lines: list[str] = []
|
||||
@@ -168,14 +194,25 @@ def generate_report(data_dir: Path) -> str:
|
||||
def w(s: str = ""):
|
||||
lines.append(s)
|
||||
|
||||
# ── 从 manifest 读取 API 请求日期范围 ──
|
||||
api_date_from = manifest.get("date_from")
|
||||
api_date_to = manifest.get("date_to")
|
||||
total_records_all = sum(t.get("record_count", 0) for t in tables)
|
||||
|
||||
# ── 报告头 ──
|
||||
w("# 飞球连接器 — 数据流结构分析报告")
|
||||
w()
|
||||
w(f"> 生成时间：{now.strftime('%Y-%m-%d %H:%M:%S')} CST")
|
||||
w(f"> 分析范围：飞球（feiqiu）连接器，共 {len(tables)} 张 ODS 表")
|
||||
w("> 数据来源：API JSON 采样 + PostgreSQL ODS/DWD 表结构 + 三层字段映射 + BD_manual 业务文档")
|
||||
if api_date_from or api_date_to:
|
||||
w(f"> API 请求日期范围：{api_date_from or '—'} ~ {api_date_to or '—'}")
|
||||
w(f"> JSON 数据总量：{total_records_all} 条记录")
|
||||
w()
|
||||
|
||||
# ── 数据源文件清单 ──
|
||||
_write_source_file_manifest(w, data_dir, tables, fm_dir=_fm_dir)
|
||||
|
||||
# ── 1. 总览表（增加 API JSON 字段数列） ──
|
||||
w("## 1. 总览")
|
||||
w()
|
||||
@@ -197,7 +234,7 @@ def generate_report(data_dir: Path) -> str:
|
||||
w()
|
||||
|
||||
# ── 1.1 字段对比差异报告 ──
|
||||
_write_field_diff_report(w, data_dir, tables)
|
||||
_write_field_diff_report(w, data_dir, tables, fm_dir=_fm_dir)
|
||||
|
||||
# ── 2. 全局统计 ──
|
||||
w("## 2. 全局统计")
|
||||
@@ -208,7 +245,7 @@ def generate_report(data_dir: Path) -> str:
|
||||
total_mapped = 0
|
||||
per_table_stats: list[dict] = []
|
||||
for t in tables:
|
||||
fm = load_json(data_dir / "field_mappings" / f"{t['table']}.json")
|
||||
fm = load_json(_fm_dir / f"{t['table']}.json")
|
||||
if not fm or "json_to_ods" not in fm:
|
||||
per_table_stats.append({
|
||||
"table": t["table"], "description": t["description"],
|
||||
@@ -261,7 +298,7 @@ def generate_report(data_dir: Path) -> str:
|
||||
|
||||
for idx, t in enumerate(tables, 1):
|
||||
table_name = t["table"]
|
||||
fm = load_json(data_dir / "field_mappings" / f"{table_name}.json")
|
||||
fm = load_json(_fm_dir / f"{table_name}.json")
|
||||
jt = load_json(data_dir / "json_trees" / f"{table_name}.json")
|
||||
ods_schema = load_json(data_dir / "db_schemas" / f"ods_{table_name}.json")
|
||||
bd = load_json(data_dir / "bd_descriptions" / f"{table_name}.json")
|
||||
@@ -303,8 +340,10 @@ def generate_report(data_dir: Path) -> str:
|
||||
|
||||
|
||||
|
||||
def _write_field_diff_report(w, data_dir: Path, tables: list[dict]):
|
||||
def _write_field_diff_report(w, data_dir: Path, tables: list[dict], fm_dir: Path | None = None):
|
||||
"""生成 API↔ODS↔DWD 字段对比差异报告（汇总表 + 逐表分表）"""
|
||||
if fm_dir is None:
|
||||
fm_dir = data_dir / "field_mappings"
|
||||
w("### 1.1 API↔ODS↔DWD 字段对比差异")
|
||||
w()
|
||||
w("以下汇总各表在三层之间的字段差异（点击数字跳转至分表详情）：")
|
||||
@@ -312,13 +351,13 @@ def _write_field_diff_report(w, data_dir: Path, tables: list[dict]):
|
||||
w("| ODS 表名 | API→ODS 未映射 | ODS 无 JSON 源 | ODS→DWD 未映射 | DWD 无 ODS 源 | 主要差异原因 |")
|
||||
w("|---------|--------------|--------------|--------------|-------------|------------|")
|
||||
|
||||
# CHANGE 2026-02-21 | 重构白名单逻辑：字段仍正常检查计数，白名单字段在分表详情中折叠
|
||||
# 收集每表差异数据，用于汇总表和分表
|
||||
etl_meta_cols = {"source_file", "source_endpoint", "fetched_at", "payload", "content_hash"}
|
||||
diff_rows: list[dict] = []
|
||||
|
||||
for t in tables:
|
||||
table_name = t["table"]
|
||||
fm = load_json(data_dir / "field_mappings" / f"{table_name}.json")
|
||||
fm = load_json(fm_dir / f"{table_name}.json")
|
||||
if not fm:
|
||||
w(f"| `{table_name}` | — | — | — | — | 无映射数据 |")
|
||||
diff_rows.append(None)
|
||||
@@ -334,43 +373,62 @@ def _write_field_diff_report(w, data_dir: Path, tables: list[dict]):
|
||||
o2d = fm.get("ods_to_dwd", {})
|
||||
d2o = fm.get("dwd_to_ods", {})
|
||||
|
||||
# ── API→ODS 未映射字段 ──
|
||||
# ── API→ODS 未映射字段（全部检查，含白名单） ──
|
||||
api_unmapped_flat: list[str] = []
|
||||
api_unmapped_nested: list[str] = []
|
||||
api_unmapped_whitelist: list[tuple[str, str]] = [] # (json_path, reason)
|
||||
for m in j2o:
|
||||
if m.get("ods_col") is None:
|
||||
jp = m.get("json_path", "")
|
||||
if "." in jp:
|
||||
wl_reason = whitelist_reason("", json_path=jp)
|
||||
if wl_reason:
|
||||
api_unmapped_whitelist.append((jp, wl_reason))
|
||||
elif "." in jp:
|
||||
api_unmapped_nested.append(jp)
|
||||
else:
|
||||
api_unmapped_flat.append(jp)
|
||||
api_unmapped_total = len(api_unmapped_flat) + len(api_unmapped_nested)
|
||||
api_unmapped_total = len(api_unmapped_flat) + len(api_unmapped_nested) + len(api_unmapped_whitelist)
|
||||
|
||||
# ── ODS 无 JSON 源 ──
|
||||
# ── ODS 无 JSON 源（全部检查，含白名单） ──
|
||||
ods_schema = load_json(data_dir / "db_schemas" / f"ods_{table_name}.json")
|
||||
ods_mapped_cols = {m["ods_col"] for m in j2o if m.get("ods_col")}
|
||||
ods_no_json_fields: list[str] = []
|
||||
ods_no_json_whitelist: list[tuple[str, str]] = [] # (col_name, reason)
|
||||
if ods_schema and "columns" in ods_schema:
|
||||
for col in ods_schema["columns"]:
|
||||
if col["name"] not in ods_mapped_cols and col["name"] not in etl_meta_cols:
|
||||
ods_no_json_fields.append(col["name"])
|
||||
if col["name"] not in ods_mapped_cols:
|
||||
wl_reason = whitelist_reason(col["name"])
|
||||
if wl_reason:
|
||||
ods_no_json_whitelist.append((col["name"], wl_reason))
|
||||
else:
|
||||
ods_no_json_fields.append(col["name"])
|
||||
|
||||
# ── ODS→DWD 未映射 ──
|
||||
# ── ODS→DWD 未映射（全部检查，含白名单） ──
|
||||
ods_cols_with_dwd = set(o2d.keys())
|
||||
ods_no_dwd_fields: list[str] = []
|
||||
ods_no_dwd_whitelist: list[tuple[str, str]] = []
|
||||
if ods_schema and "columns" in ods_schema:
|
||||
for col in ods_schema["columns"]:
|
||||
if col["name"] not in ods_cols_with_dwd and col["name"] not in etl_meta_cols:
|
||||
ods_no_dwd_fields.append(col["name"])
|
||||
if col["name"] not in ods_cols_with_dwd:
|
||||
wl_reason = whitelist_reason(col["name"])
|
||||
if wl_reason:
|
||||
ods_no_dwd_whitelist.append((col["name"], wl_reason))
|
||||
else:
|
||||
ods_no_dwd_fields.append(col["name"])
|
||||
|
||||
# ── DWD 无 ODS 源 ──
|
||||
# ── DWD 无 ODS 源（全部检查，含白名单） ──
|
||||
dwd_no_ods_fields: list[tuple[str, str]] = [] # (dwd_table, dwd_col)
|
||||
dwd_no_ods_whitelist: list[tuple[str, str, str]] = [] # (dwd_table, dwd_col, reason)
|
||||
for dwd_name, entries in d2o.items():
|
||||
for entry in entries:
|
||||
if entry.get("ods_source") == "—":
|
||||
dwd_no_ods_fields.append((dwd_name, entry["dwd_col"]))
|
||||
wl_reason = whitelist_reason(entry["dwd_col"])
|
||||
if wl_reason:
|
||||
dwd_no_ods_whitelist.append((dwd_name, entry["dwd_col"], wl_reason))
|
||||
else:
|
||||
dwd_no_ods_fields.append((dwd_name, entry["dwd_col"]))
|
||||
|
||||
# 差异原因
|
||||
# 差异原因（含白名单统计）
|
||||
reasons: list[str] = []
|
||||
if api_unmapped_nested:
|
||||
reasons.append(f"嵌套对象 {len(api_unmapped_nested)} 个")
|
||||
@@ -378,15 +436,18 @@ def _write_field_diff_report(w, data_dir: Path, tables: list[dict]):
|
||||
reasons.append(f"平层未映射 {len(api_unmapped_flat)} 个")
|
||||
if dwd_no_ods_fields:
|
||||
reasons.append(f"SCD2/派生列 {len(dwd_no_ods_fields)} 个")
|
||||
wl_total = len(api_unmapped_whitelist) + len(ods_no_json_whitelist) + len(ods_no_dwd_whitelist) + len(dwd_no_ods_whitelist)
|
||||
if wl_total:
|
||||
reasons.append(f"白名单 {wl_total} 个")
|
||||
reason_str = "；".join(reasons) if reasons else "—"
|
||||
|
||||
# 汇总表单元格：数量 + 跳转链接
|
||||
# 汇总表单元格：数量 + 跳转链接（白名单字段也计入总数）
|
||||
def _cell(count: int) -> str:
|
||||
if count == 0:
|
||||
return "0"
|
||||
return f"[{count}](#{diff_anchor})"
|
||||
|
||||
w(f"| `{table_name}` | {_cell(api_unmapped_total)} | {_cell(len(ods_no_json_fields))} | {_cell(len(ods_no_dwd_fields))} | {_cell(len(dwd_no_ods_fields))} | {reason_str} |")
|
||||
w(f"| `{table_name}` | {_cell(api_unmapped_total)} | {_cell(len(ods_no_json_fields) + len(ods_no_json_whitelist))} | {_cell(len(ods_no_dwd_fields) + len(ods_no_dwd_whitelist))} | {_cell(len(dwd_no_ods_fields) + len(dwd_no_ods_whitelist))} | {reason_str} |")
|
||||
|
||||
diff_rows.append({
|
||||
"table_name": table_name,
|
||||
@@ -396,21 +457,28 @@ def _write_field_diff_report(w, data_dir: Path, tables: list[dict]):
|
||||
"dwd_anchors": dwd_anchors,
|
||||
"api_unmapped_flat": api_unmapped_flat,
|
||||
"api_unmapped_nested": api_unmapped_nested,
|
||||
"api_unmapped_whitelist": api_unmapped_whitelist,
|
||||
"ods_no_json_fields": ods_no_json_fields,
|
||||
"ods_no_json_whitelist": ods_no_json_whitelist,
|
||||
"ods_no_dwd_fields": ods_no_dwd_fields,
|
||||
"ods_no_dwd_whitelist": ods_no_dwd_whitelist,
|
||||
"dwd_no_ods_fields": dwd_no_ods_fields,
|
||||
"dwd_no_ods_whitelist": dwd_no_ods_whitelist,
|
||||
})
|
||||
|
||||
w()
|
||||
|
||||
# ── 逐表差异分表 ──
|
||||
# CHANGE 2026-02-21 | 白名单字段折叠显示，不展开详细表格行，注明白名单原因
|
||||
sub_idx = 0
|
||||
for row in diff_rows:
|
||||
if row is None:
|
||||
continue
|
||||
has_any = (row["api_unmapped_flat"] or row["api_unmapped_nested"]
|
||||
or row["ods_no_json_fields"] or row["ods_no_dwd_fields"]
|
||||
or row["dwd_no_ods_fields"])
|
||||
or row["api_unmapped_whitelist"]
|
||||
or row["ods_no_json_fields"] or row["ods_no_json_whitelist"]
|
||||
or row["ods_no_dwd_fields"] or row["ods_no_dwd_whitelist"]
|
||||
or row["dwd_no_ods_fields"] or row["dwd_no_ods_whitelist"])
|
||||
if not has_any:
|
||||
continue
|
||||
|
||||
@@ -464,78 +532,105 @@ def _write_field_diff_report(w, data_dir: Path, tables: list[dict]):
|
||||
desc = desc[:37] + "..."
|
||||
return _esc(desc)
|
||||
|
||||
def _write_whitelist_summary(w, items: list, category: str):
|
||||
"""白名单字段折叠汇总（不展开详细表格行）"""
|
||||
if not items:
|
||||
return
|
||||
# 按原因分组
|
||||
by_reason: dict[str, list[str]] = {}
|
||||
for item in items:
|
||||
if isinstance(item, tuple) and len(item) == 3:
|
||||
name, _, reason = item # (dwd_table, dwd_col, reason)
|
||||
elif isinstance(item, tuple) and len(item) == 2:
|
||||
name, reason = item
|
||||
else:
|
||||
name, reason = str(item), "白名单"
|
||||
by_reason.setdefault(reason, []).append(name)
|
||||
parts = []
|
||||
for reason, names in by_reason.items():
|
||||
parts.append(f"{reason}: `{'`, `'.join(names[:5])}`{'...' if len(names) > 5 else ''} ({len(names)} 个)")
|
||||
w(f"> ℹ️ {category}白名单字段（已检查，不展开详情）：{'；'.join(parts)}")
|
||||
w()
|
||||
|
||||
# ── API→ODS 未映射（平层） ──
|
||||
if row["api_unmapped_flat"]:
|
||||
w(f"**API→ODS 未映射（平层）** — {len(row['api_unmapped_flat'])} 个")
|
||||
w()
|
||||
w("| # | JSON 字段 | 推测用途 | 置信度 | 示例值 | 说明 | 状态 |")
|
||||
w("|---|----------|---------|-------|-------|------|------|")
|
||||
w("| # | JSON 字段 | 示例值 | 说明 | 状态 |")
|
||||
w("|---|----------|-------|------|------|")
|
||||
for i, f in enumerate(row["api_unmapped_flat"], 1):
|
||||
purpose, conf = _guess_field_purpose(f, table_name, "API")
|
||||
sample = _sample_str(f, "API")
|
||||
desc = _desc_str(f, "API")
|
||||
w(f"| {i} | **[`{_esc(f)}`](#{api_anchor})** | {_esc(purpose)} | {conf} | {sample} | {desc} | **⚠️ 未映射** |")
|
||||
w(f"| {i} | **[`{_esc(f)}`](#{api_anchor})** | {sample} | {desc} | **⚠️ 未映射** |")
|
||||
w()
|
||||
|
||||
# ── API→ODS 未映射（嵌套对象） ──
|
||||
# ── API→ODS 未映射（嵌套对象，非白名单） ──
|
||||
if row["api_unmapped_nested"]:
|
||||
w(f"<details><summary>API→ODS 未映射（嵌套对象）— {len(row['api_unmapped_nested'])} 个</summary>")
|
||||
w()
|
||||
w("| # | JSON 字段 | 推测用途 | 置信度 | 示例值 | 说明 | 状态 |")
|
||||
w("|---|----------|---------|-------|-------|------|------|")
|
||||
w("| # | JSON 字段 | 示例值 | 说明 | 状态 |")
|
||||
w("|---|----------|-------|------|------|")
|
||||
for i, f in enumerate(row["api_unmapped_nested"], 1):
|
||||
purpose, conf = _guess_field_purpose(f, table_name, "API")
|
||||
sample = _sample_str(f, "API")
|
||||
desc = _desc_str(f, "API")
|
||||
w(f"| {i} | [`{_esc(f)}`](#{api_anchor}) | {_esc(purpose)} | {conf} | {sample} | {desc} | 📦 嵌套 |")
|
||||
w(f"| {i} | [`{_esc(f)}`](#{api_anchor}) | {sample} | {desc} | 📦 嵌套 |")
|
||||
w()
|
||||
w("</details>")
|
||||
w()
|
||||
|
||||
# ── API 白名单字段汇总 ──
|
||||
_write_whitelist_summary(w, row["api_unmapped_whitelist"], "API→ODS ")
|
||||
|
||||
# ── ODS 无 JSON 源 ──
|
||||
if row["ods_no_json_fields"]:
|
||||
w(f"**ODS 无 JSON 源** — {len(row['ods_no_json_fields'])} 个")
|
||||
w()
|
||||
w("| # | ODS 列 | 推测用途 | 置信度 | 说明 | 状态 |")
|
||||
w("|---|-------|---------|-------|------|------|")
|
||||
w("| # | ODS 列 | 说明 | 状态 |")
|
||||
w("|---|-------|------|------|")
|
||||
for i, f in enumerate(row["ods_no_json_fields"], 1):
|
||||
purpose, conf = _guess_field_purpose(f, table_name, "ODS")
|
||||
desc = _desc_str(f, "ODS")
|
||||
w(f"| {i} | **[`{_esc(f)}`](#{ods_anchor})** | {_esc(purpose)} | {conf} | {desc} | **⚠️ 无 JSON 源** |")
|
||||
w(f"| {i} | **[`{_esc(f)}`](#{ods_anchor})** | {desc} | **⚠️ 无 JSON 源** |")
|
||||
w()
|
||||
|
||||
# ── ODS 无 JSON 源 白名单汇总 ──
|
||||
_write_whitelist_summary(w, row["ods_no_json_whitelist"], "ODS 无 JSON 源 ")
|
||||
|
||||
# ── ODS→DWD 未映射 ──
|
||||
if row["ods_no_dwd_fields"]:
|
||||
w(f"**ODS→DWD 未映射** — {len(row['ods_no_dwd_fields'])} 个")
|
||||
w()
|
||||
w("| # | ODS 列 | 推测用途 | 置信度 | 说明 | 状态 |")
|
||||
w("|---|-------|---------|-------|------|------|")
|
||||
w("| # | ODS 列 | 说明 | 状态 |")
|
||||
w("|---|-------|------|------|")
|
||||
for i, f in enumerate(row["ods_no_dwd_fields"], 1):
|
||||
purpose, conf = _guess_field_purpose(f, table_name, "ODS")
|
||||
desc = _desc_str(f, "ODS")
|
||||
w(f"| {i} | **[`{_esc(f)}`](#{ods_anchor})** | {_esc(purpose)} | {conf} | {desc} | **⚠️ 无 DWD 目标** |")
|
||||
w(f"| {i} | **[`{_esc(f)}`](#{ods_anchor})** | {desc} | **⚠️ 无 DWD 目标** |")
|
||||
w()
|
||||
|
||||
# ── ODS→DWD 白名单汇总 ──
|
||||
_write_whitelist_summary(w, row["ods_no_dwd_whitelist"], "ODS→DWD ")
|
||||
|
||||
# ── DWD 无 ODS 源 ──
|
||||
if row["dwd_no_ods_fields"]:
|
||||
w(f"**DWD 无 ODS 源** — {len(row['dwd_no_ods_fields'])} 个")
|
||||
w()
|
||||
w("| # | DWD 表 | DWD 列 | 推测用途 | 置信度 | 说明 | 状态 |")
|
||||
w("|---|-------|-------|---------|-------|------|------|")
|
||||
w("| # | DWD 表 | DWD 列 | 说明 | 状态 |")
|
||||
w("|---|-------|-------|------|------|")
|
||||
for i, (dwd_name, dwd_col) in enumerate(row["dwd_no_ods_fields"], 1):
|
||||
dwd_a = dwd_anchors.get(dwd_name, f"dwd-{dwd_name.replace('_', '-')}")
|
||||
purpose, conf = _guess_field_purpose(dwd_col, table_name, "DWD")
|
||||
desc = _desc_str(dwd_col, "DWD", dwd_tbl=dwd_name)
|
||||
w(f"| {i} | {dwd_name} | **[`{_esc(dwd_col)}`](#{dwd_a})** | {_esc(purpose)} | {conf} | {desc} | **⚠️ 无 ODS 源** |")
|
||||
w(f"| {i} | {dwd_name} | **[`{_esc(dwd_col)}`](#{dwd_a})** | {desc} | **⚠️ 无 ODS 源** |")
|
||||
w()
|
||||
|
||||
# ── DWD 无 ODS 源 白名单汇总 ──
|
||||
_write_whitelist_summary(w, row["dwd_no_ods_whitelist"], "DWD 无 ODS 源 ")
|
||||
|
||||
w()
|
||||
|
||||
|
||||
|
||||
|
||||
def _write_api_section(w, fm, jt, bd, table_name, api_anchor, ods_anchor):
|
||||
"""生成 API 源字段区块（增加业务描述列，合并说明+示例值）"""
|
||||
"""生成 API 源字段区块（增加业务描述列，合并说明+示例值，白名单字段折叠）"""
|
||||
w(f'<a id="{api_anchor}"></a>')
|
||||
w()
|
||||
w(f"#### API 源字段 — {table_name} [🔗 ODS](#{ods_anchor})")
|
||||
@@ -556,17 +651,30 @@ def _write_api_section(w, fm, jt, bd, table_name, api_anchor, ods_anchor):
|
||||
# BD_manual ODS 描述（用于交叉引用 JSON 字段的业务含义）
|
||||
ods_descs = bd.get("ods_fields", {}) if bd else {}
|
||||
|
||||
# CHANGE 2026-02-21 | 白名单字段从表格中排除，折叠汇总
|
||||
normal_items: list[dict] = []
|
||||
whitelist_items: list[tuple[str, str]] = [] # (json_path, reason)
|
||||
for m in j2o:
|
||||
jp = m.get("json_path", "")
|
||||
wl_reason = whitelist_reason("", json_path=jp)
|
||||
if wl_reason:
|
||||
whitelist_items.append((jp, wl_reason))
|
||||
else:
|
||||
normal_items.append(m)
|
||||
|
||||
mapped_count = sum(1 for m in j2o if m.get("ods_col") is not None)
|
||||
total_count = len(j2o)
|
||||
if total_count > 0:
|
||||
w(f"已映射 {mapped_count}/{total_count}，覆盖率 {mapped_count / total_count * 100:.1f}%")
|
||||
if whitelist_items:
|
||||
w(f"（其中 {len(whitelist_items)} 个白名单字段已折叠）")
|
||||
else:
|
||||
w("无字段")
|
||||
w()
|
||||
w("| # | JSON 字段 | 类型 | → ODS 列 | 业务描述 | 示例值与说明 |")
|
||||
w("|---|----------|------|---------|---------|------------|")
|
||||
|
||||
for i, m in enumerate(j2o, 1):
|
||||
for i, m in enumerate(normal_items, 1):
|
||||
json_path = m["json_path"]
|
||||
json_type = m.get("json_type", "")
|
||||
ods_col = m.get("ods_col")
|
||||
@@ -597,7 +705,7 @@ def _write_api_section(w, fm, jt, bd, table_name, api_anchor, ods_anchor):
|
||||
|
||||
# 合并说明+示例值
|
||||
notes_parts: list[str] = []
|
||||
if json_path.startswith("siteProfile.") or ("." in json_path and match_type == "unmapped"):
|
||||
if "." in json_path and match_type == "unmapped":
|
||||
notes_parts.append("📦 嵌套对象")
|
||||
if match_type == "case_insensitive":
|
||||
notes_parts.append("大小写匹配")
|
||||
@@ -616,9 +724,20 @@ def _write_api_section(w, fm, jt, bd, table_name, api_anchor, ods_anchor):
|
||||
|
||||
w()
|
||||
|
||||
# 白名单字段折叠汇总
|
||||
if whitelist_items:
|
||||
by_reason: dict[str, list[str]] = {}
|
||||
for jp, reason in whitelist_items:
|
||||
by_reason.setdefault(reason, []).append(jp)
|
||||
parts = []
|
||||
for reason, names in by_reason.items():
|
||||
parts.append(f"{reason}: `{'`, `'.join(names[:5])}`{'...' if len(names) > 5 else ''} ({len(names)} 个)")
|
||||
w(f"> ℹ️ 白名单字段（已检查，不展开详情）：{'；'.join(parts)}")
|
||||
w()
|
||||
|
||||
|
||||
def _write_ods_section(w, fm, ods_schema, bd, table_name, ods_anchor, api_anchor, dwd_anchors):
|
||||
"""生成 ODS 表结构区块（含上下游双向映射列 + 业务描述）"""
|
||||
"""生成 ODS 表结构区块（含上下游双向映射列 + 业务描述，白名单字段折叠）"""
|
||||
w(f'<a id="{ods_anchor}"></a>')
|
||||
w()
|
||||
w(f"#### ODS 表结构 — ods.{table_name} [🔗 API](#{api_anchor})")
|
||||
@@ -645,12 +764,25 @@ def _write_ods_section(w, fm, ods_schema, bd, table_name, ods_anchor, api_anchor
|
||||
ods_descs = bd.get("ods_fields", {}) if bd else {}
|
||||
|
||||
cols = ods_schema["columns"]
|
||||
|
||||
# CHANGE 2026-02-21 | 白名单字段从表格中排除，折叠汇总
|
||||
normal_cols: list[dict] = []
|
||||
whitelist_cols: list[tuple[str, str]] = [] # (col_name, reason)
|
||||
for col in cols:
|
||||
wl_reason = whitelist_reason(col["name"])
|
||||
if wl_reason:
|
||||
whitelist_cols.append((col["name"], wl_reason))
|
||||
else:
|
||||
normal_cols.append(col)
|
||||
|
||||
w(f"共 {len(cols)} 列")
|
||||
if whitelist_cols:
|
||||
w(f"（其中 {len(whitelist_cols)} 个白名单列已折叠）")
|
||||
w()
|
||||
w("| # | ODS 列名 | 类型 | ← JSON 源 | → DWD 目标 | 业务描述 |")
|
||||
w("|---|---------|------|----------|-----------|---------|")
|
||||
|
||||
for i, col in enumerate(cols, 1):
|
||||
for i, col in enumerate(normal_cols, 1):
|
||||
col_name = col["name"]
|
||||
col_type = col["data_type"]
|
||||
|
||||
@@ -684,9 +816,20 @@ def _write_ods_section(w, fm, ods_schema, bd, table_name, ods_anchor, api_anchor
|
||||
|
||||
w()
|
||||
|
||||
# 白名单列折叠汇总
|
||||
if whitelist_cols:
|
||||
by_reason: dict[str, list[str]] = {}
|
||||
for cn, reason in whitelist_cols:
|
||||
by_reason.setdefault(reason, []).append(cn)
|
||||
parts = []
|
||||
for reason, names in by_reason.items():
|
||||
parts.append(f"{reason}: `{'`, `'.join(names)}` ({len(names)} 个)")
|
||||
w(f"> ℹ️ 白名单列（已检查，不展开详情）：{'；'.join(parts)}")
|
||||
w()
|
||||
|
||||
|
||||
def _write_dwd_section(w, fm, dwd_schema, bd, dwd_name, dwd_anchor, ods_anchor, table_name):
|
||||
"""生成 DWD 表结构区块（增加业务描述列）"""
|
||||
"""生成 DWD 表结构区块（增加业务描述列，白名单字段折叠）"""
|
||||
w(f'<a id="{dwd_anchor}"></a>')
|
||||
w()
|
||||
w(f"#### DWD 表结构 — dwd.{dwd_name} [🔗 ODS](#{ods_anchor})")
|
||||
@@ -709,12 +852,25 @@ def _write_dwd_section(w, fm, dwd_schema, bd, dwd_name, dwd_anchor, ods_anchor,
|
||||
dwd_descs = bd["dwd_fields"].get(dwd_name, {})
|
||||
|
||||
cols = dwd_schema["columns"]
|
||||
|
||||
# CHANGE 2026-02-21 | 白名单字段从表格中排除，折叠汇总
|
||||
normal_cols: list[dict] = []
|
||||
whitelist_cols: list[tuple[str, str]] = [] # (col_name, reason)
|
||||
for col in cols:
|
||||
wl_reason = whitelist_reason(col["name"])
|
||||
if wl_reason:
|
||||
whitelist_cols.append((col["name"], wl_reason))
|
||||
else:
|
||||
normal_cols.append(col)
|
||||
|
||||
w(f"共 {len(cols)} 列")
|
||||
if whitelist_cols:
|
||||
w(f"（其中 {len(whitelist_cols)} 个白名单列已折叠）")
|
||||
w()
|
||||
w("| # | DWD 列名 | 类型 | ← ODS 来源 | 转换 | 业务描述 |")
|
||||
w("|---|---------|------|----------|------|---------|")
|
||||
|
||||
for i, col in enumerate(cols, 1):
|
||||
for i, col in enumerate(normal_cols, 1):
|
||||
col_name = col["name"]
|
||||
col_type = col["data_type"]
|
||||
|
||||
@@ -728,8 +884,6 @@ def _write_dwd_section(w, fm, dwd_schema, bd, dwd_name, dwd_anchor, ods_anchor,
|
||||
ods_link = "—"
|
||||
transform = ""
|
||||
note = ""
|
||||
if col_name in ("valid_from", "valid_to", "is_current", "etl_loaded_at", "etl_batch_id"):
|
||||
transform = "ETL 生成"
|
||||
|
||||
# 业务描述（优先 BD_manual，其次 mapping note，最后 DB comment）
|
||||
biz_desc = dwd_descs.get(col_name.lower(), "")
|
||||
@@ -753,9 +907,22 @@ def _write_dwd_section(w, fm, dwd_schema, bd, dwd_name, dwd_anchor, ods_anchor,
|
||||
|
||||
w()
|
||||
|
||||
# 白名单列折叠汇总
|
||||
if whitelist_cols:
|
||||
by_reason: dict[str, list[str]] = {}
|
||||
for cn, reason in whitelist_cols:
|
||||
by_reason.setdefault(reason, []).append(cn)
|
||||
parts = []
|
||||
for reason, names in by_reason.items():
|
||||
parts.append(f"{reason}: `{'`, `'.join(names)}` ({len(names)} 个)")
|
||||
w(f"> ℹ️ 白名单列（已检查，不展开详情）：{'；'.join(parts)}")
|
||||
w()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_dotenv(Path(".env"), override=False)
|
||||
# _env_paths 在 import 时已通过绝对路径加载根 .env，无需相对路径 load_dotenv
|
||||
# CHANGE 2026-02-21 | 移除 load_dotenv(Path(".env"))，避免 cwd 不在项目根时失效
|
||||
from _env_paths import get_output_path # noqa: F401 — 触发 .env 加载
|
||||
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
Reference in New Issue
Block a user