"""
字段排查脚本 — 数据流字段补全 Spec Task 1.1

对 12 张目标表执行排查流程：
1. 查 DWD 现有列
2. 查 ODS 现有列
3. 解析 FACT_MAPPINGS 现状（从 dwd_load_task.py 源码导入）
4. 判断自动映射（ODS 列名 == DWD 列名）
5. 输出排查记录表（markdown），标注每个字段的排查结论和建议操作

用法:
    cd C:\\NeoZQYY
    python scripts/ops/field_audit.py
    python scripts/ops/field_audit.py --output path/to/output.md
"""

from __future__ import annotations

import argparse
import os
import sys
from datetime import datetime
from pathlib import Path

import psycopg2
from psycopg2.extras import RealDictCursor
from dotenv import load_dotenv

# ── 项目根目录 & 路径设置 ──
ROOT = Path(__file__).resolve().parents[2]
ETL_ROOT = ROOT / "apps" / "etl" / "connectors" / "feiqiu"
sys.path.insert(0, str(ETL_ROOT))

# 导入 FACT_MAPPINGS / TABLE_MAP（仅读取类属性，不实例化）
from tasks.dwd.dwd_load_task import DwdLoadTask

# ── SCD2 列集合（排查时忽略） ──
SCD2_COLS = {"scd2_start_time", "scd2_end_time", "scd2_is_current", "scd2_version"}

# ── 需要排查的表及其疑似缺失字段 ──
AUDIT_TARGETS: list[dict] = [
    {
        "ods_table": "assistant_accounts_master",
        "dwd_tables": ["dim_assistant", "dim_assistant_ex"],
        "suspect_ods_cols": ["system_role_id", "job_num", "cx_unit_price", "pd_unit_price"],
        "category": "A",
        "notes": "4 个 ODS→DWD 未映射",
    },
    {
        "ods_table": "assistant_service_records",
        "dwd_tables": ["dwd_assistant_service_log", "dwd_assistant_service_log_ex"],
        "suspect_ods_cols": ["site_assistant_id", "operator_id", "operator_name"],
        "category": "A",
        "notes": "3 个 ODS→DWD 未映射（site_assistant_id 可能已映射为 order_assistant_id）",
    },
    {
        "ods_table": "store_goods_sales_records",
        "dwd_tables": ["dwd_store_goods_sale", "dwd_store_goods_sale_ex"],
        "suspect_ods_cols": ["discount_price"],
        "category": "A",
        "notes": "1 个 ODS→DWD 未映射（可能已映射为 discount_money）",
    },
    {
        "ods_table": "member_balance_changes",
        "dwd_tables": ["dwd_member_balance_change", "dwd_member_balance_change_ex"],
        "suspect_ods_cols": ["relate_id"],
        "category": "A",
        "notes": "1 个 ODS→DWD 未映射",
    },
    {
        "ods_table": "tenant_goods_master",
        "dwd_tables": ["dim_tenant_goods", "dim_tenant_goods_ex"],
        "suspect_ods_cols": ["commoditycode"],
        "category": "A",
        "notes": "1 个 ODS→DWD 未映射（可能已映射为 commodity_code_list）",
    },
    {
        "ods_table": "site_tables_master",
        "dwd_tables": ["dim_table", "dim_table_ex"],
        "suspect_ods_cols": [
            "sitename", "appletqrcodeurl", "audit_status", "charge_free",
            "create_time", "delay_lights_time", "is_rest_area", "light_status",
            "only_allow_groupon", "order_delay_time", "self_table",
            "tablestatusname", "temporary_light_second", "virtual_table",
        ],
        "category": "A",
        "notes": "14 个 ODS→DWD 未映射",
    },
    {
        "ods_table": "recharge_settlements",
        "dwd_tables": ["dwd_recharge_order", "dwd_recharge_order_ex"],
        "suspect_ods_cols": [
            "electricityadjustmoney", "electricitymoney",
            "mervousalesamount", "plcouponsaleamount", "realelectricitymoney",
        ],
        "category": "B",
        "notes": "5 个 ODS→DWD 未映射 + 5 个 DWD 无 ODS 源（驼峰/蛇形命名差异）",
    },
    {
        "ods_table": "store_goods_master",
        "dwd_tables": ["dim_store_goods", "dim_store_goods_ex"],
        "suspect_ods_cols": [
            "time_slot_sale", "batch_stock_quantity", "provisional_total_cost",
        ],
        "category": "B",
        "notes": "平层 + 嵌套展开 + ODS→DWD 补全",
    },
    {
        "ods_table": "goods_stock_summary",
        "dwd_tables": [],  # 无 DWD 表，需新建
        "suspect_ods_cols": [
            "sitegoodsid", "goodsname", "goodsunit", "goodscategoryid",
            "goodscategorysecondid", "categoryname", "rangestartstock",
            "rangeendstock", "rangein", "rangeout", "rangesale",
            "rangesalemoney", "rangeinventory", "currentstock",
        ],
        "category": "C",
        "notes": "14 个 ODS 字段，无 DWD 目标表，需新建",
    },
    {
        "ods_table": "goods_stock_movements",
        "dwd_tables": [],  # 无 DWD 表，需新建
        "suspect_ods_cols": [
            # ODS 实际列名为驼峰式（无下划线）
            "sitegoodsstockid", "tenantid", "siteid", "sitegoodsid",
            "goodsname", "goodscategoryid", "goodssecondcategoryid",
            "unit", "price", "stocktype", "changenum", "startnum",
            "endnum", "changenuma", "startnuma", "endnuma",
            "remark", "operatorname", "createtime",
        ],
        "category": "C",
        "notes": "19 个 ODS 字段，无 DWD 目标表，需新建",
    },
]

# ── recharge_settlements 已知的 DWD 无 ODS 源字段（用于交叉比对） ──
RECHARGE_DWD_ORPHANS = [
    "pl_coupon_sale_amount", "mervou_sales_amount",
    "electricity_money", "real_electricity_money", "electricity_adjust_money",
]


def get_db_columns(cur, schema: str, table: str) -> list[str]:
    """查询数据库表的列名列表（小写）。"""
    cur.execute(
        "SELECT column_name FROM information_schema.columns "
        "WHERE table_schema = %s AND table_name = %s ORDER BY ordinal_position",
        (schema, table),
    )
    return [r["column_name"].lower() for r in cur.fetchall()]


def get_sample_values(conn, schema: str, table: str, column: str, limit: int = 5) -> list:
    """获取指定列的非空采样值（最多 limit 个）。失败时回滚并返回空列表。"""
    cur = conn.cursor(cursor_factory=RealDictCursor)
    try:
        cur.execute(
            f'SELECT DISTINCT "{column}" FROM "{schema}"."{table}" '
            f'WHERE "{column}" IS NOT NULL LIMIT %s',
            (limit,),
        )
        return [r[column] for r in cur.fetchall()]
    except Exception:
        conn.rollback()
        return []
    finally:
        cur.close()


def parse_fact_mappings() -> dict[str, dict[str, str]]:
    """
    解析 FACT_MAPPINGS，返回 {dwd_full_table: {dwd_col: ods_expr}} 的映射。
    同时构建反向索引 {dwd_full_table: {ods_expr_lower: dwd_col}}。
    """
    forward: dict[str, dict[str, str]] = {}
    reverse: dict[str, dict[str, str]] = {}
    for dwd_table, entries in DwdLoadTask.FACT_MAPPINGS.items():
        fwd = {}
        rev = {}
        for dwd_col, ods_expr, _cast in entries:
            fwd[dwd_col.lower()] = ods_expr
            # 反向索引：ods 表达式 → dwd 列名
            # 处理简单列名和 JSON 表达式
            ods_key = ods_expr.lower().strip('"')
            rev[ods_key] = dwd_col.lower()
        forward[dwd_table] = fwd
        reverse[dwd_table] = rev
    return forward, reverse


def audit_one_table(
    conn,
    target: dict,
    fm_forward: dict,
    fm_reverse: dict,
) -> list[dict]:
    """
    对单张表执行排查，返回排查记录列表。
    每条记录: {ods_col, dwd_table, dwd_col_match, fm_status, conclusion, action, samples}
    """
    cur = conn.cursor(cursor_factory=RealDictCursor)
    ods_table = target["ods_table"]
    dwd_tables = target["dwd_tables"]
    suspect_cols = target["suspect_ods_cols"]

    # 查 ODS 现有列
    ods_cols = set(get_db_columns(cur, "ods", ods_table))

    # 查各 DWD 表现有列
    dwd_cols_map: dict[str, set[str]] = {}
    for dt in dwd_tables:
        dwd_cols_map[dt] = set(get_db_columns(cur, "dwd", dt))

    records = []
    for ods_col in suspect_cols:
        ods_col_lower = ods_col.lower()
        record = {
            "ods_col": ods_col_lower,
            "ods_exists": ods_col_lower in ods_cols,
            "dwd_matches": [],
            "fm_status": "未配置",
            "conclusion": "",
            "action": "",
            "samples": [],
        }

        # 采样值
        if record["ods_exists"]:
            record["samples"] = get_sample_values(conn, "ods", ods_table, ods_col_lower)

        # 遍历所有关联 DWD 表检查
        for dt in dwd_tables:
            dwd_full = f"dwd.{dt}"
            dwd_cols = dwd_cols_map.get(dt, set())
            fm_fwd = fm_forward.get(dwd_full, {})
            fm_rev = fm_reverse.get(dwd_full, {})

            # 检查 1: FACT_MAPPINGS 反向索引 — ODS 列是否已被映射
            if ods_col_lower in fm_rev:
                mapped_to = fm_rev[ods_col_lower]
                record["dwd_matches"].append(f"{dt}.{mapped_to}")
                record["fm_status"] = f"已映射 → {dt}.{mapped_to}"
                record["conclusion"] = "已映射（FACT_MAPPINGS 显式配置）"
                record["action"] = "无需变更"
                break

            # 检查 2: DWD 表中是否有同名列（自动映射）
            if ods_col_lower in dwd_cols:
                record["dwd_matches"].append(f"{dt}.{ods_col_lower}")
                record["fm_status"] = "自动映射（同名列）"
                record["conclusion"] = "已映射（自动匹配）"
                record["action"] = "无需变更"
                break

            # 检查 3: DWD 表中是否有近似列名（蛇形/驼峰转换）
            snake = _camel_to_snake(ods_col_lower)
            if snake != ods_col_lower and snake in dwd_cols:
                record["dwd_matches"].append(f"{dt}.{snake}")
                # 还需检查 FACT_MAPPINGS 是否已配置此映射
                if snake in fm_fwd:
                    record["fm_status"] = f"已映射 → {dt}.{snake}（命名转换）"
                    record["conclusion"] = "已映射（命名差异，FACT_MAPPINGS 已覆盖）"
                    record["action"] = "无需变更"
                else:
                    record["fm_status"] = f"DWD 列存在 {dt}.{snake}，但 FACT_MAPPINGS 未配置"
                    record["conclusion"] = "映射遗漏（DWD 列已存在，缺 FACT_MAPPINGS）"
                    record["action"] = "仅补充 FACT_MAPPINGS"
                break
        else:
            # 所有 DWD 表都没找到匹配
            if not record["ods_exists"]:
                record["conclusion"] = "ODS 列不存在"
                record["action"] = "需确认 API 是否返回该字段"
            elif not dwd_tables:
                record["conclusion"] = "无 DWD 目标表"
                record["action"] = "需新建 DWD 表"
            else:
                record["conclusion"] = "确实缺失"
                record["action"] = "需新增 DWD 列 + FACT_MAPPINGS"

        records.append(record)

    # 额外排查：recharge_settlements 的 DWD 无 ODS 源字段
    if ods_table == "recharge_settlements":
        for dwd_orphan in RECHARGE_DWD_ORPHANS:
            orphan_record = {
                "ods_col": f"(DWD orphan) {dwd_orphan}",
                "ods_exists": False,
                "dwd_matches": [],
                "fm_status": "",
                "conclusion": "",
                "action": "",
                "samples": [],
            }
            # 检查是否已在 FACT_MAPPINGS 中被映射
            for dt in dwd_tables:
                dwd_full = f"dwd.{dt}"
                fm_fwd = fm_forward.get(dwd_full, {})
                if dwd_orphan in fm_fwd:
                    src = fm_fwd[dwd_orphan]
                    orphan_record["fm_status"] = f"已映射 ← {src}"
                    orphan_record["conclusion"] = "已映射（FACT_MAPPINGS 已覆盖）"
                    orphan_record["action"] = "无需变更"
                    orphan_record["dwd_matches"].append(f"{dt}.{dwd_orphan}")
                    break
            else:
                orphan_record["conclusion"] = "DWD 列存在但无 ODS 映射"
                orphan_record["action"] = "需补充 FACT_MAPPINGS"
            records.append(orphan_record)

    return records


def _camel_to_snake(name: str) -> str:
    """简易驼峰转蛇形：在大写字母前插入下划线。"""
    import re
    s1 = re.sub(r"([A-Z])", r"_\1", name)
    return s1.lower().lstrip("_")



def generate_report(all_results: dict[str, list[dict]]) -> str:
    """生成 Markdown 排查报告。"""
    lines: list[str] = []
    now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    lines.append(f"# 字段排查报告\n")
    lines.append(f"> 生成时间：{now_str}\n")
    lines.append(f"> 排查范围：{len(all_results)} 张表\n")

    # 汇总统计
    total_fields = 0
    already_mapped = 0
    need_fm_only = 0
    need_new_col = 0
    need_new_table = 0
    ods_missing = 0

    for table, records in all_results.items():
        for r in records:
            total_fields += 1
            action = r["action"]
            if "无需变更" in action:
                already_mapped += 1
            elif "仅补充" in action:
                need_fm_only += 1
            elif "新增 DWD 列" in action:
                need_new_col += 1
            elif "新建 DWD 表" in action:
                need_new_table += 1
            elif "需确认" in action:
                ods_missing += 1

    lines.append("\n## 汇总\n")
    lines.append(f"| 指标 | 数量 |")
    lines.append(f"|------|------|")
    lines.append(f"| 排查字段总数 | {total_fields} |")
    lines.append(f"| 已映射（无需变更） | {already_mapped} |")
    lines.append(f"| 映射遗漏（仅补 FACT_MAPPINGS） | {need_fm_only} |")
    lines.append(f"| 确实缺失（需新增 DWD 列） | {need_new_col} |")
    lines.append(f"| 无 DWD 表（需新建） | {need_new_table} |")
    lines.append(f"| ODS 列不存在（需确认 API） | {ods_missing} |")

    # 逐表详情
    for target_info, records in all_results.items():
        ods_table, category, notes = target_info
        lines.append(f"\n---\n")
        lines.append(f"## {ods_table}（{category} 类）\n")
        lines.append(f"> {notes}\n")
        lines.append(f"| # | ODS 列 | ODS 存在 | DWD 匹配 | FACT_MAPPINGS 状态 | 排查结论 | 建议操作 | 采样值 |")
        lines.append(f"|---|--------|---------|---------|-------------------|---------|---------|--------|")
        for i, r in enumerate(records, 1):
            ods_exists = "✅" if r["ods_exists"] else "❌"
            dwd_match = ", ".join(r["dwd_matches"]) if r["dwd_matches"] else "—"
            samples_str = ", ".join(str(s)[:30] for s in r["samples"][:3]) if r["samples"] else "—"
            lines.append(
                f"| {i} | `{r['ods_col']}` | {ods_exists} | {dwd_match} "
                f"| {r['fm_status']} | {r['conclusion']} | **{r['action']}** | {samples_str} |"
            )

    # TABLE_MAP 覆盖检查
    lines.append(f"\n---\n")
    lines.append(f"## TABLE_MAP 注册状态\n")
    lines.append(f"| DWD 表 | ODS 源表 | 已注册 |")
    lines.append(f"|--------|---------|--------|")
    for target in AUDIT_TARGETS:
        for dt in target["dwd_tables"]:
            dwd_full = f"dwd.{dt}"
            ods_full = f"ods.{target['ods_table']}"
            registered = dwd_full in DwdLoadTask.TABLE_MAP
            reg_str = "✅" if registered else "❌ 未注册"
            if registered:
                actual_ods = DwdLoadTask.TABLE_MAP[dwd_full]
                if actual_ods != ods_full:
                    reg_str = f"⚠️ 映射到 {actual_ods}"
            lines.append(f"| `{dwd_full}` | `{ods_full}` | {reg_str} |")
    # C 类无 DWD 表的
    for target in AUDIT_TARGETS:
        if not target["dwd_tables"]:
            lines.append(f"| （待新建） | `ods.{target['ods_table']}` | ❌ 无 DWD 表 |")

    return "\n".join(lines)


def main():
    parser = argparse.ArgumentParser(description="字段排查脚本")
    parser.add_argument(
        "--output", type=str, default=None,
        help="输出文件路径（默认 $FIELD_AUDIT_ROOT/field_audit_report.md）",
    )
    args = parser.parse_args()

    # 加载环境变量
    load_dotenv(ROOT / ".env")
    load_dotenv(ROOT / ".env.local", override=True)

    dsn = os.environ.get("PG_DSN")
    if not dsn:
        print("错误：未配置 PG_DSN 环境变量", file=sys.stderr)
        sys.exit(1)

    print(f"连接数据库...")
    conn = psycopg2.connect(dsn)
    conn.autocommit = True

    print(f"解析 FACT_MAPPINGS...")
    fm_forward, fm_reverse = parse_fact_mappings()

    # 执行排查
    # key = (ods_table, category, notes) 用于报告分组
    all_results: dict[tuple, list[dict]] = {}
    for target in AUDIT_TARGETS:
        key = (target["ods_table"], target["category"], target["notes"])
        print(f"排查 {target['ods_table']}（{target['category']} 类）...")
        records = audit_one_table(conn, target, fm_forward, fm_reverse)
        all_results[key] = records
        # 打印简要结果
        for r in records:
            icon = "✅" if "无需变更" in r["action"] else "⚠️"
            print(f"  {icon} {r['ods_col']}: {r['conclusion']} → {r['action']}")

    conn.close()

    # 生成报告
    report = generate_report(all_results)
    # 从 .env 读取 FIELD_AUDIT_ROOT
    from _env_paths import get_output_path
    default_dir = get_output_path("FIELD_AUDIT_ROOT")
    output_path = Path(args.output) if args.output else default_dir / "field_audit_report.md"
    output_path.parent.mkdir(parents=True, exist_ok=True)
    output_path.write_text(report, encoding="utf-8")
    print(f"\n排查报告已生成：{output_path}")


if __name__ == "__main__":
    main()
