# -*- coding: utf-8 -*-
r"""
黑盒集成测试报告 -- 从 API 输入侧与 DB 输出侧（ODS/DWD/DWS）进行全链路对比。

用法:
    cd C:\NeoZQYY
    uv run python scripts/ops/blackbox_test_report.py

输出: ETL_REPORT_ROOT / blackbox_report_<timestamp>.md
"""
from __future__ import annotations

import json
import os
import re
import sys
from datetime import datetime
from pathlib import Path
from zoneinfo import ZoneInfo

from dotenv import load_dotenv

_ROOT = Path(__file__).resolve().parents[2]
load_dotenv(_ROOT / ".env", override=False)

# ── 路径 ──────────────────────────────────────────────────────────────
def _env(var: str) -> Path:
    val = os.environ.get(var)
    if not val:
        raise KeyError(f"环境变量 {var} 未定义")
    p = Path(val)
    p.mkdir(parents=True, exist_ok=True)
    return p

REPORT_ROOT = _env("ETL_REPORT_ROOT")
JSON_ROOT   = _env("FETCH_ROOT")
LOG_ROOT    = _env("LOG_ROOT")

TZ = ZoneInfo("Asia/Shanghai")
NOW = datetime.now(TZ)
TS  = NOW.strftime("%Y%m%d_%H%M%S")

import psycopg2
import psycopg2.extras

DSN = os.environ["PG_DSN"]

# ── 1. 解析 ETL 日志 ─────────────────────────────────────────────────
def find_latest_log() -> Path | None:
    """找到最新的 ETL 日志文件"""
    logs = sorted(LOG_ROOT.glob("*.log"), key=lambda p: p.stat().st_mtime, reverse=True)
    return logs[0] if logs else None

def parse_etl_log(log_path: Path) -> dict:
    """解析 ETL 日志，提取任务执行结果"""
    results = {}
    current_task = None
    task_start_times = {}

    with open(log_path, "r", encoding="utf-8") as f:
        for line in f:
            # 匹配任务开始
            m = re.match(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*开始执行(\w+) \(ODS\)", line)
            if m:
                current_task = m.group(2)
                task_start_times[current_task] = m.group(1)
                continue

            # 匹配 ODS 任务完成
            m = re.match(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*?(\w+) ODS 任务完成: (\{.*\})", line)
            if m:
                task_name = m.group(2)
                end_time = m.group(1)
                try:
                    counts = eval(m.group(3))
                except Exception:
                    counts = {}
                start_time = task_start_times.get(task_name, "")
                results[task_name] = {
                    "status": "SUCC",
                    "layer": "ODS",
                    "start": start_time,
                    "end": end_time,
                    "counts": counts,
                }
                continue

            # 匹配 DWD 完成
            m = re.match(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*DWD_LOAD_FROM_ODS.*完成.*?(\d+).*表", line)
            if m:
                results["DWD_LOAD_FROM_ODS"] = {
                    "status": "SUCC",
                    "layer": "DWD",
                    "end": m.group(1),
                }
                continue

            # 匹配 DWS/INDEX 成功
            m = re.match(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*?(\w+): 工具类任务执行成功", line)
            if m:
                results[m.group(2)] = {
                    "status": "SUCC",
                    "layer": "DWS/INDEX",
                    "end": m.group(1),
                }
                continue

            # 匹配任务失败
            m = re.match(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*?(\w+): 工具类任务执行失败: (.*)", line)
            if m:
                results[m.group(2)] = {
                    "status": "FAIL",
                    "layer": "DWS/INDEX",
                    "end": m.group(1),
                    "error": m.group(3)[:120],
                }
                continue

            # 匹配"未启用或不存在"
            m = re.match(r".*任务 (\w+) 未启用或不存在", line)
            if m:
                results[m.group(1)] = {
                    "status": "SKIP",
                    "layer": "DWS",
                    "error": "未注册",
                }

    return results

# ── 2. API 输入侧：统计 JSON 落地行数 ────────────────────────────────
def count_api_json_records(task_name: str) -> int | None:
    """统计某个 ODS 任务最新一次 JSON 落地的总记录数"""
    task_dir = JSON_ROOT / task_name
    if not task_dir.exists():
        return None
    # 找最新的子目录（按名称排序，格式 TASK-SITEID-DATE-TIME）
    subdirs = sorted(task_dir.iterdir(), key=lambda p: p.name, reverse=True)
    # 只取今天的
    today_str = NOW.strftime("%Y%m%d")
    for sd in subdirs:
        if today_str in sd.name and sd.is_dir():
            total = 0
            for jf in sd.glob("*.json"):
                try:
                    data = json.loads(jf.read_text(encoding="utf-8"))
                    if isinstance(data, list):
                        total += len(data)
                    elif isinstance(data, dict):
                        # 可能是 {"data": {"list": [...]}} 格式
                        lst = data.get("data", {}).get("list", data.get("data", []))
                        if isinstance(lst, list):
                            total += len(lst)
                        else:
                            total += 1
                except Exception:
                    pass
            return total
    return None


# ── 3. DB 输出侧：各层行数统计 ───────────────────────────────────────
ODS_TABLES = [
    "assistant_accounts_master", "assistant_service_records",
    "settlement_records",
    "table_fee_transactions", "table_fee_discount_records",
    "site_tables_master", "payment_transactions", "refund_transactions",
    "platform_coupon_redemption_records", "member_profiles",
    "member_stored_value_cards", "member_balance_changes",
    "recharge_settlements", "group_buy_packages",
    "group_buy_redemption_records", "goods_stock_summary",
    "goods_stock_movements", "stock_goods_category_tree",
    "store_goods_master", "store_goods_sales_records", "tenant_goods_master",
]

# ODS 任务名 → ODS 表名映射
ODS_TASK_TO_TABLE = {
    "ODS_ASSISTANT_ACCOUNT": "assistant_accounts_master",
    "ODS_ASSISTANT_LEDGER": "assistant_service_records",
    "ODS_SETTLEMENT_RECORDS": "settlement_records",
    "ODS_TABLE_USE": "table_fee_transactions",
    "ODS_TABLE_FEE_DISCOUNT": "table_fee_discount_records",
    "ODS_TABLES": "site_tables_master",
    "ODS_PAYMENT": "payment_transactions",
    "ODS_REFUND": "refund_transactions",
    "ODS_PLATFORM_COUPON": "platform_coupon_redemption_records",
    "ODS_MEMBER": "member_profiles",
    "ODS_MEMBER_CARD": "member_stored_value_cards",
    "ODS_MEMBER_BALANCE": "member_balance_changes",
    "ODS_RECHARGE_SETTLE": "recharge_settlements",
    "ODS_GROUP_PACKAGE": "group_buy_packages",
    "ODS_GROUP_BUY_REDEMPTION": "group_buy_redemption_records",
    "ODS_INVENTORY_STOCK": "goods_stock_summary",
    "ODS_INVENTORY_CHANGE": "goods_stock_movements",
    "ODS_GOODS_CATEGORY": "stock_goods_category_tree",
    "ODS_STORE_GOODS": "store_goods_master",
    "ODS_STORE_GOODS_SALES": "store_goods_sales_records",
    "ODS_TENANT_GOODS": "tenant_goods_master",
}

def query_row_counts(conn, schema: str, tables: list[str]) -> dict[str, int]:
    """批量查询各表行数"""
    result = {}
    with conn.cursor() as cur:
        for t in tables:
            try:
                cur.execute(f"SELECT COUNT(*) FROM {schema}.{t}")
                result[t] = cur.fetchone()[0]
            except Exception:
                conn.rollback()
                result[t] = -1
    return result

def query_dwd_tables(conn) -> list[str]:
    """获取 dwd schema 下所有表"""
    with conn.cursor() as cur:
        cur.execute("""
            SELECT table_name FROM information_schema.tables
            WHERE table_schema = 'dwd' AND table_type = 'BASE TABLE'
            ORDER BY table_name
        """)
        return [r[0] for r in cur.fetchall()]

def query_dws_tables(conn) -> list[str]:
    """获取 dws schema 下所有表"""
    with conn.cursor() as cur:
        cur.execute("""
            SELECT table_name FROM information_schema.tables
            WHERE table_schema = 'dws' AND table_type = 'BASE TABLE'
            ORDER BY table_name
        """)
        return [r[0] for r in cur.fetchall()]

# ── 4. ODS vs DWD 行数对比 ───────────────────────────────────────────
# DwdLoadTask.TABLE_MAP 的简化版（dwd_table → ods_table）
DWD_TO_ODS = {
    # 维度表 dim_*
    "dwd.dim_assistant": "ods.assistant_accounts_master",
    "dwd.dim_member": "ods.member_profiles",
    "dwd.dim_member_card_account": "ods.member_stored_value_cards",
    "dwd.dim_table": "ods.site_tables_master",
    "dwd.dim_groupbuy_package": "ods.group_buy_packages",
    "dwd.dim_store_goods": "ods.store_goods_master",
    "dwd.dim_tenant_goods": "ods.tenant_goods_master",
    "dwd.dim_goods_category": "ods.stock_goods_category_tree",
    # 事实表 dwd_*
    "dwd.dwd_assistant_service_log": "ods.assistant_service_records",
    "dwd.dwd_member_balance_change": "ods.member_balance_changes",
    "dwd.dwd_recharge_order": "ods.recharge_settlements",
    "dwd.dwd_settlement_head": "ods.settlement_records",
    "dwd.dwd_table_fee_log": "ods.table_fee_transactions",
    "dwd.dwd_table_fee_adjust": "ods.table_fee_discount_records",
    "dwd.dwd_payment": "ods.payment_transactions",
    "dwd.dwd_refund": "ods.refund_transactions",
    "dwd.dwd_platform_coupon_redemption": "ods.platform_coupon_redemption_records",
    "dwd.dwd_groupbuy_redemption": "ods.group_buy_redemption_records",
    "dwd.dwd_store_goods_sale": "ods.store_goods_sales_records",
}

def compare_ods_dwd(conn) -> list[dict]:
    """对比 ODS 与 DWD 行数"""
    rows = []
    with conn.cursor() as cur:
        for dwd_full, ods_full in sorted(DWD_TO_ODS.items()):
            dwd_s, dwd_t = dwd_full.split(".")
            ods_s, ods_t = ods_full.split(".")
            try:
                # ODS: 去重 id 计数（因为 content_hash 变化会产生多行）
                cur.execute(f"SELECT COUNT(DISTINCT id) FROM {ods_s}.{ods_t}")
                ods_distinct = cur.fetchone()[0]
                cur.execute(f"SELECT COUNT(*) FROM {ods_s}.{ods_t}")
                ods_total = cur.fetchone()[0]
            except Exception:
                conn.rollback()
                ods_distinct = -1
                ods_total = -1
            try:
                cur.execute(f"SELECT COUNT(*) FROM {dwd_s}.{dwd_t}")
                dwd_count = cur.fetchone()[0]
            except Exception:
                conn.rollback()
                dwd_count = -1

            # dim 表用 SCD2，行数可能 > ODS distinct id
            # fact 表行数应 ≈ ODS distinct id
            is_dim = dwd_t.startswith("dim_")
            rows.append({
                "dwd_table": dwd_full,
                "ods_table": ods_full,
                "ods_total_rows": ods_total,
                "ods_distinct_ids": ods_distinct,
                "dwd_rows": dwd_count,
                "type": "维度(SCD2)" if is_dim else "事实",
                "ratio": round(dwd_count / ods_distinct, 2) if ods_distinct > 0 else "N/A",
            })
    return rows


# ── 5. DWD 数据质量抽样 ──────────────────────────────────────────────
def check_dwd_null_rates(conn, tables: list[str], sample_cols: int = 5) -> list[dict]:
    """检查 DWD 表关键列的 NULL 率"""
    results = []
    with conn.cursor() as cur:
        for t in tables:
            try:
                cur.execute(f"""
                    SELECT column_name FROM information_schema.columns
                    WHERE table_schema = 'dwd' AND table_name = %s
                    AND column_name NOT IN ('created_at','updated_at','fetched_at','content_hash','record_index','source_file','source_endpoint','payload')
                    ORDER BY ordinal_position
                    LIMIT %s
                """, (t, sample_cols))
                cols = [r[0] for r in cur.fetchall()]
                if not cols:
                    continue

                cur.execute(f"SELECT COUNT(*) FROM dwd.{t}")
                total = cur.fetchone()[0]
                if total == 0:
                    results.append({"table": t, "total": 0, "null_cols": "空表"})
                    continue

                null_info = []
                for c in cols:
                    cur.execute(f"SELECT COUNT(*) FROM dwd.{t} WHERE {c} IS NULL")
                    null_count = cur.fetchone()[0]
                    rate = round(null_count / total * 100, 1)
                    if rate > 0:
                        null_info.append(f"{c}={rate}%")

                results.append({
                    "table": t,
                    "total": total,
                    "null_cols": ", ".join(null_info) if null_info else "无 NULL",
                })
            except Exception as e:
                conn.rollback()
                results.append({"table": t, "total": -1, "null_cols": str(e)[:80]})
    return results


# ── 6. DWS 汇总合理性检查 ────────────────────────────────────────────
def check_dws_sanity(conn) -> list[dict]:
    """DWS 表基本合理性检查"""
    checks = []
    with conn.cursor() as cur:
        # 检查各 DWS 表行数和最新数据日期
        dws_tables = query_dws_tables(conn)
        for t in dws_tables:
            try:
                cur.execute(f"SELECT COUNT(*) FROM dws.{t}")
                cnt = cur.fetchone()[0]

                # 尝试找日期列
                date_col = None
                for candidate in ["stat_date", "salary_month", "report_date", "calc_date", "snapshot_date", "stock_date"]:
                    cur.execute("""
                        SELECT 1 FROM information_schema.columns
                        WHERE table_schema='dws' AND table_name=%s AND column_name=%s
                    """, (t, candidate))
                    if cur.fetchone():
                        date_col = candidate
                        break

                latest = None
                if date_col and cnt > 0:
                    cur.execute(f"SELECT MAX({date_col}) FROM dws.{t}")
                    latest = cur.fetchone()[0]

                checks.append({
                    "table": f"dws.{t}",
                    "rows": cnt,
                    "latest_date": str(latest) if latest else "N/A",
                    "status": "✅" if cnt > 0 else "⚠️ 空表",
                })
            except Exception as e:
                conn.rollback()
                checks.append({
                    "table": f"dws.{t}",
                    "rows": -1,
                    "latest_date": "ERROR",
                    "status": f"❌ {str(e)[:60]}",
                })
    return checks

# ── 7. API JSON 输入侧 vs ODS 行数对比 ───────────────────────────────
def compare_api_vs_ods(conn, log_results: dict) -> list[dict]:
    """对比 API JSON 落地记录数 vs ODS 表行数（仅本次 ETL 涉及的任务）"""
    rows = []
    with conn.cursor() as cur:
        for task_name, ods_table in sorted(ODS_TASK_TO_TABLE.items()):
            log_info = log_results.get(task_name, {})
            api_fetched = log_info.get("counts", {}).get("fetched", None)
            api_json_count = count_api_json_records(task_name)

            try:
                cur.execute(f"SELECT COUNT(*) FROM ods.{ods_table}")
                ods_total = cur.fetchone()[0]
                cur.execute(f"SELECT COUNT(DISTINCT id) FROM ods.{ods_table}")
                ods_distinct = cur.fetchone()[0]
            except Exception:
                conn.rollback()
                ods_total = -1
                ods_distinct = -1

            status = log_info.get("status", "N/A")
            rows.append({
                "task": task_name,
                "ods_table": ods_table,
                "api_fetched": api_fetched if api_fetched is not None else "N/A",
                "json_records": api_json_count if api_json_count is not None else "N/A",
                "ods_total": ods_total,
                "ods_distinct": ods_distinct,
                "etl_status": status,
            })
    return rows


# ── 8. 生成 Markdown 报告 ────────────────────────────────────────────
def generate_report(
    log_path: Path | None,
    log_results: dict,
    api_vs_ods: list[dict],
    ods_dwd_compare: list[dict],
    dwd_quality: list[dict],
    dws_sanity: list[dict],
    dws_row_counts: dict[str, int],
) -> str:
    lines = []
    lines.append(f"# 黑盒集成测试报告")
    lines.append(f"")
    lines.append(f"生成时间: {NOW.strftime('%Y-%m-%d %H:%M:%S')}")
    lines.append(f"ETL 日志: `{log_path.name if log_path else 'N/A'}`")
    lines.append(f"测试窗口: 2025-11-01 ~ 2026-02-20 (full_window 模式)")
    lines.append(f"")

    # ── 总览 ──
    succ = sum(1 for v in log_results.values() if v.get("status") == "SUCC")
    fail = sum(1 for v in log_results.values() if v.get("status") == "FAIL")
    skip = sum(1 for v in log_results.values() if v.get("status") == "SKIP")
    lines.append(f"## 1. ETL 执行总览")
    lines.append(f"")
    lines.append(f"| 指标 | 值 |")
    lines.append(f"|------|-----|")
    lines.append(f"| 成功任务 | {succ} |")
    lines.append(f"| 失败任务 | {fail} |")
    lines.append(f"| 跳过任务 | {skip} |")
    lines.append(f"| 总计 | {len(log_results)} |")
    lines.append(f"")

    # 失败详情
    if fail > 0:
        lines.append(f"### 失败任务详情")
        lines.append(f"")
        for k, v in log_results.items():
            if v.get("status") == "FAIL":
                lines.append(f"- **{k}**: {v.get('error', '未知错误')}")
        lines.append(f"")

    # 跳过详情
    if skip > 0:
        lines.append(f"### 跳过任务（未注册）")
        lines.append(f"")
        for k, v in log_results.items():
            if v.get("status") == "SKIP":
                lines.append(f"- {k}")
        lines.append(f"")

    # ── API 输入侧 vs ODS 输出侧 ──
    lines.append(f"## 2. 输入侧（API）vs 输出侧（ODS）对比")
    lines.append(f"")
    lines.append(f"| 任务 | ODS 表 | API 抓取数 | ODS 总行数 | ODS 去重ID | ETL 状态 |")
    lines.append(f"|------|--------|-----------|-----------|-----------|---------|")
    for r in api_vs_ods:
        lines.append(
            f"| {r['task']} | {r['ods_table']} | {r['api_fetched']} "
            f"| {r['ods_total']} | {r['ods_distinct']} | {r['etl_status']} |"
        )
    lines.append(f"")
    lines.append(f"> 说明: ODS 采用快照模式 (id, content_hash) 为 PK，content_hash 变化产生新行，")
    lines.append(f"> 因此 ODS 总行数 ≥ ODS 去重 ID 数。API 抓取数 = 本次 ETL 从 API 获取的记录数。")
    lines.append(f"")

    # ── ODS vs DWD ──
    lines.append(f"## 3. ODS → DWD 行数对比")
    lines.append(f"")
    lines.append(f"| DWD 表 | ODS 表 | 类型 | ODS 总行 | ODS 去重ID | DWD 行数 | 比率 |")
    lines.append(f"|--------|--------|------|---------|-----------|---------|------|")
    for r in ods_dwd_compare:
        lines.append(
            f"| {r['dwd_table']} | {r['ods_table']} | {r['type']} "
            f"| {r['ods_total_rows']} | {r['ods_distinct_ids']} "
            f"| {r['dwd_rows']} | {r['ratio']} |"
        )
    lines.append(f"")
    lines.append(f"> 说明: 维度表(SCD2)的 DWD 行数可能 > ODS 去重 ID（历史版本保留）。")
    lines.append(f"> 事实表的 DWD 行数应 ≈ ODS 去重 ID 数。比率 = DWD行数 / ODS去重ID。")
    lines.append(f"")

    # ── DWD 数据质量 ──
    lines.append(f"## 4. DWD 数据质量（NULL 率抽样）")
    lines.append(f"")
    lines.append(f"| DWD 表 | 总行数 | NULL 列情况 |")
    lines.append(f"|--------|--------|------------|")
    for r in dwd_quality:
        lines.append(f"| dwd.{r['table']} | {r['total']} | {r['null_cols']} |")
    lines.append(f"")

    # ── DWS 汇总 ──
    lines.append(f"## 5. DWS 汇总层检查")
    lines.append(f"")
    lines.append(f"| DWS 表 | 行数 | 最新日期 | 状态 |")
    lines.append(f"|--------|------|---------|------|")
    for r in dws_sanity:
        lines.append(f"| {r['table']} | {r['rows']} | {r['latest_date']} | {r['status']} |")
    lines.append(f"")

    # ── 结论 ──
    lines.append(f"## 6. 结论")
    lines.append(f"")
    total_ods_ok = sum(1 for r in api_vs_ods if r["etl_status"] == "SUCC")
    total_dwd_ok = sum(1 for r in ods_dwd_compare if r["dwd_rows"] > 0)
    total_dws_ok = sum(1 for r in dws_sanity if r["rows"] > 0)
    total_dws_all = len(dws_sanity)

    lines.append(f"- ODS 层: {total_ods_ok}/{len(api_vs_ods)} 个任务成功入库")
    lines.append(f"- DWD 层: {total_dwd_ok}/{len(ods_dwd_compare)} 个表有数据")
    lines.append(f"- DWS 层: {total_dws_ok}/{total_dws_all} 个表有数据")
    lines.append(f"- 失败任务: {fail} 个（详见第 1 节）")
    lines.append(f"- 跳过任务: {skip} 个（未注册的 DWS 任务）")
    lines.append(f"")

    return "\n".join(lines)


# ── main ──────────────────────────────────────────────────────────────
def main():
    print("=== 黑盒集成测试报告生成 ===")

    # 1. 解析日志
    log_path = find_latest_log()
    if log_path:
        print(f"解析日志: {log_path.name}")
        log_results = parse_etl_log(log_path)
        print(f"  解析到 {len(log_results)} 个任务结果")
    else:
        print("未找到 ETL 日志")
        log_results = {}

    # 2. 连接数据库
    print("连接数据库...")
    conn = psycopg2.connect(DSN)
    conn.autocommit = True

    try:
        # 3. API vs ODS
        print("对比 API 输入侧 vs ODS...")
        api_vs_ods = compare_api_vs_ods(conn, log_results)

        # 4. ODS vs DWD
        print("对比 ODS vs DWD...")
        ods_dwd_compare = compare_ods_dwd(conn)

        # 5. DWD 质量
        print("检查 DWD 数据质量...")
        dwd_tables = query_dwd_tables(conn)
        dwd_quality = check_dwd_null_rates(conn, dwd_tables)

        # 6. DWS 合理性
        print("检查 DWS 汇总层...")
        dws_sanity = check_dws_sanity(conn)
        dws_tables = query_dws_tables(conn)
        dws_row_counts = query_row_counts(conn, "dws", dws_tables)

        # 7. 生成报告
        print("生成报告...")
        report_md = generate_report(
            log_path, log_results,
            api_vs_ods, ods_dwd_compare,
            dwd_quality, dws_sanity, dws_row_counts,
        )

        out_path = REPORT_ROOT / f"blackbox_report_{TS}.md"
        out_path.write_text(report_md, encoding="utf-8")
        print(f"报告已生成: {out_path}")

    finally:
        conn.close()


if __name__ == "__main__":
    main()
