Updata2

tmp/test_backfill_feature.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+"""
+测试 ODS 回填特性
+"""
+import os
+import sys
+from pathlib import Path
+
+project_root = Path(__file__).parent.parent / "etl_billiards"
+sys.path.insert(0, str(project_root))
+
+from dotenv import load_dotenv
+load_dotenv(project_root / ".env")
+
+from database.connection import DatabaseConnection
+
+dsn = os.getenv("PG_DSN")
+db = DatabaseConnection(dsn)
+
+print("=== 测试 ODS 回填特性 ===")
+
+# 1. 创建一个测试场景：找一条有 NULL 值的记录
+result = db.query("""
+    SELECT id, plcouponsaleamount, mervousalesamount, 
+           payload->'settleList'->>'plCouponSaleAmount' as payload_val
+    FROM billiards_ods.settlement_records
+    WHERE plcouponsaleamount IS NOT NULL
+    LIMIT 1
+""")
+
+if result:
+    row = result[0]
+    print(f"找到测试记录: id={row['id']}")
+    print(f"  当前 plcouponsaleamount: {row['plcouponsaleamount']}")
+    print(f"  payload 中的值: {row['payload_val']}")
+else:
+    print("未找到测试记录")
+
+# 2. 模拟生成的 SQL 语句
+print("\n=== 生成的 SQL 示例 ===")
+
+# 获取表结构
+cols = db.query("""
+    SELECT column_name 
+    FROM information_schema.columns 
+    WHERE table_schema = 'billiards_ods' 
+      AND table_name = 'settlement_records'
+    ORDER BY ordinal_position
+""")
+
+col_names = [c["column_name"] for c in cols]
+pk_cols = ["id"]  # 假设主键是 id
+
+meta_cols = {"payload", "source_file", "source_endpoint", "fetched_at", "content_hash"}
+pk_cols_lower = {c.lower() for c in pk_cols}
+update_cols = [
+    c for c in col_names
+    if c.lower() not in pk_cols_lower and c.lower() not in meta_cols
+]
+
+print(f"表有 {len(col_names)} 列")
+print(f"可更新列: {len(update_cols)} 列")
+
+# 生成 SQL
+table = "billiards_ods.settlement_records"
+pk_clause = ", ".join(f'"{c}"' for c in pk_cols)
+set_clause = ", ".join(
+    f'"{c}" = COALESCE({table}."{c}", EXCLUDED."{c}")'
+    for c in update_cols[:3]  # 只显示前3个
+)
+where_clause = " OR ".join(f'{table}."{c}" IS NULL' for c in update_cols[:3])
+
+print(f"\nSQL 示例 (前3列):")
+print(f"INSERT INTO {table} (...) VALUES ...")
+print(f"ON CONFLICT ({pk_clause}) DO UPDATE SET")
+print(f"  {set_clause}")
+print(f"WHERE {where_clause}")
+
+print("\n=== 特性说明 ===")
+print("1. 新记录 -> 正常插入")
+print("2. 已存在记录 -> 只更新 NULL 列 (COALESCE)")
+print("3. 已有值的列 -> 保持不变")
+print("4. 可通过配置 run.ods_backfill_null_columns=false 禁用")
+
+db.close()
+print("\n测试完成!")