# -*- coding: utf-8 -*- """ 测试 ODS 回填特性 """ import os import sys from pathlib import Path project_root = Path(__file__).parent.parent / "etl_billiards" sys.path.insert(0, str(project_root)) from dotenv import load_dotenv load_dotenv(project_root / ".env") from database.connection import DatabaseConnection dsn = os.getenv("PG_DSN") db = DatabaseConnection(dsn) print("=== 测试 ODS 回填特性 ===") # 1. 创建一个测试场景:找一条有 NULL 值的记录 result = db.query(""" SELECT id, plcouponsaleamount, mervousalesamount, payload->'settleList'->>'plCouponSaleAmount' as payload_val FROM billiards_ods.settlement_records WHERE plcouponsaleamount IS NOT NULL LIMIT 1 """) if result: row = result[0] print(f"找到测试记录: id={row['id']}") print(f" 当前 plcouponsaleamount: {row['plcouponsaleamount']}") print(f" payload 中的值: {row['payload_val']}") else: print("未找到测试记录") # 2. 模拟生成的 SQL 语句 print("\n=== 生成的 SQL 示例 ===") # 获取表结构 cols = db.query(""" SELECT column_name FROM information_schema.columns WHERE table_schema = 'billiards_ods' AND table_name = 'settlement_records' ORDER BY ordinal_position """) col_names = [c["column_name"] for c in cols] pk_cols = ["id"] # 假设主键是 id meta_cols = {"payload", "source_file", "source_endpoint", "fetched_at", "content_hash"} pk_cols_lower = {c.lower() for c in pk_cols} update_cols = [ c for c in col_names if c.lower() not in pk_cols_lower and c.lower() not in meta_cols ] print(f"表有 {len(col_names)} 列") print(f"可更新列: {len(update_cols)} 列") # 生成 SQL table = "billiards_ods.settlement_records" pk_clause = ", ".join(f'"{c}"' for c in pk_cols) set_clause = ", ".join( f'"{c}" = COALESCE({table}."{c}", EXCLUDED."{c}")' for c in update_cols[:3] # 只显示前3个 ) where_clause = " OR ".join(f'{table}."{c}" IS NULL' for c in update_cols[:3]) print(f"\nSQL 示例 (前3列):") print(f"INSERT INTO {table} (...) VALUES ...") print(f"ON CONFLICT ({pk_clause}) DO UPDATE SET") print(f" {set_clause}") print(f"WHERE {where_clause}") print("\n=== 特性说明 ===") print("1. 新记录 -> 正常插入") print("2. 已存在记录 -> 只更新 NULL 列 (COALESCE)") print("3. 已有值的列 -> 保持不变") print("4. 可通过配置 run.ods_backfill_null_columns=false 禁用") db.close() print("\n测试完成!")