87 lines
2.4 KiB
Python
87 lines
2.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
测试 ODS 回填特性
|
|
"""
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
project_root = Path(__file__).parent.parent / "etl_billiards"
|
|
sys.path.insert(0, str(project_root))
|
|
|
|
from dotenv import load_dotenv
|
|
load_dotenv(project_root / ".env")
|
|
|
|
from database.connection import DatabaseConnection
|
|
|
|
dsn = os.getenv("PG_DSN")
|
|
db = DatabaseConnection(dsn)
|
|
|
|
print("=== 测试 ODS 回填特性 ===")
|
|
|
|
# 1. 创建一个测试场景：找一条有 NULL 值的记录
|
|
result = db.query("""
|
|
SELECT id, plcouponsaleamount, mervousalesamount,
|
|
payload->'settleList'->>'plCouponSaleAmount' as payload_val
|
|
FROM billiards_ods.settlement_records
|
|
WHERE plcouponsaleamount IS NOT NULL
|
|
LIMIT 1
|
|
""")
|
|
|
|
if result:
|
|
row = result[0]
|
|
print(f"找到测试记录: id={row['id']}")
|
|
print(f" 当前 plcouponsaleamount: {row['plcouponsaleamount']}")
|
|
print(f" payload 中的值: {row['payload_val']}")
|
|
else:
|
|
print("未找到测试记录")
|
|
|
|
# 2. 模拟生成的 SQL 语句
|
|
print("\n=== 生成的 SQL 示例 ===")
|
|
|
|
# 获取表结构
|
|
cols = db.query("""
|
|
SELECT column_name
|
|
FROM information_schema.columns
|
|
WHERE table_schema = 'billiards_ods'
|
|
AND table_name = 'settlement_records'
|
|
ORDER BY ordinal_position
|
|
""")
|
|
|
|
col_names = [c["column_name"] for c in cols]
|
|
pk_cols = ["id"] # 假设主键是 id
|
|
|
|
meta_cols = {"payload", "source_file", "source_endpoint", "fetched_at", "content_hash"}
|
|
pk_cols_lower = {c.lower() for c in pk_cols}
|
|
update_cols = [
|
|
c for c in col_names
|
|
if c.lower() not in pk_cols_lower and c.lower() not in meta_cols
|
|
]
|
|
|
|
print(f"表有 {len(col_names)} 列")
|
|
print(f"可更新列: {len(update_cols)} 列")
|
|
|
|
# 生成 SQL
|
|
table = "billiards_ods.settlement_records"
|
|
pk_clause = ", ".join(f'"{c}"' for c in pk_cols)
|
|
set_clause = ", ".join(
|
|
f'"{c}" = COALESCE({table}."{c}", EXCLUDED."{c}")'
|
|
for c in update_cols[:3] # 只显示前3个
|
|
)
|
|
where_clause = " OR ".join(f'{table}."{c}" IS NULL' for c in update_cols[:3])
|
|
|
|
print(f"\nSQL 示例 (前3列):")
|
|
print(f"INSERT INTO {table} (...) VALUES ...")
|
|
print(f"ON CONFLICT ({pk_clause}) DO UPDATE SET")
|
|
print(f" {set_clause}")
|
|
print(f"WHERE {where_clause}")
|
|
|
|
print("\n=== 特性说明 ===")
|
|
print("1. 新记录 -> 正常插入")
|
|
print("2. 已存在记录 -> 只更新 NULL 列 (COALESCE)")
|
|
print("3. 已有值的列 -> 保持不变")
|
|
print("4. 可通过配置 run.ods_backfill_null_columns=false 禁用")
|
|
|
|
db.close()
|
|
print("\n测试完成!")
|