feiqiu-ETL/tmp/detailed_field_compare.py

# -*- coding: utf-8 -*-
"""
详细双向对比 - 针对可能相关的字段
"""
import os
import json
import re
import psycopg2
from psycopg2.extras import RealDictCursor

DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test'

# 需要详细审核的字段对
REVIEW_PAIRS = [
    {
        'code': 'ODS_TABLE_USE',
        'table': 'billiards_ods.table_fee_transactions',
        'keywords': ['service', 'money', 'real'],
    },
    {
        'code': 'ODS_ASSISTANT_LEDGER', 
        'table': 'billiards_ods.assistant_service_records',
        'keywords': ['service', 'money', 'real'],
    },
    {
        'code': 'ODS_MEMBER_CARD',
        'table': 'billiards_ods.member_stored_value_cards',
        'keywords': ['balance', 'principal', 'freeze', 'recharge'],
    },
    {
        'code': 'ODS_MEMBER_BALANCE',
        'table': 'billiards_ods.member_balance_changes',
        'keywords': ['before', 'after', 'principal', 'change'],
    },
    {
        'code': 'ODS_SETTLEMENT_RECORDS',
        'table': 'billiards_ods.settlement_records',
        'keywords': ['coupon', 'sale', 'amount', 'pl', 'tenant'],
    },
    {
        'code': 'ODS_RECHARGE_SETTLE',
        'table': 'billiards_ods.recharge_settlements',
        'keywords': ['coupon', 'sale', 'amount', 'pl', 'tenant'],
    },
    {
        'code': 'ODS_GROUP_PACKAGE',
        'table': 'billiards_ods.group_buy_packages',
        'keywords': ['table', 'area', 'name', 'list', 'tenant'],
    },
]

def get_ods_columns(conn, table_name):
    """获取 ODS 表字段"""
    if '.' in table_name:
        schema, name = table_name.split('.', 1)
    else:
        schema, name = 'public', table_name
    
    sql = """
        SELECT column_name, data_type
        FROM information_schema.columns
        WHERE table_schema = %s AND table_name = %s
        ORDER BY ordinal_position
    """
    with conn.cursor(cursor_factory=RealDictCursor) as cur:
        cur.execute(sql, (schema, name))
        return {row['column_name']: row['data_type'] for row in cur.fetchall()}

def normalize(s):
    """标准化：去除下划线，全小写"""
    return s.lower().replace('_', '')

def filter_by_keywords(fields, keywords):
    """按关键词筛选字段"""
    result = []
    for f in fields:
        f_norm = normalize(f)
        for kw in keywords:
            if kw in f_norm:
                result.append(f)
                break
    return sorted(set(result))

def main():
    # 读取 API 字段
    json_path = os.path.join(os.path.dirname(__file__), 'api_ods_comparison.json')
    with open(json_path, 'r', encoding='utf-8') as f:
        results = json.load(f)
    
    conn = psycopg2.connect(DSN)
    
    print("=" * 100)
    print("双向详细对比 - 可能相关的字段")
    print("=" * 100)
    
    for review in REVIEW_PAIRS:
        code = review['code']
        table = review['table']
        keywords = review['keywords']
        
        if code not in results:
            continue
        
        data = results[code]
        api_fields = data.get('api_fields', [])
        
        # 获取 ODS 字段
        ods_columns = get_ods_columns(conn, table)
        
        # 按关键词筛选
        api_related = filter_by_keywords(api_fields, keywords)
        ods_related = filter_by_keywords(ods_columns.keys(), keywords)
        
        print(f"\n{'='*80}")
        print(f"### {code}")
        print(f"表: {table}")
        print(f"关键词: {keywords}")
        print(f"{'='*80}")
        
        print(f"\n**API 相关字段 ({len(api_related)}):**")
        for f in api_related:
            print(f"  - {f}")
        
        print(f"\n**ODS 相关字段 ({len(ods_related)}):**")
        for f in ods_related:
            dtype = ods_columns.get(f, '')
            print(f"  - {f} ({dtype})")
        
        # 匹配分析
        print(f"\n**匹配分析:**")
        
        # 建立映射
        matched_api = set()
        matched_ods = set()
        mappings = []
        
        for api_f in api_related:
            api_norm = normalize(api_f)
            for ods_f in ods_related:
                ods_norm = normalize(ods_f)
                
                # 完全匹配
                if api_norm == ods_norm:
                    mappings.append((api_f, ods_f, 'exact', '完全匹配'))
                    matched_api.add(api_f)
                    matched_ods.add(ods_f)
                # 包含关系
                elif api_norm in ods_norm or ods_norm in api_norm:
                    if api_f not in matched_api:
                        mappings.append((api_f, ods_f, 'partial', '部分匹配'))
        
        if mappings:
            print("\n| API 字段 | ODS 字段 | 类型 | 说明 |")
            print("|----------|----------|------|------|")
            for api_f, ods_f, mtype, desc in mappings:
                print(f"| `{api_f}` | `{ods_f}` | {mtype} | {desc} |")
        
        # 未匹配的 API 字段
        unmatched_api = set(api_related) - matched_api
        if unmatched_api:
            print(f"\n**API 未匹配字段:**")
            for f in sorted(unmatched_api):
                print(f"  - {f}")
        
        # 未匹配的 ODS 字段
        unmatched_ods = set(ods_related) - matched_ods
        if unmatched_ods:
            print(f"\n**ODS 未匹配字段:**")
            for f in sorted(unmatched_ods):
                print(f"  - {f}")
    
    conn.close()
    
    # 输出最终结论
    print("\n")
    print("=" * 100)
    print("最终审核结论")
    print("=" * 100)

if __name__ == '__main__':
    main()