初始提交：飞球 ETL 系统全量代码

2026-02-13 08:05:34 +08:00
commit 3c51f5485d
441 changed files with 117631 additions and 0 deletions
--- a/tasks/verification/init.py
+++ b/tasks/verification/init.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+"""批量后置校验框架
+
+提供各层数据的批量校验和补齐功能：
+- ODS 层：主键 + content_hash 对比，批量 UPSERT
+- DWD 层：维度 SCD2 / 事实主键对比，批量 UPSERT
+- DWS 层：聚合对比，批量重算 UPSERT
+- INDEX 层：实体覆盖对比，批量重算 UPSERT
+"""
+
+from .models import (
+    VerificationResult,
+    VerificationSummary,
+    VerificationStatus,
+    WindowSegment,
+    build_window_segments,
+    filter_verify_tables,
+)
+from .base_verifier import BaseVerifier
+from .ods_verifier import OdsVerifier
+from .dwd_verifier import DwdVerifier
+from .dws_verifier import DwsVerifier
+from .index_verifier import IndexVerifier
+
+__all__ = [
+    # 模型
+    "VerificationResult",
+    "VerificationSummary",
+    "VerificationStatus",
+    "WindowSegment",
+    "build_window_segments",
+    "filter_verify_tables",
+    # 校验器
+    "BaseVerifier",
+    "OdsVerifier",
+    "DwdVerifier",
+    "DwsVerifier",
+    "IndexVerifier",
+]
+
+
+def get_verifier_for_layer(layer: str, db_connection, logger=None, **kwargs):
+    """
+    根据层名获取对应的校验器实例
+    
+    Args:
+        layer: 层名 ("ODS", "DWD", "DWS", "INDEX")
+        db_connection: 数据库连接
+        logger: 日志器
+        **kwargs: 额外参数
+            - api_client: API 客户端（ODS 层需要）
+            - fetch_from_api: 是否从 API 获取源数据（ODS 层需要）
+            - local_dump_dirs: 本地 JSON dump 目录映射（ODS 层需要）
+            - use_local_json: 是否优先使用本地 JSON（ODS 层需要）
+    
+    Returns:
+        对应的校验器实例
+    """
+    verifier_map = {
+        "ODS": OdsVerifier,
+        "DWD": DwdVerifier,
+        "DWS": DwsVerifier,
+        "INDEX": IndexVerifier,
+    }
+    
+    verifier_class = verifier_map.get(layer.upper())
+    if verifier_class is None:
+        raise ValueError(f"未知的数据层: {layer}")
+    
+    # ODS 层支持额外参数
+    if layer.upper() == "ODS":
+        api_client = kwargs.pop("api_client", None)
+        fetch_from_api = kwargs.pop("fetch_from_api", False)
+        local_dump_dirs = kwargs.pop("local_dump_dirs", None)
+        use_local_json = kwargs.pop("use_local_json", False)
+        return verifier_class(
+            db_connection, 
+            api_client=api_client, 
+            logger=logger, 
+            fetch_from_api=fetch_from_api,
+            local_dump_dirs=local_dump_dirs,
+            use_local_json=use_local_json,
+            **kwargs
+        )
+    
+    return verifier_class(db_connection, logger=logger, **kwargs)