在准备环境前提交次全部更改。
This commit is contained in:
391
apps/etl/connectors/feiqiu/orchestration/flow_runner.py
Normal file
391
apps/etl/connectors/feiqiu/orchestration/flow_runner.py
Normal file
@@ -0,0 +1,391 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Flow 运行器：Flow 定义、层→任务映射、校验编排。
|
||||
|
||||
从原 ETLScheduler 中提取 Flow 编排逻辑，委托 TaskExecutor 执行具体任务。
|
||||
所有依赖通过构造函数注入，不自行创建资源。
|
||||
|
||||
术语说明：代码中保留 pipeline 参数名以兼容调用方，概念上统一使用 Flow。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from tasks.verification import filter_verify_tables
|
||||
from orchestration.topological_sort import topological_sort
|
||||
|
||||
|
||||
class FlowRunner:
|
||||
"""Flow 编排器：根据 Flow 定义执行多层 ETL 任务并可选运行后置校验。"""
|
||||
|
||||
# Flow 定义：每个 Flow 包含的层（从 scheduler.py 模块级常量迁移至此）
|
||||
FLOW_LAYERS: dict[str, list[str]] = {
|
||||
"api_ods": ["ODS"],
|
||||
"api_ods_dwd": ["ODS", "DWD"],
|
||||
"api_full": ["ODS", "DWD", "DWS", "INDEX"],
|
||||
"ods_dwd": ["DWD"],
|
||||
"dwd_dws": ["DWS"],
|
||||
"dwd_dws_index": ["DWS", "INDEX"],
|
||||
"dwd_index": ["INDEX"],
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
task_executor,
|
||||
task_registry,
|
||||
db_conn,
|
||||
api_client,
|
||||
logger: logging.Logger,
|
||||
):
|
||||
self.config = config
|
||||
self.task_executor = task_executor
|
||||
self.task_registry = task_registry
|
||||
self.db_conn = db_conn
|
||||
self.api_client = api_client
|
||||
self.logger = logger
|
||||
self.tz = ZoneInfo(config.get("app.timezone", "Asia/Shanghai"))
|
||||
|
||||
def run(
|
||||
self,
|
||||
pipeline: str | None = None,
|
||||
layers: list[str] | None = None,
|
||||
processing_mode: str = "increment_only",
|
||||
data_source: str = "hybrid",
|
||||
window_start: datetime | None = None,
|
||||
window_end: datetime | None = None,
|
||||
window_split: str | None = None,
|
||||
task_codes: list[str] | None = None,
|
||||
fetch_before_verify: bool = False,
|
||||
verify_tables: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""执行 Flow，返回汇总结果。
|
||||
|
||||
Args:
|
||||
pipeline: Flow 名称 (api_ods, api_ods_dwd, api_full, ...)，与 layers 二选一（参数名保留以兼容调用方）
|
||||
layers: 直接指定层列表 (["ODS", "DWD"] 等)，与 flow 名称二选一
|
||||
processing_mode: 处理模式 (increment_only / verify_only / increment_verify)
|
||||
data_source: 数据源模式 (online / offline / hybrid)
|
||||
window_start: 时间窗口开始
|
||||
window_end: 时间窗口结束
|
||||
window_split: 时间窗口切分 (none / day / week / month)
|
||||
task_codes: 要执行的任务代码列表（作为 Flow 内的任务过滤器）
|
||||
fetch_before_verify: 校验前是否先从 API 获取数据（仅在 verify_only 模式下有效）
|
||||
verify_tables: 指定校验的表名列表（可用于单表验证）
|
||||
|
||||
Returns:
|
||||
执行结果字典，包含 status / pipeline / layers / results / verification_summary
|
||||
"""
|
||||
from utils.task_logger import TaskLogger
|
||||
|
||||
# 解析层列表：Flow 名称查找 或 直接使用 layers 参数
|
||||
if pipeline is not None:
|
||||
if pipeline not in self.FLOW_LAYERS:
|
||||
raise ValueError(f"无效的 Flow 名称: {pipeline}")
|
||||
resolved_layers = self.FLOW_LAYERS[pipeline]
|
||||
run_label = pipeline
|
||||
elif layers is not None:
|
||||
resolved_layers = layers
|
||||
run_label = f"layers({','.join(layers)})"
|
||||
else:
|
||||
raise ValueError("必须指定 flow 名称或 layers 参数之一")
|
||||
|
||||
run_uuid = uuid.uuid4().hex
|
||||
flow_logger = TaskLogger(f"FLOW_{run_label.upper()}", self.logger)
|
||||
flow_logger.start(f"开始执行 Flow: {run_label}")
|
||||
|
||||
layers = resolved_layers
|
||||
results: list[dict[str, Any]] = []
|
||||
verification_summary: dict[str, Any] | None = None
|
||||
ods_dump_dirs: dict[str, str] = {}
|
||||
use_local_json = bool(self.config.get("verification.ods_use_local_json", False))
|
||||
|
||||
# 设置默认时间窗口
|
||||
if window_end is None:
|
||||
window_end = datetime.now(self.tz)
|
||||
if window_start is None:
|
||||
window_start = window_end - timedelta(hours=24)
|
||||
|
||||
try:
|
||||
if processing_mode == "verify_only":
|
||||
# 仅校验模式
|
||||
if fetch_before_verify:
|
||||
self.logger.info("Flow %s: 校验模式（先获取 API 数据）", run_label)
|
||||
|
||||
if task_codes:
|
||||
ods_tasks = [t for t in task_codes if t.startswith("ODS_")]
|
||||
if ods_tasks:
|
||||
self.logger.info("从 API 获取数据: %s", ods_tasks)
|
||||
results = self.task_executor.run_tasks(ods_tasks, data_source=data_source)
|
||||
else:
|
||||
auto_tasks = self._resolve_tasks(["ODS"])
|
||||
if auto_tasks:
|
||||
self.logger.info("从 API 获取数据: %s", auto_tasks)
|
||||
results = self.task_executor.run_tasks(auto_tasks, data_source=data_source)
|
||||
|
||||
ods_dump_dirs = {
|
||||
r.get("task_code"): r.get("dump_dir")
|
||||
for r in results
|
||||
if r.get("task_code") and r.get("dump_dir")
|
||||
}
|
||||
self.logger.info("API 数据获取完成，开始校验并修复")
|
||||
else:
|
||||
self.logger.info("Flow %s: 仅校验模式，跳过增量 ETL，直接执行校验并修复", run_label)
|
||||
|
||||
verification_summary = self._run_verification(
|
||||
layers=layers,
|
||||
window_start=window_start,
|
||||
window_end=window_end,
|
||||
window_split=window_split,
|
||||
fetch_from_api=fetch_before_verify,
|
||||
ods_dump_dirs=ods_dump_dirs,
|
||||
use_local_json=use_local_json,
|
||||
verify_tables=verify_tables,
|
||||
)
|
||||
flow_logger.set_verification_result(verification_summary)
|
||||
else:
|
||||
# 增量 ETL（increment_only 或 increment_verify）
|
||||
self.logger.info("Flow %s: 执行增量 ETL，层=%s", run_label, layers)
|
||||
|
||||
if task_codes:
|
||||
results = self.task_executor.run_tasks(task_codes, data_source=data_source)
|
||||
else:
|
||||
auto_tasks = self._resolve_tasks(layers)
|
||||
results = self.task_executor.run_tasks(auto_tasks, data_source=data_source)
|
||||
|
||||
# increment_verify 模式：增量后执行校验
|
||||
if processing_mode == "increment_verify":
|
||||
self.logger.info("Flow %s: 开始校验并修复", run_label)
|
||||
verification_summary = self._run_verification(
|
||||
layers=layers,
|
||||
window_start=window_start,
|
||||
window_end=window_end,
|
||||
window_split=window_split,
|
||||
ods_dump_dirs=ods_dump_dirs,
|
||||
use_local_json=use_local_json,
|
||||
verify_tables=verify_tables,
|
||||
)
|
||||
flow_logger.set_verification_result(verification_summary)
|
||||
|
||||
# 汇总计数
|
||||
flow_logger.set_counts(
|
||||
fetched=sum(r.get("counts", {}).get("fetched", 0) for r in results),
|
||||
inserted=sum(r.get("counts", {}).get("inserted", 0) for r in results),
|
||||
updated=sum(r.get("counts", {}).get("updated", 0) for r in results),
|
||||
errors=sum(r.get("counts", {}).get("errors", 0) for r in results),
|
||||
)
|
||||
|
||||
summary_text = flow_logger.end(status="成功")
|
||||
self.logger.info("\n%s", summary_text)
|
||||
|
||||
return {
|
||||
"status": "SUCCESS",
|
||||
"pipeline": run_label,
|
||||
"layers": layers,
|
||||
"results": results,
|
||||
"verification_summary": verification_summary,
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
summary_text = flow_logger.end(status="失败", error_message=str(exc))
|
||||
self.logger.error("\n%s", summary_text)
|
||||
raise
|
||||
|
||||
def _resolve_tasks(self, layers: list[str]) -> list[str]:
|
||||
"""根据层列表解析任务代码。
|
||||
|
||||
优先级：配置值 > TaskRegistry.get_tasks_by_layer() > 空列表（记录警告）。
|
||||
"""
|
||||
# CHANGE [2026-07-17] intent: 移除所有硬编码回退列表，统一走 Registry（需求 7.1, 7.2, 7.3）
|
||||
# assumptions: TaskRegistry 已注册所有层的任务；DWD 层新增 run.dwd_tasks 配置键
|
||||
# prompt: "统一 _resolve_tasks() 去掉硬编码回退"
|
||||
# 配置键与层名的映射
|
||||
_LAYER_CONFIG_KEY = {
|
||||
"ODS": "run.ods_tasks",
|
||||
"DWD": "run.dwd_tasks",
|
||||
"DWS": "run.dws_tasks",
|
||||
"INDEX": "run.index_tasks",
|
||||
}
|
||||
|
||||
tasks: list[str] = []
|
||||
|
||||
for layer in layers:
|
||||
layer_upper = layer.upper()
|
||||
|
||||
# 1. 优先使用配置值
|
||||
config_key = _LAYER_CONFIG_KEY.get(layer_upper)
|
||||
if config_key:
|
||||
config_tasks = self.config.get(config_key, [])
|
||||
if config_tasks:
|
||||
tasks.extend(config_tasks)
|
||||
continue
|
||||
|
||||
# 2. 回退到 Registry
|
||||
registry_tasks = self.task_registry.get_tasks_by_layer(layer_upper)
|
||||
if registry_tasks:
|
||||
tasks.extend(registry_tasks)
|
||||
else:
|
||||
# 3. Registry 也为空，记录警告并跳过
|
||||
self.logger.warning(
|
||||
"层 %s 在 Registry 中无已注册任务且无配置覆盖，跳过", layer_upper
|
||||
)
|
||||
|
||||
# CHANGE [2026-07-18] intent: 对收集到的任务执行拓扑排序，确保依赖方在被依赖方之后（需求 8.3, 8.4, 8.5）
|
||||
if tasks:
|
||||
tasks = topological_sort(tasks, self.task_registry)
|
||||
|
||||
return tasks
|
||||
|
||||
|
||||
def _run_verification(
|
||||
self,
|
||||
layers: list[str],
|
||||
window_start: datetime,
|
||||
window_end: datetime,
|
||||
window_split: str | None = None,
|
||||
fetch_from_api: bool = False,
|
||||
ods_dump_dirs: dict[str, str] | None = None,
|
||||
use_local_json: bool = False,
|
||||
verify_tables: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""对指定层执行后置校验（从原 _run_layer_verification 迁移）。"""
|
||||
try:
|
||||
from tasks.verification import get_verifier_for_layer, build_window_segments
|
||||
except ImportError:
|
||||
self.logger.warning("校验框架未安装，跳过后置校验")
|
||||
return {"status": "SKIPPED", "message": "校验框架未安装"}
|
||||
|
||||
total_tables = 0
|
||||
consistent_tables = 0
|
||||
total_backfilled = 0
|
||||
total_error_tables = 0
|
||||
layer_results: dict[str, Any] = {}
|
||||
skip_ods_on_fetch = bool(self.config.get("verification.skip_ods_when_fetch_before_verify", True))
|
||||
ods_dump_dirs = ods_dump_dirs or {}
|
||||
|
||||
segments = build_window_segments(window_start, window_end, window_split)
|
||||
|
||||
for layer in layers:
|
||||
try:
|
||||
if layer.upper() == "ODS" and fetch_from_api and skip_ods_on_fetch:
|
||||
self.logger.info("ODS 层在 fetch_before_verify 下已完成入库，跳过二次校验")
|
||||
layer_results[layer] = {
|
||||
"status": "SKIPPED",
|
||||
"reason": "fetch_before_verify",
|
||||
}
|
||||
continue
|
||||
|
||||
# CHANGE [2025-07-18] intent: DWS/INDEX 层跳过完整性校验，仅记录日志（需求 6.5）
|
||||
# assumptions: DWS/INDEX 层无轻量级 verifier，跳过最安全
|
||||
# prompt: "实现 DWS/INDEX 层轻量级校验"
|
||||
if layer.upper() in ("DWS", "INDEX"):
|
||||
self.logger.info(
|
||||
"DWS/INDEX 层使用轻量级校验，跳过完整性检查: %s", layer
|
||||
)
|
||||
layer_results[layer] = {
|
||||
"status": "SKIPPED",
|
||||
"reason": "lightweight_dws_index",
|
||||
}
|
||||
continue
|
||||
|
||||
if layer.upper() == "ODS" and fetch_from_api:
|
||||
if use_local_json:
|
||||
if not ods_dump_dirs:
|
||||
self.logger.warning("ODS 校验配置为使用本地 JSON，但未找到 dump 目录，跳过 ODS 校验")
|
||||
layer_results[layer] = {
|
||||
"status": "SKIPPED",
|
||||
"reason": "local_json_missing",
|
||||
}
|
||||
continue
|
||||
verifier = get_verifier_for_layer(
|
||||
layer,
|
||||
self.db_conn,
|
||||
self.logger,
|
||||
api_client=self.api_client,
|
||||
fetch_from_api=True,
|
||||
local_dump_dirs=ods_dump_dirs,
|
||||
use_local_json=True,
|
||||
)
|
||||
self.logger.info("ODS 层使用本地 JSON 校验（不请求 API）")
|
||||
else:
|
||||
verifier = get_verifier_for_layer(
|
||||
layer,
|
||||
self.db_conn,
|
||||
self.logger,
|
||||
api_client=self.api_client,
|
||||
fetch_from_api=True,
|
||||
)
|
||||
self.logger.info("ODS 层启用 API 数据校验")
|
||||
else:
|
||||
verifier_kwargs: dict[str, Any] = {}
|
||||
if layer.upper() == "INDEX":
|
||||
try:
|
||||
lookback_days = int(self.config.get("run.index_lookback_days", 60))
|
||||
except (TypeError, ValueError):
|
||||
lookback_days = 60
|
||||
verifier_kwargs = {
|
||||
"lookback_days": lookback_days,
|
||||
"config": self.config,
|
||||
}
|
||||
self.logger.info("INDEX 层校验使用回溯天数: %s", lookback_days)
|
||||
if layer.upper() == "DWD":
|
||||
verifier_kwargs["config"] = self.config
|
||||
verifier = get_verifier_for_layer(
|
||||
layer,
|
||||
self.db_conn,
|
||||
self.logger,
|
||||
**verifier_kwargs,
|
||||
)
|
||||
|
||||
# 使用 filter_verify_tables 替代原内联静态方法
|
||||
layer_tables = filter_verify_tables(layer, verify_tables)
|
||||
if verify_tables and not layer_tables:
|
||||
self.logger.info("层 %s 无匹配表，跳过校验", layer)
|
||||
layer_results[layer] = {
|
||||
"status": "SKIPPED",
|
||||
"reason": "table_filter",
|
||||
}
|
||||
continue
|
||||
|
||||
self.logger.info("开始校验层: %s，时间窗口: %s ~ %s", layer, window_start, window_end)
|
||||
|
||||
layer_summary = verifier.verify_and_backfill(
|
||||
window_start=window_start,
|
||||
window_end=window_end,
|
||||
auto_backfill=True,
|
||||
split_unit=window_split or "month",
|
||||
tables=layer_tables,
|
||||
)
|
||||
|
||||
layer_results[layer] = layer_summary.to_dict() if hasattr(layer_summary, 'to_dict') else {}
|
||||
|
||||
if hasattr(layer_summary, 'total_tables'):
|
||||
total_tables += layer_summary.total_tables
|
||||
consistent_tables += layer_summary.consistent_tables
|
||||
total_backfilled += layer_summary.total_backfilled
|
||||
total_error_tables += getattr(layer_summary, 'error_tables', 0)
|
||||
|
||||
self.logger.info(
|
||||
"层 %s 校验完成: 表数=%d, 一致=%d, 错误=%d, 补齐=%d",
|
||||
layer,
|
||||
getattr(layer_summary, 'total_tables', 0),
|
||||
getattr(layer_summary, 'consistent_tables', 0),
|
||||
getattr(layer_summary, 'error_tables', 0),
|
||||
getattr(layer_summary, 'total_backfilled', 0),
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
self.logger.error("层 %s 校验失败: %s", layer, exc, exc_info=True)
|
||||
layer_results[layer] = {"status": "ERROR", "error": str(exc)}
|
||||
|
||||
return {
|
||||
"status": "COMPLETED",
|
||||
"total_tables": total_tables,
|
||||
"consistent_tables": consistent_tables,
|
||||
"total_backfilled": total_backfilled,
|
||||
"error_tables": total_error_tables,
|
||||
"layers": layer_results,
|
||||
}
|
||||
Reference in New Issue
Block a user