开发机迁移
This commit is contained in:
@@ -224,6 +224,9 @@ class FlowRunner:
|
||||
summary_text = flow_logger.end(status="成功")
|
||||
self.logger.info("\n%s", summary_text)
|
||||
|
||||
# CHANGE 2026-04-07 | Fix-12:ETL 完成后通知后端触发任务编排
|
||||
self._notify_backend_etl_completed(run_label)
|
||||
|
||||
return {
|
||||
"status": "SUCCESS",
|
||||
"flow": run_label,
|
||||
@@ -243,6 +246,33 @@ class FlowRunner:
|
||||
self.logger.error("\n%s", summary_text)
|
||||
raise
|
||||
|
||||
def _notify_backend_etl_completed(self, pipeline: str) -> None:
|
||||
"""ETL 完成后通知后端触发任务编排(recall_detector → task_generator)。
|
||||
|
||||
CHANGE 2026-04-07 | Fix-12:失败不阻断主流程,仅记录警告。
|
||||
"""
|
||||
import os
|
||||
backend_url = os.getenv("BACKEND_API_URL", "http://127.0.0.1:8000")
|
||||
internal_token = os.getenv("INTERNAL_API_TOKEN", "")
|
||||
if not internal_token:
|
||||
self.logger.warning("ETL 完成回调跳过:INTERNAL_API_TOKEN 未配置")
|
||||
return
|
||||
|
||||
try:
|
||||
import httpx
|
||||
resp = httpx.post(
|
||||
f"{backend_url}/api/internal/etl-completed",
|
||||
json={"pipeline": pipeline},
|
||||
headers={"Authorization": f"Internal-Token {internal_token}"},
|
||||
timeout=300,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
self.logger.info("ETL 完成回调成功: %s", resp.json())
|
||||
else:
|
||||
self.logger.warning("ETL 完成回调失败: status=%d, body=%s", resp.status_code, resp.text)
|
||||
except Exception:
|
||||
self.logger.warning("ETL 完成回调异常(不阻断主流程)", exc_info=True)
|
||||
|
||||
def _run_post_consistency_check(self, timer: EtlTimer) -> str | None:
|
||||
"""ETL 完成后运行数据一致性检查,输出黑盒测试报告。
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import psycopg2
|
||||
|
||||
from api.recording_client import RecordingAPIClient
|
||||
from api.local_json_client import LocalJsonClient
|
||||
from orchestration.cursor_manager import CursorManager
|
||||
@@ -91,6 +93,17 @@ class TaskExecutor:
|
||||
self.logger.info("开始运行任务: %s, run_uuid=%s", task_codes, run_uuid)
|
||||
|
||||
for task_code in task_codes:
|
||||
# 每个任务前检测连接状态,断线自动重连
|
||||
if not self.db_ops.ensure_open():
|
||||
self.logger.error("数据库连接不可用且重连失败,跳过任务 %s", task_code)
|
||||
results.append({
|
||||
"task_code": task_code,
|
||||
"status": "失败",
|
||||
"error": "数据库连接不可用",
|
||||
"counts": {},
|
||||
})
|
||||
continue
|
||||
|
||||
# 为每个任务创建独立的日志缓冲区,避免多任务日志交叉
|
||||
task_log_buf = TaskLogBuffer(task_code, self.logger)
|
||||
try:
|
||||
@@ -113,9 +126,13 @@ class TaskExecutor:
|
||||
task_log_buf.error("任务失败: %s", exc)
|
||||
# CHANGE 2026-02-24 | 任务失败后 rollback,防止 InFailedSqlTransaction 级联
|
||||
try:
|
||||
self.db.rollback()
|
||||
self.db_ops.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
# 连接级异常:尝试重连,避免后续任务全部级联失败
|
||||
if isinstance(exc, (psycopg2.OperationalError, psycopg2.InterfaceError)):
|
||||
self.logger.warning("检测到连接异常,尝试重连...")
|
||||
self.db_ops.ensure_open()
|
||||
results.append({
|
||||
"task_code": task_code,
|
||||
"status": "失败",
|
||||
|
||||
@@ -43,6 +43,7 @@ from tasks.dws import (
|
||||
FinanceRechargeTask,
|
||||
FinanceIncomeStructureTask,
|
||||
FinanceDiscountDetailTask,
|
||||
FinanceAreaDailyTask,
|
||||
# 库存汇总任务
|
||||
GoodsStockDailyTask,
|
||||
GoodsStockWeeklyTask,
|
||||
@@ -167,6 +168,7 @@ default_registry.register("DWS_FINANCE_DAILY", FinanceDailyTask, layer="DWS")
|
||||
default_registry.register("DWS_FINANCE_RECHARGE", FinanceRechargeTask, layer="DWS")
|
||||
default_registry.register("DWS_FINANCE_INCOME_STRUCTURE", FinanceIncomeStructureTask, layer="DWS")
|
||||
default_registry.register("DWS_FINANCE_DISCOUNT_DETAIL", FinanceDiscountDetailTask, layer="DWS")
|
||||
default_registry.register("DWS_FINANCE_AREA_DAILY", FinanceAreaDailyTask, layer="DWS", depends_on=["DWS_FINANCE_DAILY"])
|
||||
# CHANGE [2026-03-27] intent: 移除对 DWD_LOAD_FROM_ODS 的显式依赖,dwd_dws Flow 下该依赖不在批次内只产生无意义 warning
|
||||
default_registry.register("DWS_GOODS_STOCK_DAILY", GoodsStockDailyTask, layer="DWS")
|
||||
default_registry.register("DWS_GOODS_STOCK_WEEKLY", GoodsStockWeeklyTask, layer="DWS")
|
||||
@@ -182,6 +184,7 @@ default_registry.register("DWS_MAINTENANCE", DwsMaintenanceTask, layer="DWS", de
|
||||
"DWS_ASSISTANT_PROJECT_TAG", "DWS_MEMBER_PROJECT_TAG",
|
||||
"DWS_FINANCE_DAILY", "DWS_FINANCE_RECHARGE",
|
||||
"DWS_FINANCE_INCOME_STRUCTURE", "DWS_FINANCE_DISCOUNT_DETAIL",
|
||||
"DWS_FINANCE_AREA_DAILY",
|
||||
"DWS_BUILD_ORDER_SUMMARY",
|
||||
"DWS_GOODS_STOCK_DAILY", "DWS_GOODS_STOCK_WEEKLY", "DWS_GOODS_STOCK_MONTHLY",
|
||||
])
|
||||
|
||||
Reference in New Issue
Block a user