开发机迁移

This commit is contained in:
Neo
2026-04-10 06:24:13 +08:00
parent f65c1d038b
commit 79d3c2e97e
50 changed files with 1565 additions and 318 deletions

View File

@@ -224,6 +224,9 @@ class FlowRunner:
summary_text = flow_logger.end(status="成功")
self.logger.info("\n%s", summary_text)
# CHANGE 2026-04-07 | Fix-12ETL 完成后通知后端触发任务编排
self._notify_backend_etl_completed(run_label)
return {
"status": "SUCCESS",
"flow": run_label,
@@ -243,6 +246,33 @@ class FlowRunner:
self.logger.error("\n%s", summary_text)
raise
def _notify_backend_etl_completed(self, pipeline: str) -> None:
"""ETL 完成后通知后端触发任务编排recall_detector → task_generator
CHANGE 2026-04-07 | Fix-12失败不阻断主流程仅记录警告。
"""
import os
backend_url = os.getenv("BACKEND_API_URL", "http://127.0.0.1:8000")
internal_token = os.getenv("INTERNAL_API_TOKEN", "")
if not internal_token:
self.logger.warning("ETL 完成回调跳过INTERNAL_API_TOKEN 未配置")
return
try:
import httpx
resp = httpx.post(
f"{backend_url}/api/internal/etl-completed",
json={"pipeline": pipeline},
headers={"Authorization": f"Internal-Token {internal_token}"},
timeout=300,
)
if resp.status_code == 200:
self.logger.info("ETL 完成回调成功: %s", resp.json())
else:
self.logger.warning("ETL 完成回调失败: status=%d, body=%s", resp.status_code, resp.text)
except Exception:
self.logger.warning("ETL 完成回调异常(不阻断主流程)", exc_info=True)
def _run_post_consistency_check(self, timer: EtlTimer) -> str | None:
"""ETL 完成后运行数据一致性检查,输出黑盒测试报告。

View File

@@ -21,6 +21,8 @@ from pathlib import Path
from typing import Any, Dict, List
from zoneinfo import ZoneInfo
import psycopg2
from api.recording_client import RecordingAPIClient
from api.local_json_client import LocalJsonClient
from orchestration.cursor_manager import CursorManager
@@ -91,6 +93,17 @@ class TaskExecutor:
self.logger.info("开始运行任务: %s, run_uuid=%s", task_codes, run_uuid)
for task_code in task_codes:
# 每个任务前检测连接状态,断线自动重连
if not self.db_ops.ensure_open():
self.logger.error("数据库连接不可用且重连失败,跳过任务 %s", task_code)
results.append({
"task_code": task_code,
"status": "失败",
"error": "数据库连接不可用",
"counts": {},
})
continue
# 为每个任务创建独立的日志缓冲区,避免多任务日志交叉
task_log_buf = TaskLogBuffer(task_code, self.logger)
try:
@@ -113,9 +126,13 @@ class TaskExecutor:
task_log_buf.error("任务失败: %s", exc)
# CHANGE 2026-02-24 | 任务失败后 rollback防止 InFailedSqlTransaction 级联
try:
self.db.rollback()
self.db_ops.rollback()
except Exception:
pass
# 连接级异常:尝试重连,避免后续任务全部级联失败
if isinstance(exc, (psycopg2.OperationalError, psycopg2.InterfaceError)):
self.logger.warning("检测到连接异常,尝试重连...")
self.db_ops.ensure_open()
results.append({
"task_code": task_code,
"status": "失败",

View File

@@ -43,6 +43,7 @@ from tasks.dws import (
FinanceRechargeTask,
FinanceIncomeStructureTask,
FinanceDiscountDetailTask,
FinanceAreaDailyTask,
# 库存汇总任务
GoodsStockDailyTask,
GoodsStockWeeklyTask,
@@ -167,6 +168,7 @@ default_registry.register("DWS_FINANCE_DAILY", FinanceDailyTask, layer="DWS")
default_registry.register("DWS_FINANCE_RECHARGE", FinanceRechargeTask, layer="DWS")
default_registry.register("DWS_FINANCE_INCOME_STRUCTURE", FinanceIncomeStructureTask, layer="DWS")
default_registry.register("DWS_FINANCE_DISCOUNT_DETAIL", FinanceDiscountDetailTask, layer="DWS")
default_registry.register("DWS_FINANCE_AREA_DAILY", FinanceAreaDailyTask, layer="DWS", depends_on=["DWS_FINANCE_DAILY"])
# CHANGE [2026-03-27] intent: 移除对 DWD_LOAD_FROM_ODS 的显式依赖dwd_dws Flow 下该依赖不在批次内只产生无意义 warning
default_registry.register("DWS_GOODS_STOCK_DAILY", GoodsStockDailyTask, layer="DWS")
default_registry.register("DWS_GOODS_STOCK_WEEKLY", GoodsStockWeeklyTask, layer="DWS")
@@ -182,6 +184,7 @@ default_registry.register("DWS_MAINTENANCE", DwsMaintenanceTask, layer="DWS", de
"DWS_ASSISTANT_PROJECT_TAG", "DWS_MEMBER_PROJECT_TAG",
"DWS_FINANCE_DAILY", "DWS_FINANCE_RECHARGE",
"DWS_FINANCE_INCOME_STRUCTURE", "DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_FINANCE_AREA_DAILY",
"DWS_BUILD_ORDER_SUMMARY",
"DWS_GOODS_STOCK_DAILY", "DWS_GOODS_STOCK_WEEKLY", "DWS_GOODS_STOCK_MONTHLY",
])