166 lines
5.0 KiB
Python
166 lines
5.0 KiB
Python
"""
|
||
通过 ETL CLI 直接重跑集成测试中失败的 DWS/INDEX 任务。
|
||
不走后端 API，直接调用 CLI 子进程，与集成测试使用相同参数。
|
||
|
||
参数对齐集成测试：
|
||
--layers DWS,INDEX
|
||
--tasks 失败任务列表
|
||
--window-start 2025-11-01
|
||
--window-end 2026-02-26
|
||
--window-split-days 30
|
||
--force-full
|
||
--processing-mode full_window（CLI 不支持此参数，用 --window-start/end 等效）
|
||
"""
|
||
import os
|
||
import sys
|
||
import subprocess
|
||
import time
|
||
from pathlib import Path
|
||
from datetime import datetime
|
||
from dotenv import load_dotenv
|
||
from zoneinfo import ZoneInfo
|
||
|
||
load_dotenv(Path(__file__).resolve().parents[2] / ".env")
|
||
|
||
TZ = ZoneInfo("Asia/Shanghai")
|
||
ETL_CWD = Path(__file__).resolve().parents[2] / "apps" / "etl" / "connectors" / "feiqiu"
|
||
|
||
# 之前失败的任务
|
||
FAILED_TASKS = [
|
||
"DWS_MEMBER_VISIT",
|
||
"DWS_MEMBER_CONSUMPTION",
|
||
"DWS_FINANCE_DAILY",
|
||
"DWS_FINANCE_RECHARGE",
|
||
"DWS_FINANCE_INCOME_STRUCTURE",
|
||
"DWS_FINANCE_DISCOUNT_DETAIL",
|
||
"DWS_ASSISTANT_MONTHLY",
|
||
"DWS_ASSISTANT_FINANCE",
|
||
"DWS_WINBACK_INDEX",
|
||
"DWS_NEWCONV_INDEX",
|
||
"DWS_RELATION_INDEX",
|
||
"DWS_SPENDING_POWER_INDEX",
|
||
]
|
||
|
||
|
||
def run_etl(tasks: list[str]) -> tuple[int, str, str]:
|
||
"""运行 ETL CLI"""
|
||
# 使用 uv run --package etl-feiqiu 确保 ETL 子包依赖可用
|
||
cmd = [
|
||
"uv", "run", "--package", "etl-feiqiu",
|
||
"python", "-m", "cli.main",
|
||
"--layers", "DWS,INDEX",
|
||
"--tasks", ",".join(tasks),
|
||
"--window-start", "2025-11-01 00:00:00",
|
||
"--window-end", "2026-02-27 00:00:00",
|
||
"--window-split-days", "30",
|
||
"--force-full",
|
||
]
|
||
|
||
print(f"命令: {' '.join(cmd)}")
|
||
print(f"工作目录: {ETL_CWD}")
|
||
print()
|
||
|
||
proc = subprocess.Popen(
|
||
cmd,
|
||
cwd=str(ETL_CWD),
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.STDOUT,
|
||
text=True,
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
bufsize=1,
|
||
)
|
||
|
||
output_lines = []
|
||
errors = []
|
||
warnings = []
|
||
task_results = {}
|
||
|
||
for line in proc.stdout:
|
||
line = line.rstrip()
|
||
output_lines.append(line)
|
||
|
||
# 实时输出关键信息
|
||
if "ERROR" in line or "CRITICAL" in line:
|
||
print(f" ❌ {line}")
|
||
errors.append(line)
|
||
elif "WARNING" in line or "WARN" in line:
|
||
if len(warnings) < 20: # 限制警告输出
|
||
print(f" ⚠️ {line}")
|
||
warnings.append(line)
|
||
elif any(kw in line for kw in ["成功", "完成", "SUCCESS", "DONE"]):
|
||
print(f" ✅ {line}")
|
||
elif any(kw in line for kw in ["开始", "执行", "START", "RUNNING"]):
|
||
print(f" ▶ {line}")
|
||
elif "失败" in line or "FAILED" in line:
|
||
print(f" ❌ {line}")
|
||
|
||
# 解析任务结果
|
||
for task in FAILED_TASKS:
|
||
if task in line:
|
||
if any(kw in line.upper() for kw in ["SUCCESS", "成功", "完成"]):
|
||
task_results[task] = "SUCCESS"
|
||
elif any(kw in line.upper() for kw in ["FAIL", "失败", "ERROR"]):
|
||
task_results[task] = "FAILED"
|
||
|
||
proc.wait()
|
||
return proc.returncode, "\n".join(output_lines), {
|
||
"errors": errors,
|
||
"warnings": warnings,
|
||
"task_results": task_results,
|
||
}
|
||
|
||
|
||
def main():
|
||
now = datetime.now(TZ)
|
||
print(f"{'='*60}")
|
||
print(f"失败任务重跑验证（CLI 直连）")
|
||
print(f"{'='*60}")
|
||
print(f"时间: {now.isoformat()}")
|
||
print(f"任务数: {len(FAILED_TASKS)}")
|
||
print(f"任务: {', '.join(FAILED_TASKS)}")
|
||
print()
|
||
|
||
start_time = time.time()
|
||
exit_code, output, analysis = run_etl(FAILED_TASKS)
|
||
elapsed = time.time() - start_time
|
||
|
||
print(f"\n{'='*60}")
|
||
print(f"=== 重跑结果 ===")
|
||
print(f"{'='*60}")
|
||
print(f"退出码: {exit_code}")
|
||
print(f"耗时: {elapsed:.0f}s ({elapsed/60:.1f}min)")
|
||
print(f"错误数: {len(analysis['errors'])}")
|
||
print(f"警告数: {len(analysis['warnings'])}")
|
||
|
||
print(f"\n--- 任务级结果 ---")
|
||
for task in FAILED_TASKS:
|
||
status = analysis['task_results'].get(task, "未检测到")
|
||
icon = "✅" if status == "SUCCESS" else "❌" if status == "FAILED" else "❓"
|
||
print(f" {icon} {task}: {status}")
|
||
|
||
if analysis['errors']:
|
||
print(f"\n--- 错误详情 ---")
|
||
for i, err in enumerate(analysis['errors'][:30], 1):
|
||
print(f" {i}. {err[:300]}")
|
||
|
||
# 保存完整输出
|
||
log_root = os.environ.get("SYSTEM_LOG_ROOT")
|
||
if log_root:
|
||
log_dir = Path(log_root)
|
||
log_dir.mkdir(parents=True, exist_ok=True)
|
||
log_file = log_dir / f"{now.strftime('%Y%m%d')}_rerun_failed_cli.log"
|
||
with open(log_file, "w", encoding="utf-8") as f:
|
||
f.write(f"退出码: {exit_code}\n")
|
||
f.write(f"耗时: {elapsed:.0f}s\n")
|
||
f.write(f"任务: {', '.join(FAILED_TASKS)}\n")
|
||
f.write(f"{'='*60}\n")
|
||
f.write(output)
|
||
print(f"\n完整日志: {log_file}")
|
||
|
||
sys.exit(exit_code)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|