""" 通过 ETL CLI 直接重跑集成测试中失败的 DWS/INDEX 任务。 不走后端 API,直接调用 CLI 子进程,与集成测试使用相同参数。 参数对齐集成测试: --layers DWS,INDEX --tasks 失败任务列表 --window-start 2025-11-01 --window-end 2026-02-26 --window-split-days 30 --force-full --processing-mode full_window(CLI 不支持此参数,用 --window-start/end 等效) """ import os import sys import subprocess import time from pathlib import Path from datetime import datetime from dotenv import load_dotenv from zoneinfo import ZoneInfo load_dotenv(Path(__file__).resolve().parents[2] / ".env") TZ = ZoneInfo("Asia/Shanghai") ETL_CWD = Path(__file__).resolve().parents[2] / "apps" / "etl" / "connectors" / "feiqiu" # 之前失败的任务 FAILED_TASKS = [ "DWS_MEMBER_VISIT", "DWS_MEMBER_CONSUMPTION", "DWS_FINANCE_DAILY", "DWS_FINANCE_RECHARGE", "DWS_FINANCE_INCOME_STRUCTURE", "DWS_FINANCE_DISCOUNT_DETAIL", "DWS_ASSISTANT_MONTHLY", "DWS_ASSISTANT_FINANCE", "DWS_WINBACK_INDEX", "DWS_NEWCONV_INDEX", "DWS_RELATION_INDEX", "DWS_SPENDING_POWER_INDEX", ] def run_etl(tasks: list[str]) -> tuple[int, str, str]: """运行 ETL CLI""" # 使用 uv run --package etl-feiqiu 确保 ETL 子包依赖可用 cmd = [ "uv", "run", "--package", "etl-feiqiu", "python", "-m", "cli.main", "--layers", "DWS,INDEX", "--tasks", ",".join(tasks), "--window-start", "2025-11-01 00:00:00", "--window-end", "2026-02-27 00:00:00", "--window-split-days", "30", "--force-full", ] print(f"命令: {' '.join(cmd)}") print(f"工作目录: {ETL_CWD}") print() proc = subprocess.Popen( cmd, cwd=str(ETL_CWD), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", bufsize=1, ) output_lines = [] errors = [] warnings = [] task_results = {} for line in proc.stdout: line = line.rstrip() output_lines.append(line) # 实时输出关键信息 if "ERROR" in line or "CRITICAL" in line: print(f" ❌ {line}") errors.append(line) elif "WARNING" in line or "WARN" in line: if len(warnings) < 20: # 限制警告输出 print(f" ⚠️ {line}") warnings.append(line) elif any(kw in line for kw in ["成功", "完成", "SUCCESS", "DONE"]): print(f" ✅ {line}") elif any(kw in line for kw in ["开始", "执行", "START", "RUNNING"]): print(f" ▶ {line}") elif "失败" in line or "FAILED" in line: print(f" ❌ {line}") # 解析任务结果 for task in FAILED_TASKS: if task in line: if any(kw in line.upper() for kw in ["SUCCESS", "成功", "完成"]): task_results[task] = "SUCCESS" elif any(kw in line.upper() for kw in ["FAIL", "失败", "ERROR"]): task_results[task] = "FAILED" proc.wait() return proc.returncode, "\n".join(output_lines), { "errors": errors, "warnings": warnings, "task_results": task_results, } def main(): now = datetime.now(TZ) print(f"{'='*60}") print(f"失败任务重跑验证(CLI 直连)") print(f"{'='*60}") print(f"时间: {now.isoformat()}") print(f"任务数: {len(FAILED_TASKS)}") print(f"任务: {', '.join(FAILED_TASKS)}") print() start_time = time.time() exit_code, output, analysis = run_etl(FAILED_TASKS) elapsed = time.time() - start_time print(f"\n{'='*60}") print(f"=== 重跑结果 ===") print(f"{'='*60}") print(f"退出码: {exit_code}") print(f"耗时: {elapsed:.0f}s ({elapsed/60:.1f}min)") print(f"错误数: {len(analysis['errors'])}") print(f"警告数: {len(analysis['warnings'])}") print(f"\n--- 任务级结果 ---") for task in FAILED_TASKS: status = analysis['task_results'].get(task, "未检测到") icon = "✅" if status == "SUCCESS" else "❌" if status == "FAILED" else "❓" print(f" {icon} {task}: {status}") if analysis['errors']: print(f"\n--- 错误详情 ---") for i, err in enumerate(analysis['errors'][:30], 1): print(f" {i}. {err[:300]}") # 保存完整输出 log_root = os.environ.get("SYSTEM_LOG_ROOT") if log_root: log_dir = Path(log_root) log_dir.mkdir(parents=True, exist_ok=True) log_file = log_dir / f"{now.strftime('%Y%m%d')}_rerun_failed_cli.log" with open(log_file, "w", encoding="utf-8") as f: f.write(f"退出码: {exit_code}\n") f.write(f"耗时: {elapsed:.0f}s\n") f.write(f"任务: {', '.join(FAILED_TASKS)}\n") f.write(f"{'='*60}\n") f.write(output) print(f"\n完整日志: {log_file}") sys.exit(exit_code) if __name__ == "__main__": main()