微信小程序页面迁移校验之前 P5任务处理之前

scripts/ops/_rerun_failed_cli.py

@@ -0,0 +1,165 @@
+"""
+通过 ETL CLI 直接重跑集成测试中失败的 DWS/INDEX 任务。
+不走后端 API，直接调用 CLI 子进程，与集成测试使用相同参数。
+
+参数对齐集成测试：
+  --layers DWS,INDEX
+  --tasks 失败任务列表
+  --window-start 2025-11-01
+  --window-end 2026-02-26
+  --window-split-days 30
+  --force-full
+  --processing-mode full_window（CLI 不支持此参数，用 --window-start/end 等效）
+"""
+import os
+import sys
+import subprocess
+import time
+from pathlib import Path
+from datetime import datetime
+from dotenv import load_dotenv
+from zoneinfo import ZoneInfo
+
+load_dotenv(Path(__file__).resolve().parents[2] / ".env")
+
+TZ = ZoneInfo("Asia/Shanghai")
+ETL_CWD = Path(__file__).resolve().parents[2] / "apps" / "etl" / "connectors" / "feiqiu"
+
+# 之前失败的任务
+FAILED_TASKS = [
+    "DWS_MEMBER_VISIT",
+    "DWS_MEMBER_CONSUMPTION",
+    "DWS_FINANCE_DAILY",
+    "DWS_FINANCE_RECHARGE",
+    "DWS_FINANCE_INCOME_STRUCTURE",
+    "DWS_FINANCE_DISCOUNT_DETAIL",
+    "DWS_ASSISTANT_MONTHLY",
+    "DWS_ASSISTANT_FINANCE",
+    "DWS_WINBACK_INDEX",
+    "DWS_NEWCONV_INDEX",
+    "DWS_RELATION_INDEX",
+    "DWS_SPENDING_POWER_INDEX",
+]
+
+
+def run_etl(tasks: list[str]) -> tuple[int, str, str]:
+    """运行 ETL CLI"""
+    # 使用 uv run --package etl-feiqiu 确保 ETL 子包依赖可用
+    cmd = [
+        "uv", "run", "--package", "etl-feiqiu",
+        "python", "-m", "cli.main",
+        "--layers", "DWS,INDEX",
+        "--tasks", ",".join(tasks),
+        "--window-start", "2025-11-01 00:00:00",
+        "--window-end", "2026-02-27 00:00:00",
+        "--window-split-days", "30",
+        "--force-full",
+    ]
+
+    print(f"命令: {' '.join(cmd)}")
+    print(f"工作目录: {ETL_CWD}")
+    print()
+
+    proc = subprocess.Popen(
+        cmd,
+        cwd=str(ETL_CWD),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        bufsize=1,
+    )
+
+    output_lines = []
+    errors = []
+    warnings = []
+    task_results = {}
+
+    for line in proc.stdout:
+        line = line.rstrip()
+        output_lines.append(line)
+
+        # 实时输出关键信息
+        if "ERROR" in line or "CRITICAL" in line:
+            print(f"  ❌ {line}")
+            errors.append(line)
+        elif "WARNING" in line or "WARN" in line:
+            if len(warnings) < 20:  # 限制警告输出
+                print(f"  ⚠️  {line}")
+            warnings.append(line)
+        elif any(kw in line for kw in ["成功", "完成", "SUCCESS", "DONE"]):
+            print(f"  ✅ {line}")
+        elif any(kw in line for kw in ["开始", "执行", "START", "RUNNING"]):
+            print(f"  ▶ {line}")
+        elif "失败" in line or "FAILED" in line:
+            print(f"  ❌ {line}")
+
+        # 解析任务结果
+        for task in FAILED_TASKS:
+            if task in line:
+                if any(kw in line.upper() for kw in ["SUCCESS", "成功", "完成"]):
+                    task_results[task] = "SUCCESS"
+                elif any(kw in line.upper() for kw in ["FAIL", "失败", "ERROR"]):
+                    task_results[task] = "FAILED"
+
+    proc.wait()
+    return proc.returncode, "\n".join(output_lines), {
+        "errors": errors,
+        "warnings": warnings,
+        "task_results": task_results,
+    }
+
+
+def main():
+    now = datetime.now(TZ)
+    print(f"{'='*60}")
+    print(f"失败任务重跑验证（CLI 直连）")
+    print(f"{'='*60}")
+    print(f"时间: {now.isoformat()}")
+    print(f"任务数: {len(FAILED_TASKS)}")
+    print(f"任务: {', '.join(FAILED_TASKS)}")
+    print()
+
+    start_time = time.time()
+    exit_code, output, analysis = run_etl(FAILED_TASKS)
+    elapsed = time.time() - start_time
+
+    print(f"\n{'='*60}")
+    print(f"=== 重跑结果 ===")
+    print(f"{'='*60}")
+    print(f"退出码: {exit_code}")
+    print(f"耗时: {elapsed:.0f}s ({elapsed/60:.1f}min)")
+    print(f"错误数: {len(analysis['errors'])}")
+    print(f"警告数: {len(analysis['warnings'])}")
+
+    print(f"\n--- 任务级结果 ---")
+    for task in FAILED_TASKS:
+        status = analysis['task_results'].get(task, "未检测到")
+        icon = "✅" if status == "SUCCESS" else "❌" if status == "FAILED" else "❓"
+        print(f"  {icon} {task}: {status}")
+
+    if analysis['errors']:
+        print(f"\n--- 错误详情 ---")
+        for i, err in enumerate(analysis['errors'][:30], 1):
+            print(f"  {i}. {err[:300]}")
+
+    # 保存完整输出
+    log_root = os.environ.get("SYSTEM_LOG_ROOT")
+    if log_root:
+        log_dir = Path(log_root)
+        log_dir.mkdir(parents=True, exist_ok=True)
+        log_file = log_dir / f"{now.strftime('%Y%m%d')}_rerun_failed_cli.log"
+        with open(log_file, "w", encoding="utf-8") as f:
+            f.write(f"退出码: {exit_code}\n")
+            f.write(f"耗时: {elapsed:.0f}s\n")
+            f.write(f"任务: {', '.join(FAILED_TASKS)}\n")
+            f.write(f"{'='*60}\n")
+            f.write(output)
+        print(f"\n完整日志: {log_file}")
+
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()