168 lines
5.1 KiB
Python
168 lines
5.1 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""第三次执行：验证 BUG 2 (monthly UniqueViolation) + BUG 3 (customer UndefinedColumn) 修复。
|
||
|
||
复用 resubmit_failed.py 的逻辑，提交同样的 31 个任务。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import requests
|
||
|
||
TOKEN_FILE = Path(__file__).parent / ".monitor_token"
|
||
BASE = "http://localhost:8000"
|
||
|
||
REFRESH_TOKEN = (
|
||
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9."
|
||
"eyJzdWIiOiIxIiwic2l0ZV9pZCI6Mjc5MDY4NTQxNTQ0MzI2OSwidHlwZSI6InJlZnJlc2giLCJleHAiOjE3NzIyNjM0NjN9."
|
||
"XYoda5lfxNtTSAGWoLlYhS9cA-hTK9iqK0SqUyn2KV4"
|
||
)
|
||
|
||
|
||
def refresh_access_token() -> str:
|
||
resp = requests.post(
|
||
f"{BASE}/api/auth/refresh",
|
||
json={"refresh_token": REFRESH_TOKEN},
|
||
timeout=10,
|
||
)
|
||
if resp.status_code != 200:
|
||
print(f"❌ 刷新 token 失败: {resp.status_code} {resp.text}")
|
||
sys.exit(1)
|
||
token = resp.json()["access_token"]
|
||
TOKEN_FILE.write_text(token, encoding="utf-8")
|
||
print("✅ access_token 已刷新")
|
||
return token
|
||
|
||
|
||
TOKEN = refresh_access_token()
|
||
HEADERS = {"Authorization": f"Bearer {TOKEN}", "Content-Type": "application/json"}
|
||
|
||
TASKS = [
|
||
"DWS_ASSISTANT_DAILY",
|
||
"DWS_ASSISTANT_MONTHLY",
|
||
"DWS_ASSISTANT_CUSTOMER",
|
||
"DWS_ASSISTANT_SALARY",
|
||
"DWS_ASSISTANT_FINANCE",
|
||
"ODS_SETTLEMENT_RECORDS",
|
||
"ODS_PAYMENT",
|
||
"ODS_REFUND",
|
||
"DWS_BUILD_ORDER_SUMMARY",
|
||
"DWS_MEMBER_CONSUMPTION",
|
||
"DWS_MEMBER_VISIT",
|
||
"ODS_GOODS_CATEGORY",
|
||
"ODS_STORE_GOODS",
|
||
"ODS_STORE_GOODS_SALES",
|
||
"ODS_TENANT_GOODS",
|
||
"ODS_PLATFORM_COUPON",
|
||
"ODS_GROUP_PACKAGE",
|
||
"ODS_GROUP_BUY_REDEMPTION",
|
||
"ODS_INVENTORY_STOCK",
|
||
"ODS_INVENTORY_CHANGE",
|
||
"DWS_GOODS_STOCK_DAILY",
|
||
"DWS_GOODS_STOCK_WEEKLY",
|
||
"DWS_GOODS_STOCK_MONTHLY",
|
||
"DWS_FINANCE_DAILY",
|
||
"DWS_FINANCE_RECHARGE",
|
||
"DWS_FINANCE_INCOME_STRUCTURE",
|
||
"DWS_FINANCE_DISCOUNT_DETAIL",
|
||
"DWS_WINBACK_INDEX",
|
||
"DWS_NEWCONV_INDEX",
|
||
"DWS_RELATION_INDEX",
|
||
"DWD_LOAD_FROM_ODS",
|
||
]
|
||
|
||
config = {
|
||
"tasks": TASKS,
|
||
"flow": "api_full",
|
||
"processing_mode": "full_window",
|
||
"window_mode": "custom",
|
||
"window_start": "2025-11-01",
|
||
"window_end": "2026-02-20",
|
||
"window_split": "month",
|
||
"window_split_days": 30,
|
||
"force_full": True,
|
||
"dry_run": False,
|
||
"lookback_hours": 24,
|
||
"overlap_seconds": 600,
|
||
}
|
||
|
||
print(f"📤 第三次执行：提交 {len(TASKS)} 个任务...")
|
||
resp = requests.post(f"{BASE}/api/execution/run", headers=HEADERS, json=config, timeout=30)
|
||
if resp.status_code != 200:
|
||
print(f"❌ 提交失败: {resp.status_code} {resp.text}")
|
||
sys.exit(1)
|
||
|
||
data = resp.json()
|
||
execution_id = data["execution_id"]
|
||
print(f"✅ execution_id={execution_id}")
|
||
print(" 轮询等待完成...")
|
||
|
||
poll_interval = 20
|
||
max_wait = 1800
|
||
elapsed = 0
|
||
|
||
while elapsed < max_wait:
|
||
time.sleep(poll_interval)
|
||
elapsed += poll_interval
|
||
mm, ss = divmod(elapsed, 60)
|
||
|
||
try:
|
||
hist_resp = requests.get(
|
||
f"{BASE}/api/execution/history?limit=5",
|
||
headers=HEADERS,
|
||
timeout=15,
|
||
)
|
||
if hist_resp.status_code == 401:
|
||
print(f" [{mm}m{ss}s] token 过期，刷新...")
|
||
TOKEN = refresh_access_token()
|
||
HEADERS["Authorization"] = f"Bearer {TOKEN}"
|
||
continue
|
||
if hist_resp.status_code != 200:
|
||
print(f" [{mm}m{ss}s] 查询失败: {hist_resp.status_code}")
|
||
continue
|
||
|
||
history = hist_resp.json()
|
||
target = next((h for h in history if h["id"] == execution_id), None)
|
||
if not target:
|
||
print(f" [{mm}m{ss}s] 等待执行记录...")
|
||
continue
|
||
|
||
status = target.get("status", "unknown")
|
||
duration_ms = target.get("duration_ms")
|
||
dur = f"{duration_ms / 1000:.1f}s" if duration_ms else "—"
|
||
|
||
if status in ("success", "failed", "cancelled"):
|
||
print(f"\n🏁 完成: status={status}, 耗时={dur}, exit_code={target.get('exit_code')}")
|
||
|
||
log_resp = requests.get(
|
||
f"{BASE}/api/execution/{execution_id}/logs",
|
||
headers=HEADERS,
|
||
timeout=30,
|
||
)
|
||
if log_resp.status_code == 200:
|
||
log_data = log_resp.json()
|
||
# 保存完整日志到文件供后续分析
|
||
from _env_paths import get_output_path
|
||
out_dir = get_output_path("SYSTEM_LOG_ROOT")
|
||
raw_file = out_dir / "2026-02-21__etl_run_raw_v3.json"
|
||
raw_file.write_text(json.dumps(log_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
print(f" 原始日志已保存: {raw_file}")
|
||
|
||
error_log = log_data.get("error_log", "") or ""
|
||
lines = error_log.strip().split("\n")
|
||
print(f"\n--- error_log 末尾 80 行 ---")
|
||
for line in lines[-80:]:
|
||
print(line)
|
||
break
|
||
else:
|
||
print(f" [{mm}m{ss}s] status={status}")
|
||
|
||
except Exception as e:
|
||
print(f" [{mm}m{ss}s] 异常: {e}")
|
||
|
||
else:
|
||
print(f"\n⏰ 超时（{max_wait}s），请手动检查 execution_id={execution_id}")
|