73 lines
2.9 KiB
Python
73 lines
2.9 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""提取 v8 的 DWD_LOAD_FROM_ODS 完成统计和所有任务最终状态。"""
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv(Path(__file__).resolve().parents[2] / ".env")
|
|
from _env_paths import get_output_path
|
|
|
|
log_dir = get_output_path("SYSTEM_LOG_ROOT")
|
|
raw = json.loads((log_dir / "2026-02-21__etl_run_raw_v8.json").read_text("utf-8"))
|
|
|
|
error_log = raw.get("error_log", "")
|
|
output_log = raw.get("output_log", "")
|
|
full = output_log + "\n" + error_log
|
|
|
|
# 提取 DWD_LOAD_FROM_ODS 完成行
|
|
for line in full.split("\n"):
|
|
if "DWD_LOAD_FROM_ODS: 完成" in line:
|
|
# 解析 JSON 部分
|
|
idx = line.find("统计=")
|
|
if idx >= 0:
|
|
stats_str = line[idx + len("统计="):]
|
|
# 尝试解析为 Python dict
|
|
try:
|
|
stats = eval(stats_str)
|
|
print("=== DWD_LOAD_FROM_ODS 统计 ===")
|
|
for t in stats.get("tables", []):
|
|
icon = "✅" if t.get("inserted", 0) + t.get("updated", 0) > 0 or t.get("processed", 0) > 0 else "⚪"
|
|
print(f" {icon} {t['table']} | mode={t.get('mode','?')} | processed={t.get('processed',0)} ins={t.get('inserted',0)} upd={t.get('updated',0)} skip={t.get('skipped',0)}")
|
|
errors = stats.get("errors", [])
|
|
if errors:
|
|
print(f"\n 错误 ({len(errors)} 个):")
|
|
for e in errors:
|
|
print(f" ❌ {e.get('table','?')}: {str(e.get('error',''))[:100]}")
|
|
except Exception as ex:
|
|
print(f"解析失败: {ex}")
|
|
print(stats_str[:500])
|
|
break
|
|
|
|
# 查找所有任务的最终状态（从 output_log 中找"所有任务执行完成"之前的状态）
|
|
print("\n=== 任务执行顺序与状态 ===")
|
|
task_status = {}
|
|
for line in full.split("\n"):
|
|
m = re.search(r"任务\s+(\S+)\s+执行成功", line)
|
|
if m:
|
|
task_status[m.group(1)] = "✅ 成功"
|
|
m = re.search(r"任务\s+(\S+)\s+失败", line)
|
|
if m and m.group(1) not in task_status:
|
|
task_status[m.group(1)] = "❌ 失败"
|
|
|
|
# 预期任务列表
|
|
expected = [
|
|
"ODS_FETCH", "DWD_LOAD_FROM_ODS",
|
|
"DWS_ASSISTANT_DAILY", "DWS_ASSISTANT_MONTHLY",
|
|
"DWS_ASSISTANT_CUSTOMER", "DWS_ASSISTANT_SALARY",
|
|
"DWS_ASSISTANT_FINANCE",
|
|
"DWS_MEMBER_CONSUMPTION", "DWS_MEMBER_VISIT",
|
|
"DWS_GOODS_STOCK_DAILY", "DWS_GOODS_STOCK_WEEKLY", "DWS_GOODS_STOCK_MONTHLY",
|
|
"DWS_FINANCE_DAILY", "DWS_FINANCE_RECHARGE",
|
|
"DWS_FINANCE_INCOME_STRUCTURE", "DWS_FINANCE_DISCOUNT_DETAIL",
|
|
"DWS_WINBACK_INDEX", "DWS_NEWCONV_INDEX", "DWS_RELATION_INDEX",
|
|
]
|
|
|
|
for t in expected:
|
|
status = task_status.get(t, "⚪ 未知")
|
|
print(f" {status} — {t}")
|
|
|
|
s_count = sum(1 for v in task_status.values() if "成功" in v)
|
|
f_count = sum(1 for v in task_status.values() if "失败" in v)
|
|
print(f"\n合计: {s_count} 成功, {f_count} 失败, {len(expected) - s_count - f_count} 未知")
|