root/feiqiu-ETL
Template
1
0
Fork 0
Code Pull Requests Packages Releases Wiki Activity

改 相对路径 完成客户端

Browse Source
This commit is contained in:
Neo
2026-01-27 22:14:01 +08:00
parent 04c064793a
commit 9f8976e75a
292 changed files with 307062 additions and 678 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
etl_billiards/gui/workers/task_worker.py
View File

@@ -144,24 +144,166 @@ class TaskWorker(QThread):
if not self._output_lines:
return "无输出"
# 查找关键信息
return self._parse_detailed_summary()
def _parse_detailed_summary(self) -> str:
"""解析详细的执行摘要"""
import re
import json
summary_parts = []
for line in self._output_lines[-20:]: # 只看最后 20 行
line_lower = line.lower()
if "success" in line_lower or "完成" in line or "成功" in line:
summary_parts.append(line)
elif "error" in line_lower or "失败" in line or "错误" in line:
summary_parts.append(line)
elif "inserted" in line_lower or "updated" in line_lower:
summary_parts.append(line)
elif "fetched" in line_lower or "抓取" in line:
summary_parts.append(line)
# 统计各类信息
ods_stats = [] # ODS 抓取统计
dwd_stats = [] # DWD 装载统计
integrity_stats = {} # 数据校验统计
errors = [] # 错误信息
task_results = [] # 任务结果
for line in self._output_lines:
# 1. 解析 ODS 抓取完成信息
# 格式: "xxx: 抓取完成，文件=xxx，记录数=123"
match = re.search(r'(\w+): 抓取完成.*记录数[=:]\s*(\d+)', line)
if match:
task_name = match.group(1)
record_count = int(match.group(2))
if record_count > 0:
ods_stats.append(f"{task_name}: {record_count}条")
continue
# 2. 解析 DWD 装载完成信息
# 格式: "DWD 装载完成：xxx，用时 1.02s"
match = re.search(r'DWD 装载完成[：:]\s*(\S+).*用时\s*([\d.]+)s', line)
if match:
table_name = match.group(1).replace('billiards_dwd.', '')
continue
# 3. 解析任务完成统计 (JSON格式)
# 格式: "xxx: 完成，统计={'tables': [...]}"
if "完成，统计=" in line or "完成,统计=" in line:
try:
match = re.search(r"统计=(\{.+\})", line)
if match:
stats_str = match.group(1).replace("'", '"')
stats = json.loads(stats_str)
# 解析 DWD 装载统计
if 'tables' in stats:
total_processed = 0
total_inserted = 0
tables_with_data = []
for tbl in stats['tables']:
table_name = tbl.get('table', '').replace('billiards_dwd.', '')
processed = tbl.get('processed', 0)
inserted = tbl.get('inserted', 0)
if processed > 0:
total_processed += processed
tables_with_data.append(f"{table_name}({processed})")
elif inserted > 0:
total_inserted += inserted
tables_with_data.append(f"{table_name}(+{inserted})")
if total_processed > 0 or total_inserted > 0:
dwd_stats.append(f"处理维度: {total_processed}条, 新增事实: {total_inserted}条")
if len(tables_with_data) <= 5:
dwd_stats.append(f"涉及表: {', '.join(tables_with_data)}")
else:
dwd_stats.append(f"涉及 {len(tables_with_data)} 张表")
except Exception:
pass
continue
# 4. 解析数据校验结果
# 格式: "CHECK_DONE task=xxx missing=1 records=136 errors=0"
match = re.search(r'CHECK_DONE task=(\w+) missing=(\d+) records=(\d+)', line)
if match:
task_name = match.group(1)
missing = int(match.group(2))
records = int(match.group(3))
if missing > 0:
if 'missing_tasks' not in integrity_stats:
integrity_stats['missing_tasks'] = []
integrity_stats['missing_tasks'].append(f"{task_name}: 缺失{missing}/{records}")
integrity_stats['total_records'] = integrity_stats.get('total_records', 0) + records
integrity_stats['total_missing'] = integrity_stats.get('total_missing', 0) + missing
continue
# 5. 解析数据校验最终结果
# 格式: "结果统计: {'missing': 463, 'errors': 0, 'backfilled': 0}"
if "结果统计:" in line or "结果统计：" in line:
try:
match = re.search(r"\{.+\}", line)
if match:
stats_str = match.group(0).replace("'", '"')
stats = json.loads(stats_str)
integrity_stats['final_missing'] = stats.get('missing', 0)
integrity_stats['final_errors'] = stats.get('errors', 0)
integrity_stats['backfilled'] = stats.get('backfilled', 0)
except Exception:
pass
continue
# 6. 解析错误信息
if "[ERROR]" in line or "错误" in line.lower() or "error" in line.lower():
if "Traceback" not in line and "File " not in line:
errors.append(line.strip()[:100])
# 7. 解析任务完成信息
if "任务执行成功" in line or "ETL运行完成" in line:
task_results.append("✓ " + line.split("]")[-1].strip() if "]" in line else line.strip())
elif "任务执行失败" in line:
task_results.append("✗ " + line.split("]")[-1].strip() if "]" in line else line.strip())
# 构建摘要
if ods_stats:
summary_parts.append("【ODS 抓取】" + ", ".join(ods_stats[:5]))
if len(ods_stats) > 5:
summary_parts[-1] += f" 等{len(ods_stats)}项"
if dwd_stats:
summary_parts.append("【DWD 装载】" + "; ".join(dwd_stats))
if integrity_stats:
total_missing = integrity_stats.get('final_missing', integrity_stats.get('total_missing', 0))
total_records = integrity_stats.get('total_records', 0)
backfilled = integrity_stats.get('backfilled', 0)
int_summary = f"【数据校验】检查 {total_records} 条记录"
if total_missing > 0:
int_summary += f", 发现 {total_missing} 条缺失"
if backfilled > 0:
int_summary += f", 已补全 {backfilled} 条"
else:
int_summary += ", 数据完整"
summary_parts.append(int_summary)
# 显示缺失详情
if integrity_stats.get('missing_tasks'):
missing_detail = integrity_stats['missing_tasks'][:3]
summary_parts.append(" 缺失: " + "; ".join(missing_detail))
if len(integrity_stats['missing_tasks']) > 3:
summary_parts[-1] += f" 等{len(integrity_stats['missing_tasks'])}项"
if errors:
summary_parts.append("【错误】" + "; ".join(errors[:3]))
if task_results:
summary_parts.append("【结果】" + " | ".join(task_results))
if summary_parts:
return "\n".join(summary_parts[-5:]) # 最多返回 5 行
return "\n".join(summary_parts)
# 如果没有解析到任何信息，返回最后几行关键信息
key_lines = []
for line in self._output_lines[-10:]:
if "完成" in line or "成功" in line or "失败" in line:
key_lines.append(line.strip()[:80])
if key_lines:
return "\n".join(key_lines[-3:])
# 如果没有找到关键信息，返回最后一行
return self._output_lines[-1] if self._output_lines else "执行完成"
@property