init: 项目初始提交 - NeoZQYY Monorepo 完整代码
This commit is contained in:
0
gui/.gitkeep
Normal file
0
gui/.gitkeep
Normal file
17
gui/README.md
Normal file
17
gui/README.md
Normal file
@@ -0,0 +1,17 @@
|
||||
# gui/
|
||||
|
||||
## 作用说明
|
||||
|
||||
PySide6 桌面 GUI 应用(过渡期)。提供任务管理、调度配置、数据查看等可视化操作界面,面向门店运营人员使用。
|
||||
|
||||
## 内部结构
|
||||
|
||||
- `main.py` — GUI 入口
|
||||
- `widgets/` — 自定义控件
|
||||
- `views/` — 页面视图
|
||||
- `resources/` — 图标、样式等静态资源
|
||||
|
||||
## Roadmap
|
||||
|
||||
- 随小程序 + 管理后台功能完善,GUI 将逐步退役
|
||||
- 过渡期内保持可用,不再新增大功能
|
||||
5
gui/__init__.py
Normal file
5
gui/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""ETL GUI 客户端模块"""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
__author__ = "ETL Team"
|
||||
46
gui/main.py
Normal file
46
gui/main.py
Normal file
@@ -0,0 +1,46 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""ETL GUI 应用入口"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# 确保项目根目录在 Python 路径中
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from PySide6.QtWidgets import QApplication
|
||||
from PySide6.QtCore import Qt
|
||||
from PySide6.QtGui import QFont
|
||||
|
||||
from gui.main_window import MainWindow
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
# 设置高 DPI 支持
|
||||
QApplication.setHighDpiScaleFactorRoundingPolicy(
|
||||
Qt.HighDpiScaleFactorRoundingPolicy.PassThrough
|
||||
)
|
||||
|
||||
# 创建应用
|
||||
app = QApplication(sys.argv)
|
||||
app.setApplicationName("飞球 ETL 管理系统")
|
||||
app.setApplicationVersion("1.0.0")
|
||||
app.setOrganizationName("Billiards")
|
||||
|
||||
# 设置默认字体
|
||||
font = QFont("Microsoft YaHei", 10)
|
||||
app.setFont(font)
|
||||
|
||||
# 创建主窗口
|
||||
window = MainWindow()
|
||||
window.show()
|
||||
|
||||
# 运行应用
|
||||
sys.exit(app.exec())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
522
gui/main_window.py
Normal file
522
gui/main_window.py
Normal file
@@ -0,0 +1,522 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""主窗口"""
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QMainWindow, QWidget, QVBoxLayout, QHBoxLayout,
|
||||
QStackedWidget, QListWidget, QListWidgetItem,
|
||||
QStatusBar, QLabel, QMessageBox, QSplitter
|
||||
)
|
||||
from PySide6.QtCore import Qt, QSize, Signal
|
||||
from PySide6.QtGui import QIcon, QAction
|
||||
|
||||
from .widgets.task_panel import TaskPanel
|
||||
from .widgets.task_manager import TaskManager
|
||||
from .widgets.env_editor import EnvEditor
|
||||
from .widgets.log_viewer import LogViewer
|
||||
from .widgets.db_viewer import DBViewer
|
||||
from .widgets.status_panel import StatusPanel
|
||||
from .resources import load_stylesheet
|
||||
|
||||
|
||||
class MainWindow(QMainWindow):
|
||||
"""ETL GUI 主窗口"""
|
||||
|
||||
# 信号
|
||||
status_message = Signal(str, int) # message, timeout_ms
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.setWindowTitle("飞球 ETL 管理系统")
|
||||
self.setMinimumSize(1200, 800)
|
||||
self.resize(1400, 900)
|
||||
|
||||
# 应用样式
|
||||
self.setStyleSheet(load_stylesheet())
|
||||
|
||||
# 保存分割器引用
|
||||
self.splitter = None
|
||||
|
||||
# 首次显示标记(必须在 _restore_state 之前初始化,因为 showMaximized 会触发 showEvent)
|
||||
self._first_show = True
|
||||
|
||||
# 初始化 UI
|
||||
self._init_ui()
|
||||
self._init_menu()
|
||||
self._init_status_bar()
|
||||
self._connect_signals()
|
||||
|
||||
# 恢复保存的状态
|
||||
self._restore_state()
|
||||
|
||||
def showEvent(self, event):
|
||||
"""窗口显示事件"""
|
||||
super().showEvent(event)
|
||||
if self._first_show:
|
||||
self._first_show = False
|
||||
# 延迟检查配置,让窗口先显示
|
||||
from PySide6.QtCore import QTimer
|
||||
QTimer.singleShot(100, self._check_config_on_startup)
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
# 中央部件
|
||||
central_widget = QWidget()
|
||||
self.setCentralWidget(central_widget)
|
||||
|
||||
# 主布局
|
||||
main_layout = QHBoxLayout(central_widget)
|
||||
main_layout.setContentsMargins(0, 0, 0, 0)
|
||||
main_layout.setSpacing(0)
|
||||
|
||||
# 创建分割器
|
||||
self.splitter = QSplitter(Qt.Horizontal)
|
||||
main_layout.addWidget(self.splitter)
|
||||
|
||||
# 左侧导航栏
|
||||
nav_widget = self._create_nav_widget()
|
||||
self.splitter.addWidget(nav_widget)
|
||||
|
||||
# 右侧内容区
|
||||
self.content_stack = QStackedWidget()
|
||||
self.splitter.addWidget(self.content_stack)
|
||||
|
||||
# 设置分割比例
|
||||
self.splitter.setSizes([200, 1200])
|
||||
self.splitter.setStretchFactor(0, 0)
|
||||
self.splitter.setStretchFactor(1, 1)
|
||||
|
||||
# 创建各个面板
|
||||
self._create_panels()
|
||||
|
||||
def _create_nav_widget(self) -> QWidget:
|
||||
"""创建导航侧边栏"""
|
||||
nav_widget = QWidget()
|
||||
nav_widget.setMaximumWidth(220)
|
||||
nav_widget.setMinimumWidth(180)
|
||||
|
||||
layout = QVBoxLayout(nav_widget)
|
||||
layout.setContentsMargins(0, 0, 0, 0)
|
||||
layout.setSpacing(0)
|
||||
|
||||
# 标题
|
||||
title_label = QLabel(" ETL 控制台")
|
||||
title_label.setProperty("heading", True)
|
||||
title_label.setFixedHeight(60)
|
||||
title_label.setAlignment(Qt.AlignVCenter)
|
||||
layout.addWidget(title_label)
|
||||
|
||||
# 导航列表
|
||||
self.nav_list = QListWidget()
|
||||
self.nav_list.setObjectName("navList")
|
||||
layout.addWidget(self.nav_list)
|
||||
|
||||
# 添加导航项
|
||||
nav_items = [
|
||||
("任务配置", "配置并执行 ETL 任务"),
|
||||
("任务管理", "管理任务队列和历史记录"),
|
||||
("环境配置", "编辑 .env 配置文件"),
|
||||
("数据库", "查看数据库和执行查询"),
|
||||
("ETL 状态", "查看 ETL 运行状态"),
|
||||
("日志", "查看执行日志"),
|
||||
]
|
||||
|
||||
for name, tooltip in nav_items:
|
||||
item = QListWidgetItem(name)
|
||||
item.setToolTip(tooltip)
|
||||
item.setSizeHint(QSize(0, 44))
|
||||
self.nav_list.addItem(item)
|
||||
|
||||
return nav_widget
|
||||
|
||||
def _create_panels(self):
|
||||
"""创建各个功能面板"""
|
||||
# 任务配置面板
|
||||
self.task_panel = TaskPanel()
|
||||
self.content_stack.addWidget(self.task_panel)
|
||||
|
||||
# 任务管理面板
|
||||
self.task_manager = TaskManager()
|
||||
self.content_stack.addWidget(self.task_manager)
|
||||
|
||||
# 环境配置面板
|
||||
self.env_editor = EnvEditor()
|
||||
self.content_stack.addWidget(self.env_editor)
|
||||
|
||||
# 数据库查看器
|
||||
self.db_viewer = DBViewer()
|
||||
self.content_stack.addWidget(self.db_viewer)
|
||||
|
||||
# ETL 状态面板
|
||||
self.status_panel = StatusPanel()
|
||||
self.content_stack.addWidget(self.status_panel)
|
||||
|
||||
# 日志面板
|
||||
self.log_viewer = LogViewer()
|
||||
self.content_stack.addWidget(self.log_viewer)
|
||||
|
||||
def _init_menu(self):
|
||||
"""初始化菜单栏"""
|
||||
menubar = self.menuBar()
|
||||
|
||||
# 文件菜单
|
||||
file_menu = menubar.addMenu("文件(&F)")
|
||||
|
||||
refresh_action = QAction("刷新配置(&R)", self)
|
||||
refresh_action.setShortcut("Ctrl+R")
|
||||
refresh_action.triggered.connect(self._refresh_config)
|
||||
file_menu.addAction(refresh_action)
|
||||
|
||||
settings_action = QAction("设置(&S)...", self)
|
||||
settings_action.setShortcut("Ctrl+,")
|
||||
settings_action.triggered.connect(self._show_settings)
|
||||
file_menu.addAction(settings_action)
|
||||
|
||||
file_menu.addSeparator()
|
||||
|
||||
exit_action = QAction("退出(&X)", self)
|
||||
exit_action.setShortcut("Ctrl+Q")
|
||||
exit_action.triggered.connect(self.close)
|
||||
file_menu.addAction(exit_action)
|
||||
|
||||
# 视图菜单
|
||||
view_menu = menubar.addMenu("视图(&V)")
|
||||
|
||||
task_config_action = QAction("任务配置(&T)", self)
|
||||
task_config_action.setShortcut("Ctrl+1")
|
||||
task_config_action.triggered.connect(lambda: self._switch_panel(0))
|
||||
view_menu.addAction(task_config_action)
|
||||
|
||||
task_manager_action = QAction("任务管理(&M)", self)
|
||||
task_manager_action.setShortcut("Ctrl+2")
|
||||
task_manager_action.triggered.connect(lambda: self._switch_panel(1))
|
||||
view_menu.addAction(task_manager_action)
|
||||
|
||||
env_action = QAction("环境配置(&E)", self)
|
||||
env_action.setShortcut("Ctrl+3")
|
||||
env_action.triggered.connect(lambda: self._switch_panel(2))
|
||||
view_menu.addAction(env_action)
|
||||
|
||||
db_action = QAction("数据库(&D)", self)
|
||||
db_action.setShortcut("Ctrl+4")
|
||||
db_action.triggered.connect(lambda: self._switch_panel(3))
|
||||
view_menu.addAction(db_action)
|
||||
|
||||
status_action = QAction("ETL 状态(&S)", self)
|
||||
status_action.setShortcut("Ctrl+5")
|
||||
status_action.triggered.connect(lambda: self._switch_panel(4))
|
||||
view_menu.addAction(status_action)
|
||||
|
||||
log_action = QAction("日志(&L)", self)
|
||||
log_action.setShortcut("Ctrl+6")
|
||||
log_action.triggered.connect(lambda: self._switch_panel(5))
|
||||
view_menu.addAction(log_action)
|
||||
|
||||
# 帮助菜单
|
||||
help_menu = menubar.addMenu("帮助(&H)")
|
||||
|
||||
about_action = QAction("关于(&A)", self)
|
||||
about_action.triggered.connect(self._show_about)
|
||||
help_menu.addAction(about_action)
|
||||
|
||||
def _init_status_bar(self):
|
||||
"""初始化状态栏"""
|
||||
self.status_bar = QStatusBar()
|
||||
self.setStatusBar(self.status_bar)
|
||||
|
||||
# 连接状态
|
||||
self.conn_status_label = QLabel("数据库: 未连接")
|
||||
self.conn_status_label.setProperty("status", "warning")
|
||||
self.status_bar.addPermanentWidget(self.conn_status_label)
|
||||
|
||||
# 任务状态
|
||||
self.task_status_label = QLabel("任务: 空闲")
|
||||
self.status_bar.addPermanentWidget(self.task_status_label)
|
||||
|
||||
# 默认消息
|
||||
self.status_bar.showMessage("就绪", 3000)
|
||||
|
||||
def _connect_signals(self):
|
||||
"""连接信号"""
|
||||
# 导航切换
|
||||
self.nav_list.currentRowChanged.connect(self._on_nav_changed)
|
||||
|
||||
# 任务面板信号
|
||||
self.task_panel.task_started.connect(self._on_task_started)
|
||||
self.task_panel.task_finished.connect(self._on_task_finished)
|
||||
self.task_panel.log_message.connect(self.log_viewer.append_log)
|
||||
self.task_panel.add_to_queue.connect(self._on_add_to_queue)
|
||||
self.task_panel.create_schedule.connect(self._on_create_schedule)
|
||||
|
||||
# 任务管理器信号
|
||||
self.task_manager.task_started.connect(self._on_task_started)
|
||||
self.task_manager.task_finished.connect(self._on_task_finished)
|
||||
self.task_manager.log_message.connect(self.log_viewer.append_log)
|
||||
|
||||
# 数据库连接状态
|
||||
self.db_viewer.connection_changed.connect(self._on_db_connection_changed)
|
||||
|
||||
# 状态消息
|
||||
self.status_message.connect(self._show_status_message)
|
||||
|
||||
def _on_nav_changed(self, index: int):
|
||||
"""导航项切换"""
|
||||
self.content_stack.setCurrentIndex(index)
|
||||
|
||||
def _switch_panel(self, index: int):
|
||||
"""切换到指定面板"""
|
||||
self.nav_list.setCurrentRow(index)
|
||||
|
||||
def _refresh_config(self):
|
||||
"""刷新配置"""
|
||||
self.env_editor.load_config()
|
||||
self.task_panel.refresh_tasks()
|
||||
self.status_bar.showMessage("配置已刷新", 3000)
|
||||
|
||||
def _on_task_started(self, task_info: str):
|
||||
"""任务开始时"""
|
||||
self.task_status_label.setText(f"任务: 执行中 - {task_info}")
|
||||
self.task_status_label.setProperty("status", "info")
|
||||
self.task_status_label.style().unpolish(self.task_status_label)
|
||||
self.task_status_label.style().polish(self.task_status_label)
|
||||
|
||||
def _on_task_finished(self, success: bool, message: str):
|
||||
"""任务完成时"""
|
||||
if success:
|
||||
self.task_status_label.setText("任务: 完成")
|
||||
self.task_status_label.setProperty("status", "success")
|
||||
else:
|
||||
self.task_status_label.setText("任务: 失败")
|
||||
self.task_status_label.setProperty("status", "error")
|
||||
self.task_status_label.style().unpolish(self.task_status_label)
|
||||
self.task_status_label.style().polish(self.task_status_label)
|
||||
self.status_bar.showMessage(message, 5000)
|
||||
|
||||
def _on_db_connection_changed(self, connected: bool, message: str):
|
||||
"""数据库连接状态变化"""
|
||||
if connected:
|
||||
self.conn_status_label.setText("数据库: 已连接")
|
||||
self.conn_status_label.setProperty("status", "success")
|
||||
else:
|
||||
self.conn_status_label.setText("数据库: 未连接")
|
||||
self.conn_status_label.setProperty("status", "warning")
|
||||
self.conn_status_label.style().unpolish(self.conn_status_label)
|
||||
self.conn_status_label.style().polish(self.conn_status_label)
|
||||
if message:
|
||||
self.status_bar.showMessage(message, 3000)
|
||||
|
||||
def _show_status_message(self, message: str, timeout: int):
|
||||
"""显示状态栏消息"""
|
||||
self.status_bar.showMessage(message, timeout)
|
||||
|
||||
def _on_add_to_queue(self, config):
|
||||
"""添加任务到队列并自动执行"""
|
||||
task_id = self.task_manager.add_task(config)
|
||||
self.status_bar.showMessage(f"任务已添加到队列 (ID: {task_id})", 3000)
|
||||
|
||||
# 自动切换到任务管理面板
|
||||
self._switch_panel(1)
|
||||
|
||||
# 如果当前没有任务在运行,自动开始执行
|
||||
if not self.task_manager._is_running():
|
||||
from PySide6.QtCore import QTimer
|
||||
# 稍微延迟以确保 UI 更新
|
||||
QTimer.singleShot(100, self.task_manager._run_next)
|
||||
|
||||
def _on_create_schedule(self, name: str, task_codes: list, task_config: dict):
|
||||
"""创建调度任务"""
|
||||
# 打开调度编辑对话框
|
||||
from .widgets.task_manager import ScheduleEditDialog
|
||||
from .models.schedule_model import ScheduledTask, ScheduleConfig
|
||||
import uuid
|
||||
|
||||
# 创建一个预填充的调度任务
|
||||
task = ScheduledTask(
|
||||
id=str(uuid.uuid4())[:8],
|
||||
name=name,
|
||||
task_codes=task_codes,
|
||||
schedule=ScheduleConfig(),
|
||||
task_config=task_config,
|
||||
)
|
||||
|
||||
# 打开编辑对话框
|
||||
dialog = ScheduleEditDialog(task=task, parent=self)
|
||||
if dialog.exec():
|
||||
updated_task = dialog.get_task()
|
||||
if updated_task:
|
||||
self.task_manager.schedule_store.add_task(updated_task)
|
||||
self.task_manager._refresh_schedule_table()
|
||||
self.status_bar.showMessage(f"调度任务已创建: {updated_task.name}", 3000)
|
||||
# 切换到任务管理面板的调度选项卡
|
||||
self._switch_panel(1)
|
||||
|
||||
def _show_settings(self):
|
||||
"""显示设置对话框"""
|
||||
from .widgets.settings_dialog import SettingsDialog
|
||||
dialog = SettingsDialog(self)
|
||||
if dialog.exec():
|
||||
# 重新加载配置
|
||||
self._refresh_config()
|
||||
self.status_bar.showMessage("设置已保存", 3000)
|
||||
|
||||
def _check_config_on_startup(self):
|
||||
"""启动时检查配置"""
|
||||
from .utils.app_settings import app_settings
|
||||
if not app_settings.is_configured():
|
||||
QMessageBox.information(
|
||||
self,
|
||||
"首次配置",
|
||||
"欢迎使用 ETL 管理系统!\n\n"
|
||||
"请先配置 ETL 项目路径,否则无法执行任务。\n\n"
|
||||
"点击 文件 → 设置 进行配置。"
|
||||
)
|
||||
|
||||
def _restore_state(self):
|
||||
"""从设置恢复窗口状态"""
|
||||
from .utils.app_settings import app_settings
|
||||
|
||||
# 恢复窗口位置和大小
|
||||
geometry = app_settings.window_geometry
|
||||
if geometry and len(geometry) == 4:
|
||||
x, y, width, height = geometry
|
||||
# 确保窗口在屏幕范围内
|
||||
from PySide6.QtWidgets import QApplication
|
||||
screen = QApplication.primaryScreen()
|
||||
if screen:
|
||||
screen_rect = screen.availableGeometry()
|
||||
# 检查位置是否在屏幕范围内
|
||||
if (x >= screen_rect.x() and y >= screen_rect.y() and
|
||||
x + width <= screen_rect.x() + screen_rect.width() and
|
||||
y + height <= screen_rect.y() + screen_rect.height()):
|
||||
self.setGeometry(x, y, width, height)
|
||||
|
||||
# 恢复最大化状态
|
||||
if app_settings.window_maximized:
|
||||
self.showMaximized()
|
||||
|
||||
# 恢复当前面板
|
||||
saved_panel = app_settings.current_panel
|
||||
if 0 <= saved_panel < self.nav_list.count():
|
||||
self.nav_list.setCurrentRow(saved_panel)
|
||||
else:
|
||||
self.nav_list.setCurrentRow(0)
|
||||
|
||||
# 恢复分割器大小
|
||||
splitter_sizes = app_settings.splitter_sizes
|
||||
if splitter_sizes and self.splitter:
|
||||
self.splitter.setSizes(splitter_sizes)
|
||||
|
||||
# 恢复任务管理器状态
|
||||
if hasattr(self, 'task_manager'):
|
||||
# 恢复选项卡
|
||||
saved_tab = app_settings.task_manager_tab
|
||||
if hasattr(self.task_manager, 'tab_widget'):
|
||||
if 0 <= saved_tab < self.task_manager.tab_widget.count():
|
||||
self.task_manager.tab_widget.setCurrentIndex(saved_tab)
|
||||
|
||||
# 恢复自动执行状态
|
||||
if app_settings.auto_run_enabled:
|
||||
if hasattr(self.task_manager, 'auto_run_btn'):
|
||||
self.task_manager.auto_run_btn.setChecked(True)
|
||||
|
||||
# 恢复调度器状态
|
||||
if app_settings.scheduler_enabled:
|
||||
if hasattr(self.task_manager, 'scheduler_btn'):
|
||||
self.task_manager.scheduler_btn.setChecked(True)
|
||||
|
||||
# 恢复任务面板状态
|
||||
if hasattr(self, 'task_panel'):
|
||||
if app_settings.advanced_expanded:
|
||||
if hasattr(self.task_panel, 'advanced_section'):
|
||||
self.task_panel.advanced_section.setExpanded(True)
|
||||
|
||||
def _save_state(self):
|
||||
"""保存窗口状态到设置"""
|
||||
from .utils.app_settings import app_settings
|
||||
|
||||
# 保存窗口位置和大小(仅在非最大化时保存)
|
||||
if not self.isMaximized():
|
||||
geo = self.geometry()
|
||||
app_settings.window_geometry = [geo.x(), geo.y(), geo.width(), geo.height()]
|
||||
|
||||
# 保存最大化状态
|
||||
app_settings.window_maximized = self.isMaximized()
|
||||
|
||||
# 保存当前面板
|
||||
app_settings.current_panel = self.nav_list.currentRow()
|
||||
|
||||
# 保存分割器大小
|
||||
if self.splitter:
|
||||
app_settings.splitter_sizes = self.splitter.sizes()
|
||||
|
||||
# 保存任务管理器状态
|
||||
if hasattr(self, 'task_manager'):
|
||||
# 保存选项卡
|
||||
if hasattr(self.task_manager, 'tab_widget'):
|
||||
app_settings.task_manager_tab = self.task_manager.tab_widget.currentIndex()
|
||||
|
||||
# 保存自动执行状态
|
||||
if hasattr(self.task_manager, 'auto_run_btn'):
|
||||
app_settings.auto_run_enabled = self.task_manager.auto_run_btn.isChecked()
|
||||
|
||||
# 保存调度器状态
|
||||
if hasattr(self.task_manager, 'scheduler_btn'):
|
||||
app_settings.scheduler_enabled = self.task_manager.scheduler_btn.isChecked()
|
||||
|
||||
# 保存任务面板状态
|
||||
if hasattr(self, 'task_panel'):
|
||||
if hasattr(self.task_panel, 'advanced_section'):
|
||||
app_settings.advanced_expanded = self.task_panel.advanced_section.isExpanded()
|
||||
|
||||
def _show_about(self):
|
||||
"""显示关于对话框"""
|
||||
QMessageBox.about(
|
||||
self,
|
||||
"关于 飞球 ETL 管理系统",
|
||||
"<h3>飞球 ETL 管理系统</h3>"
|
||||
"<p>版本: 1.0.0</p>"
|
||||
"<p>一个用于管理台球场门店数据 ETL 的图形化工具。</p>"
|
||||
"<p>功能包括:</p>"
|
||||
"<ul>"
|
||||
"<li>任务配置与执行</li>"
|
||||
"<li>环境变量管理</li>"
|
||||
"<li>数据库查询</li>"
|
||||
"<li>ETL 状态监控</li>"
|
||||
"</ul>"
|
||||
)
|
||||
|
||||
def closeEvent(self, event):
|
||||
"""关闭事件"""
|
||||
# 检查是否有正在运行的任务
|
||||
if hasattr(self, 'task_panel') and self.task_panel.is_running():
|
||||
reply = QMessageBox.question(
|
||||
self,
|
||||
"确认退出",
|
||||
"当前有任务正在执行,确定要退出吗?",
|
||||
QMessageBox.Yes | QMessageBox.No,
|
||||
QMessageBox.No
|
||||
)
|
||||
if reply == QMessageBox.No:
|
||||
event.ignore()
|
||||
return
|
||||
|
||||
# 检查任务管理器是否有正在运行的任务
|
||||
if hasattr(self, 'task_manager') and self.task_manager._is_running():
|
||||
reply = QMessageBox.question(
|
||||
self,
|
||||
"确认退出",
|
||||
"任务管理器中有任务正在执行,确定要退出吗?",
|
||||
QMessageBox.Yes | QMessageBox.No,
|
||||
QMessageBox.No
|
||||
)
|
||||
if reply == QMessageBox.No:
|
||||
event.ignore()
|
||||
return
|
||||
|
||||
# 保存窗口状态
|
||||
self._save_state()
|
||||
|
||||
# 关闭数据库连接
|
||||
if hasattr(self, 'db_viewer'):
|
||||
self.db_viewer.close_connection()
|
||||
|
||||
event.accept()
|
||||
35
gui/models/__init__.py
Normal file
35
gui/models/__init__.py
Normal file
@@ -0,0 +1,35 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""数据模型模块"""
|
||||
|
||||
from .task_model import TaskItem, TaskStatus, TaskHistory, TaskConfig, QueuedTask
|
||||
from .schedule_model import (
|
||||
ScheduledTask, ScheduleConfig, ScheduleType, IntervalUnit, ScheduleStore
|
||||
)
|
||||
from .task_registry import (
|
||||
TaskRegistry, TaskDefinition, BusinessDomain, DOMAIN_LABELS,
|
||||
task_registry, get_ods_task_codes, get_fact_ods_task_codes,
|
||||
get_dimension_ods_task_codes, get_all_task_tuples
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"TaskItem",
|
||||
"TaskStatus",
|
||||
"TaskHistory",
|
||||
"TaskConfig",
|
||||
"QueuedTask",
|
||||
"ScheduledTask",
|
||||
"ScheduleConfig",
|
||||
"ScheduleType",
|
||||
"IntervalUnit",
|
||||
"ScheduleStore",
|
||||
# 任务注册表
|
||||
"TaskRegistry",
|
||||
"TaskDefinition",
|
||||
"BusinessDomain",
|
||||
"DOMAIN_LABELS",
|
||||
"task_registry",
|
||||
"get_ods_task_codes",
|
||||
"get_fact_ods_task_codes",
|
||||
"get_dimension_ods_task_codes",
|
||||
"get_all_task_tuples",
|
||||
]
|
||||
391
gui/models/schedule_model.py
Normal file
391
gui/models/schedule_model.py
Normal file
@@ -0,0 +1,391 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""调度任务数据模型"""
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime, timedelta
|
||||
from enum import Enum
|
||||
from typing import Optional, List, Dict, Any
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ScheduleType(Enum):
|
||||
"""调度类型"""
|
||||
ONCE = "once" # 一次性
|
||||
INTERVAL = "interval" # 固定间隔
|
||||
DAILY = "daily" # 每天
|
||||
WEEKLY = "weekly" # 每周
|
||||
CRON = "cron" # Cron 表达式
|
||||
|
||||
|
||||
class IntervalUnit(Enum):
|
||||
"""间隔单位"""
|
||||
MINUTES = "minutes"
|
||||
HOURS = "hours"
|
||||
DAYS = "days"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScheduleConfig:
|
||||
"""调度配置"""
|
||||
schedule_type: ScheduleType = ScheduleType.ONCE
|
||||
|
||||
# 间隔调度
|
||||
interval_value: int = 1
|
||||
interval_unit: IntervalUnit = IntervalUnit.HOURS
|
||||
|
||||
# 每日调度
|
||||
daily_time: str = "04:00" # HH:MM
|
||||
|
||||
# 每周调度
|
||||
weekly_days: List[int] = field(default_factory=lambda: [1]) # 1-7, 1=周一
|
||||
weekly_time: str = "04:00"
|
||||
|
||||
# Cron 表达式
|
||||
cron_expression: str = "0 4 * * *"
|
||||
|
||||
# 通用设置
|
||||
enabled: bool = True
|
||||
start_date: Optional[str] = None # YYYY-MM-DD
|
||||
end_date: Optional[str] = None # YYYY-MM-DD
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""转换为字典"""
|
||||
return {
|
||||
"schedule_type": self.schedule_type.value,
|
||||
"interval_value": self.interval_value,
|
||||
"interval_unit": self.interval_unit.value,
|
||||
"daily_time": self.daily_time,
|
||||
"weekly_days": self.weekly_days,
|
||||
"weekly_time": self.weekly_time,
|
||||
"cron_expression": self.cron_expression,
|
||||
"enabled": self.enabled,
|
||||
"start_date": self.start_date,
|
||||
"end_date": self.end_date,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "ScheduleConfig":
|
||||
"""从字典创建"""
|
||||
return cls(
|
||||
schedule_type=ScheduleType(data.get("schedule_type", "once")),
|
||||
interval_value=data.get("interval_value", 1),
|
||||
interval_unit=IntervalUnit(data.get("interval_unit", "hours")),
|
||||
daily_time=data.get("daily_time", "04:00"),
|
||||
weekly_days=data.get("weekly_days", [1]),
|
||||
weekly_time=data.get("weekly_time", "04:00"),
|
||||
cron_expression=data.get("cron_expression", "0 4 * * *"),
|
||||
enabled=data.get("enabled", True),
|
||||
start_date=data.get("start_date"),
|
||||
end_date=data.get("end_date"),
|
||||
)
|
||||
|
||||
def get_description(self) -> str:
|
||||
"""获取调度描述"""
|
||||
if self.schedule_type == ScheduleType.ONCE:
|
||||
return "一次性执行"
|
||||
elif self.schedule_type == ScheduleType.INTERVAL:
|
||||
unit_names = {"minutes": "分钟", "hours": "小时", "days": "天"}
|
||||
return f"每 {self.interval_value} {unit_names[self.interval_unit.value]}"
|
||||
elif self.schedule_type == ScheduleType.DAILY:
|
||||
return f"每天 {self.daily_time}"
|
||||
elif self.schedule_type == ScheduleType.WEEKLY:
|
||||
day_names = {1: "一", 2: "二", 3: "三", 4: "四", 5: "五", 6: "六", 7: "日"}
|
||||
days = "、".join(f"周{day_names[d]}" for d in sorted(self.weekly_days))
|
||||
return f"每周 {days} {self.weekly_time}"
|
||||
elif self.schedule_type == ScheduleType.CRON:
|
||||
return f"Cron: {self.cron_expression}"
|
||||
return "未知"
|
||||
|
||||
# 首次执行延迟秒数
|
||||
FIRST_RUN_DELAY_SECONDS = 60
|
||||
|
||||
def get_next_run_time(self, last_run: Optional[datetime] = None) -> Optional[datetime]:
|
||||
"""计算下次运行时间
|
||||
|
||||
注意:首次执行(last_run 为 None)时会延迟 60 秒,避免创建后立即执行
|
||||
"""
|
||||
now = datetime.now()
|
||||
|
||||
# 检查日期范围
|
||||
if self.start_date:
|
||||
start = datetime.strptime(self.start_date, "%Y-%m-%d")
|
||||
if now < start:
|
||||
now = start
|
||||
|
||||
if self.end_date:
|
||||
end = datetime.strptime(self.end_date, "%Y-%m-%d") + timedelta(days=1)
|
||||
if now >= end:
|
||||
return None
|
||||
|
||||
# 首次执行延迟 60 秒
|
||||
first_run_time = now + timedelta(seconds=self.FIRST_RUN_DELAY_SECONDS)
|
||||
|
||||
if self.schedule_type == ScheduleType.ONCE:
|
||||
return None if last_run else first_run_time
|
||||
|
||||
elif self.schedule_type == ScheduleType.INTERVAL:
|
||||
if not last_run:
|
||||
return first_run_time
|
||||
if self.interval_unit == IntervalUnit.MINUTES:
|
||||
delta = timedelta(minutes=self.interval_value)
|
||||
elif self.interval_unit == IntervalUnit.HOURS:
|
||||
delta = timedelta(hours=self.interval_value)
|
||||
else:
|
||||
delta = timedelta(days=self.interval_value)
|
||||
return last_run + delta
|
||||
|
||||
elif self.schedule_type == ScheduleType.DAILY:
|
||||
hour, minute = map(int, self.daily_time.split(":"))
|
||||
next_run = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||||
if next_run <= now:
|
||||
next_run += timedelta(days=1)
|
||||
return next_run
|
||||
|
||||
elif self.schedule_type == ScheduleType.WEEKLY:
|
||||
hour, minute = map(int, self.weekly_time.split(":"))
|
||||
# 找到下一个匹配的日期
|
||||
for i in range(8):
|
||||
check_date = now + timedelta(days=i)
|
||||
weekday = check_date.isoweekday() # 1-7
|
||||
if weekday in self.weekly_days:
|
||||
next_run = check_date.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||||
if next_run > now:
|
||||
return next_run
|
||||
return None
|
||||
|
||||
elif self.schedule_type == ScheduleType.CRON:
|
||||
# 简化版 Cron 解析(只支持基本格式)
|
||||
try:
|
||||
return self._parse_simple_cron(now)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def _parse_simple_cron(self, now: datetime) -> Optional[datetime]:
|
||||
"""简化版 Cron 解析"""
|
||||
parts = self.cron_expression.split()
|
||||
if len(parts) != 5:
|
||||
return None
|
||||
|
||||
minute, hour, day, month, weekday = parts
|
||||
|
||||
# 只处理简单情况
|
||||
if minute.isdigit() and hour.isdigit():
|
||||
next_run = now.replace(
|
||||
hour=int(hour),
|
||||
minute=int(minute),
|
||||
second=0,
|
||||
microsecond=0
|
||||
)
|
||||
if next_run <= now:
|
||||
next_run += timedelta(days=1)
|
||||
return next_run
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScheduleExecutionRecord:
|
||||
"""调度执行记录"""
|
||||
task_id: str # 关联的 QueuedTask ID
|
||||
executed_at: datetime # 执行时间
|
||||
status: str = "" # 状态:success, failed, pending
|
||||
exit_code: Optional[int] = None # 退出码
|
||||
duration_seconds: float = 0.0 # 耗时(秒)
|
||||
summary: str = "" # 执行摘要
|
||||
output: str = "" # 完整执行日志
|
||||
error: str = "" # 错误信息
|
||||
|
||||
# 日志最大长度限制(字符数)
|
||||
MAX_OUTPUT_LENGTH: int = 100000 # 100KB
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"task_id": self.task_id,
|
||||
"executed_at": self.executed_at.isoformat(),
|
||||
"status": self.status,
|
||||
"exit_code": self.exit_code,
|
||||
"duration_seconds": self.duration_seconds,
|
||||
"summary": self.summary,
|
||||
"output": self.output[:self.MAX_OUTPUT_LENGTH] if self.output else "",
|
||||
"error": self.error[:5000] if self.error else "",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "ScheduleExecutionRecord":
|
||||
return cls(
|
||||
task_id=data.get("task_id", ""),
|
||||
executed_at=datetime.fromisoformat(data["executed_at"]) if data.get("executed_at") else datetime.now(),
|
||||
status=data.get("status", ""),
|
||||
exit_code=data.get("exit_code"),
|
||||
duration_seconds=data.get("duration_seconds", 0.0),
|
||||
summary=data.get("summary", ""),
|
||||
output=data.get("output", ""),
|
||||
error=data.get("error", ""),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScheduledTask:
|
||||
"""调度任务"""
|
||||
id: str
|
||||
name: str
|
||||
task_codes: List[str]
|
||||
schedule: ScheduleConfig
|
||||
task_config: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
# 运行状态
|
||||
enabled: bool = True
|
||||
last_run: Optional[datetime] = None
|
||||
next_run: Optional[datetime] = None
|
||||
run_count: int = 0
|
||||
last_status: str = ""
|
||||
|
||||
# 执行历史(最近 N 次执行记录)
|
||||
execution_history: List[ScheduleExecutionRecord] = field(default_factory=list)
|
||||
MAX_HISTORY_SIZE: int = field(default=50, repr=False) # 保留最近50次执行记录
|
||||
|
||||
created_at: datetime = field(default_factory=datetime.now)
|
||||
updated_at: datetime = field(default_factory=datetime.now)
|
||||
|
||||
def add_execution_record(self, record: ScheduleExecutionRecord):
|
||||
"""添加执行记录"""
|
||||
self.execution_history.insert(0, record)
|
||||
# 限制历史记录数量
|
||||
if len(self.execution_history) > self.MAX_HISTORY_SIZE:
|
||||
self.execution_history = self.execution_history[:self.MAX_HISTORY_SIZE]
|
||||
|
||||
def update_execution_record(self, task_id: str, status: str, exit_code: int, duration: float,
|
||||
summary: str, output: str = "", error: str = ""):
|
||||
"""更新执行记录状态"""
|
||||
for record in self.execution_history:
|
||||
if record.task_id == task_id:
|
||||
record.status = status
|
||||
record.exit_code = exit_code
|
||||
record.duration_seconds = duration
|
||||
record.summary = summary
|
||||
record.output = output
|
||||
record.error = error
|
||||
break
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""转换为字典"""
|
||||
return {
|
||||
"id": self.id,
|
||||
"name": self.name,
|
||||
"task_codes": self.task_codes,
|
||||
"schedule": self.schedule.to_dict(),
|
||||
"task_config": self.task_config,
|
||||
"enabled": self.enabled,
|
||||
"last_run": self.last_run.isoformat() if self.last_run else None,
|
||||
"next_run": self.next_run.isoformat() if self.next_run else None,
|
||||
"run_count": self.run_count,
|
||||
"last_status": self.last_status,
|
||||
"execution_history": [r.to_dict() for r in self.execution_history],
|
||||
"created_at": self.created_at.isoformat(),
|
||||
"updated_at": self.updated_at.isoformat(),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "ScheduledTask":
|
||||
"""从字典创建"""
|
||||
history_data = data.get("execution_history", [])
|
||||
execution_history = [ScheduleExecutionRecord.from_dict(r) for r in history_data]
|
||||
|
||||
return cls(
|
||||
id=data["id"],
|
||||
name=data["name"],
|
||||
task_codes=data["task_codes"],
|
||||
schedule=ScheduleConfig.from_dict(data.get("schedule", {})),
|
||||
task_config=data.get("task_config", {}),
|
||||
enabled=data.get("enabled", True),
|
||||
last_run=datetime.fromisoformat(data["last_run"]) if data.get("last_run") else None,
|
||||
next_run=datetime.fromisoformat(data["next_run"]) if data.get("next_run") else None,
|
||||
run_count=data.get("run_count", 0),
|
||||
last_status=data.get("last_status", ""),
|
||||
execution_history=execution_history,
|
||||
created_at=datetime.fromisoformat(data["created_at"]) if data.get("created_at") else datetime.now(),
|
||||
updated_at=datetime.fromisoformat(data["updated_at"]) if data.get("updated_at") else datetime.now(),
|
||||
)
|
||||
|
||||
def update_next_run(self):
|
||||
"""更新下次运行时间"""
|
||||
if self.enabled and self.schedule.enabled:
|
||||
self.next_run = self.schedule.get_next_run_time(self.last_run)
|
||||
else:
|
||||
self.next_run = None
|
||||
self.updated_at = datetime.now()
|
||||
|
||||
|
||||
class ScheduleStore:
|
||||
"""调度任务存储"""
|
||||
|
||||
def __init__(self, storage_path: Optional[Path] = None):
|
||||
if storage_path is None:
|
||||
storage_path = Path(__file__).resolve().parents[2] / "config" / "scheduled_tasks.json"
|
||||
self.storage_path = storage_path
|
||||
self.tasks: Dict[str, ScheduledTask] = {}
|
||||
self.load()
|
||||
|
||||
def load(self):
|
||||
"""加载任务"""
|
||||
if self.storage_path.exists():
|
||||
try:
|
||||
data = json.loads(self.storage_path.read_text(encoding="utf-8"))
|
||||
self.tasks = {
|
||||
task_id: ScheduledTask.from_dict(task_data)
|
||||
for task_id, task_data in data.get("tasks", {}).items()
|
||||
}
|
||||
except Exception:
|
||||
self.tasks = {}
|
||||
|
||||
def save(self):
|
||||
"""保存任务"""
|
||||
data = {
|
||||
"tasks": {
|
||||
task_id: task.to_dict()
|
||||
for task_id, task in self.tasks.items()
|
||||
}
|
||||
}
|
||||
self.storage_path.write_text(
|
||||
json.dumps(data, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
def add_task(self, task: ScheduledTask):
|
||||
"""添加任务"""
|
||||
task.update_next_run()
|
||||
self.tasks[task.id] = task
|
||||
self.save()
|
||||
|
||||
def remove_task(self, task_id: str):
|
||||
"""移除任务"""
|
||||
if task_id in self.tasks:
|
||||
del self.tasks[task_id]
|
||||
self.save()
|
||||
|
||||
def update_task(self, task: ScheduledTask):
|
||||
"""更新任务"""
|
||||
task.update_next_run()
|
||||
task.updated_at = datetime.now()
|
||||
self.tasks[task.id] = task
|
||||
self.save()
|
||||
|
||||
def get_task(self, task_id: str) -> Optional[ScheduledTask]:
|
||||
"""获取任务"""
|
||||
return self.tasks.get(task_id)
|
||||
|
||||
def get_all_tasks(self) -> List[ScheduledTask]:
|
||||
"""获取所有任务"""
|
||||
return list(self.tasks.values())
|
||||
|
||||
def get_due_tasks(self) -> List[ScheduledTask]:
|
||||
"""获取到期需要执行的任务"""
|
||||
now = datetime.now()
|
||||
due_tasks = []
|
||||
for task in self.tasks.values():
|
||||
if task.enabled and task.next_run and task.next_run <= now:
|
||||
due_tasks.append(task)
|
||||
return due_tasks
|
||||
208
gui/models/task_model.py
Normal file
208
gui/models/task_model.py
Normal file
@@ -0,0 +1,208 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# AI_CHANGELOG [2026-02-13] 移除 DWS_RECALL_INDEX/DWS_INTIMACY_INDEX 任务分类映射
|
||||
"""任务数据模型"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
|
||||
class TaskStatus(Enum):
|
||||
"""任务状态枚举"""
|
||||
PENDING = "pending" # 待执行
|
||||
RUNNING = "running" # 执行中
|
||||
SUCCESS = "success" # 成功
|
||||
FAILED = "failed" # 失败
|
||||
CANCELLED = "cancelled" # 已取消
|
||||
|
||||
|
||||
class TaskCategory(Enum):
|
||||
"""任务分类"""
|
||||
ODS = "ODS" # ODS 数据抓取任务
|
||||
DWD = "DWD" # DWD 装载任务
|
||||
DWS = "DWS" # DWS 汇总任务
|
||||
SCHEMA = "Schema" # Schema 初始化任务
|
||||
QUALITY = "Quality" # 质量检查任务
|
||||
OTHER = "Other" # 其他任务
|
||||
|
||||
|
||||
# 任务分类映射
|
||||
TASK_CATEGORIES: Dict[str, TaskCategory] = {
|
||||
# ODS 任务
|
||||
"ODS_PAYMENT": TaskCategory.ODS,
|
||||
"ODS_MEMBER": TaskCategory.ODS,
|
||||
"ODS_MEMBER_CARD": TaskCategory.ODS,
|
||||
"ODS_MEMBER_BALANCE": TaskCategory.ODS,
|
||||
"ODS_SETTLEMENT_RECORDS": TaskCategory.ODS,
|
||||
"ODS_TABLE_USE": TaskCategory.ODS,
|
||||
"ODS_ASSISTANT_ACCOUNT": TaskCategory.ODS,
|
||||
"ODS_ASSISTANT_LEDGER": TaskCategory.ODS,
|
||||
"ODS_ASSISTANT_ABOLISH": TaskCategory.ODS,
|
||||
"ODS_REFUND": TaskCategory.ODS,
|
||||
"ODS_PLATFORM_COUPON": TaskCategory.ODS,
|
||||
"ODS_RECHARGE_SETTLE": TaskCategory.ODS,
|
||||
"ODS_GROUP_PACKAGE": TaskCategory.ODS,
|
||||
"ODS_GROUP_BUY_REDEMPTION": TaskCategory.ODS,
|
||||
"ODS_INVENTORY_STOCK": TaskCategory.ODS,
|
||||
"ODS_INVENTORY_CHANGE": TaskCategory.ODS,
|
||||
"ODS_TABLES": TaskCategory.ODS,
|
||||
"ODS_GOODS_CATEGORY": TaskCategory.ODS,
|
||||
"ODS_STORE_GOODS": TaskCategory.ODS,
|
||||
"ODS_STORE_GOODS_SALES": TaskCategory.ODS,
|
||||
"ODS_TABLE_FEE_DISCOUNT": TaskCategory.ODS,
|
||||
"ODS_TENANT_GOODS": TaskCategory.ODS,
|
||||
"ODS_SETTLEMENT_TICKET": TaskCategory.ODS,
|
||||
# DWD 任务
|
||||
"DWD_LOAD_FROM_ODS": TaskCategory.DWD,
|
||||
"DWD_QUALITY_CHECK": TaskCategory.QUALITY,
|
||||
"PAYMENTS_DWD": TaskCategory.DWD,
|
||||
"MEMBERS_DWD": TaskCategory.DWD,
|
||||
"TICKET_DWD": TaskCategory.DWD,
|
||||
# DWS 任务
|
||||
"INIT_DWS_SCHEMA": TaskCategory.SCHEMA,
|
||||
"SEED_DWS_CONFIG": TaskCategory.SCHEMA,
|
||||
"DWS_BUILD_ORDER_SUMMARY": TaskCategory.DWS,
|
||||
"DWS_WINBACK_INDEX": TaskCategory.DWS,
|
||||
"DWS_NEWCONV_INDEX": TaskCategory.DWS,
|
||||
"DWS_RELATION_INDEX": TaskCategory.DWS,
|
||||
"DWS_ML_MANUAL_IMPORT": TaskCategory.DWS,
|
||||
"DWS_ASSISTANT_DAILY": TaskCategory.DWS,
|
||||
"DWS_ASSISTANT_MONTHLY": TaskCategory.DWS,
|
||||
"DWS_ASSISTANT_CUSTOMER": TaskCategory.DWS,
|
||||
"DWS_ASSISTANT_SALARY": TaskCategory.DWS,
|
||||
"DWS_ASSISTANT_FINANCE": TaskCategory.DWS,
|
||||
"DWS_MEMBER_CONSUMPTION": TaskCategory.DWS,
|
||||
"DWS_MEMBER_VISIT": TaskCategory.DWS,
|
||||
"DWS_FINANCE_DAILY": TaskCategory.DWS,
|
||||
"DWS_FINANCE_RECHARGE": TaskCategory.DWS,
|
||||
"DWS_FINANCE_INCOME_STRUCTURE": TaskCategory.DWS,
|
||||
"DWS_FINANCE_DISCOUNT_DETAIL": TaskCategory.DWS,
|
||||
"DWS_RETENTION_CLEANUP": TaskCategory.DWS,
|
||||
"DWS_MV_REFRESH_FINANCE_DAILY": TaskCategory.DWS,
|
||||
"DWS_MV_REFRESH_ASSISTANT_DAILY": TaskCategory.DWS,
|
||||
# Schema 任务
|
||||
"INIT_ODS_SCHEMA": TaskCategory.SCHEMA,
|
||||
"INIT_DWD_SCHEMA": TaskCategory.SCHEMA,
|
||||
# 其他任务
|
||||
"MANUAL_INGEST": TaskCategory.OTHER,
|
||||
"CHECK_CUTOFF": TaskCategory.OTHER,
|
||||
"DATA_INTEGRITY_CHECK": TaskCategory.QUALITY,
|
||||
"ODS_JSON_ARCHIVE": TaskCategory.OTHER,
|
||||
# 旧版任务(兼容)
|
||||
"PRODUCTS": TaskCategory.ODS,
|
||||
"TABLES": TaskCategory.ODS,
|
||||
"MEMBERS": TaskCategory.ODS,
|
||||
"ASSISTANTS": TaskCategory.ODS,
|
||||
"PACKAGES_DEF": TaskCategory.ODS,
|
||||
"ORDERS": TaskCategory.ODS,
|
||||
"PAYMENTS": TaskCategory.ODS,
|
||||
"REFUNDS": TaskCategory.ODS,
|
||||
"COUPON_USAGE": TaskCategory.ODS,
|
||||
"INVENTORY_CHANGE": TaskCategory.ODS,
|
||||
"TOPUPS": TaskCategory.ODS,
|
||||
"TABLE_DISCOUNT": TaskCategory.ODS,
|
||||
"ASSISTANT_ABOLISH": TaskCategory.ODS,
|
||||
"LEDGER": TaskCategory.ODS,
|
||||
}
|
||||
|
||||
|
||||
def get_task_category(task_code: str) -> TaskCategory:
|
||||
"""获取任务分类"""
|
||||
return TASK_CATEGORIES.get(task_code.upper(), TaskCategory.OTHER)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskItem:
|
||||
"""任务项"""
|
||||
task_code: str
|
||||
name: str = ""
|
||||
description: str = ""
|
||||
category: TaskCategory = TaskCategory.OTHER
|
||||
enabled: bool = True
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.name:
|
||||
self.name = self.task_code
|
||||
if not self.category or self.category == TaskCategory.OTHER:
|
||||
self.category = get_task_category(self.task_code)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskConfig:
|
||||
"""任务执行配置"""
|
||||
tasks: List[str] = field(default_factory=list)
|
||||
pipeline_flow: str = "FULL" # FULL, FETCH_ONLY, INGEST_ONLY
|
||||
dry_run: bool = False
|
||||
window_start: Optional[str] = None
|
||||
window_end: Optional[str] = None
|
||||
window_split: Optional[str] = None # none, day, week, month
|
||||
window_split_days: Optional[int] = None # 按天切分的天数(1/10/30)
|
||||
window_compensation: int = 0 # 补偿小时数
|
||||
ingest_source: Optional[str] = None
|
||||
store_id: Optional[int] = None
|
||||
pg_dsn: Optional[str] = None
|
||||
api_token: Optional[str] = None
|
||||
extra_args: Dict[str, Any] = field(default_factory=dict)
|
||||
env_vars: Dict[str, str] = field(default_factory=dict) # 额外环境变量
|
||||
|
||||
# 新增:管道配置
|
||||
pipeline: str = "api_ods_dwd" # 管道类型
|
||||
processing_mode: str = "increment_only" # increment_only / verify_only / increment_verify
|
||||
fetch_before_verify: bool = False # 校验前从 API 获取数据(仅 verify_only 模式有效)
|
||||
window_mode: str = "lookback" # lookback / custom
|
||||
lookback_hours: int = 24 # 回溯小时数
|
||||
overlap_seconds: int = 600 # 冗余秒数
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskHistory:
|
||||
"""任务执行历史"""
|
||||
id: str
|
||||
task_codes: List[str]
|
||||
status: TaskStatus
|
||||
start_time: datetime
|
||||
end_time: Optional[datetime] = None
|
||||
exit_code: Optional[int] = None
|
||||
command: str = ""
|
||||
output_log: str = ""
|
||||
error_message: str = ""
|
||||
summary: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def duration_seconds(self) -> Optional[float]:
|
||||
"""执行时长(秒)"""
|
||||
if self.end_time and self.start_time:
|
||||
return (self.end_time - self.start_time).total_seconds()
|
||||
return None
|
||||
|
||||
@property
|
||||
def duration_str(self) -> str:
|
||||
"""格式化的执行时长"""
|
||||
secs = self.duration_seconds
|
||||
if secs is None:
|
||||
return "-"
|
||||
if secs < 60:
|
||||
return f"{secs:.1f}秒"
|
||||
elif secs < 3600:
|
||||
mins = int(secs // 60)
|
||||
secs = secs % 60
|
||||
return f"{mins}分{secs:.0f}秒"
|
||||
else:
|
||||
hours = int(secs // 3600)
|
||||
mins = int((secs % 3600) // 60)
|
||||
return f"{hours}时{mins}分"
|
||||
|
||||
|
||||
@dataclass
|
||||
class QueuedTask:
|
||||
"""队列中的任务"""
|
||||
id: str
|
||||
config: TaskConfig
|
||||
status: TaskStatus = TaskStatus.PENDING
|
||||
created_at: datetime = field(default_factory=datetime.now)
|
||||
started_at: Optional[datetime] = None
|
||||
finished_at: Optional[datetime] = None
|
||||
output: str = ""
|
||||
error: str = ""
|
||||
exit_code: Optional[int] = None
|
||||
669
gui/models/task_registry.py
Normal file
669
gui/models/task_registry.py
Normal file
@@ -0,0 +1,669 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# AI_CHANGELOG [2026-02-13] 移除 DWS_RECALL_INDEX/DWS_INTIMACY_INDEX 任务定义
|
||||
"""任务注册表:定义所有可用任务及其业务域分组。
|
||||
|
||||
从后端 ods_tasks 动态获取任务定义,并按业务域分组,供 UI 使用。
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
# 尝试从后端导入 ODS 任务定义
|
||||
try:
|
||||
from tasks.ods.ods_tasks import ENABLED_ODS_CODES, ODS_TASK_SPECS
|
||||
_HAS_BACKEND = True
|
||||
except ImportError:
|
||||
_HAS_BACKEND = False
|
||||
ENABLED_ODS_CODES = set()
|
||||
ODS_TASK_SPECS = ()
|
||||
|
||||
|
||||
class BusinessDomain(Enum):
|
||||
"""业务域枚举"""
|
||||
MEMBER = "member" # 会员
|
||||
SETTLEMENT = "settlement" # 结算/支付
|
||||
ASSISTANT = "assistant" # 助教
|
||||
GOODS = "goods" # 商品/销售
|
||||
TABLE = "table" # 台桌
|
||||
PROMOTION = "promotion" # 团购/优惠券
|
||||
INVENTORY = "inventory" # 库存
|
||||
SCHEMA = "schema" # Schema 初始化
|
||||
DWD = "dwd" # DWD 装载
|
||||
DWS = "dws" # DWS 汇总
|
||||
INDEX = "index" # 指数计算
|
||||
QUALITY = "quality" # 质量检查
|
||||
OTHER = "other" # 其他
|
||||
|
||||
|
||||
# 业务域显示名称
|
||||
DOMAIN_LABELS: Dict[BusinessDomain, str] = {
|
||||
BusinessDomain.MEMBER: "会员",
|
||||
BusinessDomain.SETTLEMENT: "结算/支付",
|
||||
BusinessDomain.ASSISTANT: "助教",
|
||||
BusinessDomain.GOODS: "商品/销售",
|
||||
BusinessDomain.TABLE: "台桌",
|
||||
BusinessDomain.PROMOTION: "团购/优惠券",
|
||||
BusinessDomain.INVENTORY: "库存",
|
||||
BusinessDomain.SCHEMA: "Schema 初始化",
|
||||
BusinessDomain.DWD: "DWD 装载",
|
||||
BusinessDomain.DWS: "DWS 汇总",
|
||||
BusinessDomain.INDEX: "指数计算",
|
||||
BusinessDomain.QUALITY: "质量检查",
|
||||
BusinessDomain.OTHER: "其他",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskDefinition:
|
||||
"""任务定义"""
|
||||
code: str # 任务编码
|
||||
name: str # 显示名称
|
||||
description: str # 描述
|
||||
domain: BusinessDomain # 业务域
|
||||
requires_window: bool = True # 是否需要时间窗口
|
||||
is_ods: bool = False # 是否为 ODS 任务
|
||||
is_dimension: bool = False # 是否为维度类任务(校验时区分)
|
||||
default_enabled: bool = True # 默认是否选中
|
||||
|
||||
|
||||
# ODS 任务到业务域的映射
|
||||
ODS_DOMAIN_MAP: Dict[str, BusinessDomain] = {
|
||||
# 会员相关
|
||||
"ODS_MEMBER": BusinessDomain.MEMBER,
|
||||
"ODS_MEMBER_CARD": BusinessDomain.MEMBER,
|
||||
"ODS_MEMBER_BALANCE": BusinessDomain.MEMBER,
|
||||
# 结算/支付相关
|
||||
"ODS_PAYMENT": BusinessDomain.SETTLEMENT,
|
||||
"ODS_REFUND": BusinessDomain.SETTLEMENT,
|
||||
"ODS_SETTLEMENT_RECORDS": BusinessDomain.SETTLEMENT,
|
||||
"ODS_RECHARGE_SETTLE": BusinessDomain.SETTLEMENT,
|
||||
"ODS_SETTLEMENT_TICKET": BusinessDomain.SETTLEMENT,
|
||||
# 助教相关
|
||||
"ODS_ASSISTANT_ACCOUNT": BusinessDomain.ASSISTANT,
|
||||
"ODS_ASSISTANT_LEDGER": BusinessDomain.ASSISTANT,
|
||||
"ODS_ASSISTANT_ABOLISH": BusinessDomain.ASSISTANT,
|
||||
# 商品/销售相关
|
||||
"ODS_TENANT_GOODS": BusinessDomain.GOODS,
|
||||
"ODS_STORE_GOODS": BusinessDomain.GOODS,
|
||||
"ODS_STORE_GOODS_SALES": BusinessDomain.GOODS,
|
||||
"ODS_GOODS_CATEGORY": BusinessDomain.GOODS,
|
||||
# 台桌相关
|
||||
"ODS_TABLES": BusinessDomain.TABLE,
|
||||
"ODS_TABLE_USE": BusinessDomain.TABLE,
|
||||
"ODS_TABLE_FEE_DISCOUNT": BusinessDomain.TABLE,
|
||||
# 团购/优惠券相关
|
||||
"ODS_GROUP_PACKAGE": BusinessDomain.PROMOTION,
|
||||
"ODS_GROUP_BUY_REDEMPTION": BusinessDomain.PROMOTION,
|
||||
"ODS_PLATFORM_COUPON": BusinessDomain.PROMOTION,
|
||||
# 库存相关
|
||||
"ODS_INVENTORY_STOCK": BusinessDomain.INVENTORY,
|
||||
"ODS_INVENTORY_CHANGE": BusinessDomain.INVENTORY,
|
||||
}
|
||||
|
||||
# ODS 任务显示名称(中文)
|
||||
ODS_DISPLAY_NAMES: Dict[str, str] = {
|
||||
"ODS_MEMBER": "会员档案",
|
||||
"ODS_MEMBER_CARD": "会员储值卡",
|
||||
"ODS_MEMBER_BALANCE": "会员余额变动",
|
||||
"ODS_PAYMENT": "支付流水",
|
||||
"ODS_REFUND": "退款流水",
|
||||
"ODS_SETTLEMENT_RECORDS": "结账记录",
|
||||
"ODS_RECHARGE_SETTLE": "充值结算",
|
||||
"ODS_SETTLEMENT_TICKET": "结账小票",
|
||||
"ODS_ASSISTANT_ACCOUNT": "助教账号",
|
||||
"ODS_ASSISTANT_LEDGER": "助教流水",
|
||||
"ODS_ASSISTANT_ABOLISH": "助教作废",
|
||||
"ODS_TENANT_GOODS": "租户商品",
|
||||
"ODS_STORE_GOODS": "门店商品",
|
||||
"ODS_STORE_GOODS_SALES": "商品销售流水",
|
||||
"ODS_GOODS_CATEGORY": "商品分类",
|
||||
"ODS_TABLES": "台桌维表",
|
||||
"ODS_TABLE_USE": "台费计费流水",
|
||||
"ODS_TABLE_FEE_DISCOUNT": "台费折扣调账",
|
||||
"ODS_GROUP_PACKAGE": "团购套餐",
|
||||
"ODS_GROUP_BUY_REDEMPTION": "团购核销",
|
||||
"ODS_PLATFORM_COUPON": "平台券核销",
|
||||
"ODS_INVENTORY_STOCK": "库存汇总",
|
||||
"ODS_INVENTORY_CHANGE": "库存变化",
|
||||
}
|
||||
|
||||
# 维度类 ODS 任务(校验时通常单独处理)
|
||||
DIMENSION_ODS_CODES = {
|
||||
"ODS_MEMBER",
|
||||
"ODS_MEMBER_CARD",
|
||||
"ODS_ASSISTANT_ACCOUNT",
|
||||
"ODS_TENANT_GOODS",
|
||||
"ODS_STORE_GOODS",
|
||||
"ODS_GOODS_CATEGORY",
|
||||
"ODS_TABLES",
|
||||
"ODS_GROUP_PACKAGE",
|
||||
}
|
||||
|
||||
# 事实类 ODS 任务(需要时间窗口)
|
||||
FACT_ODS_CODES = {
|
||||
"ODS_MEMBER_BALANCE",
|
||||
"ODS_PAYMENT",
|
||||
"ODS_REFUND",
|
||||
"ODS_SETTLEMENT_RECORDS",
|
||||
"ODS_RECHARGE_SETTLE",
|
||||
"ODS_SETTLEMENT_TICKET",
|
||||
"ODS_ASSISTANT_LEDGER",
|
||||
"ODS_ASSISTANT_ABOLISH",
|
||||
"ODS_STORE_GOODS_SALES",
|
||||
"ODS_TABLE_USE",
|
||||
"ODS_TABLE_FEE_DISCOUNT",
|
||||
"ODS_GROUP_BUY_REDEMPTION",
|
||||
"ODS_PLATFORM_COUPON",
|
||||
"ODS_INVENTORY_CHANGE",
|
||||
}
|
||||
|
||||
# ======================== DWD 表定义 ========================
|
||||
|
||||
@dataclass
|
||||
class DwdTableDefinition:
|
||||
"""DWD 表定义(用于 GUI 表级选择)"""
|
||||
code: str # 表编码(不含 schema,如 dim_member)
|
||||
name: str # 中文显示名称
|
||||
description: str # 描述
|
||||
domain: BusinessDomain # 业务域
|
||||
is_dimension: bool = False # 是否维度表
|
||||
tables: List[str] = field(default_factory=list) # 完整表名列表(含 _ex)
|
||||
|
||||
|
||||
# DWD 表定义列表(按业务域分组)
|
||||
DWD_TABLE_DEFINITIONS: List[DwdTableDefinition] = [
|
||||
# ---- 会员 ----
|
||||
DwdTableDefinition(
|
||||
"dim_member", "会员维度", "会员基本信息维度表",
|
||||
BusinessDomain.MEMBER, True,
|
||||
["billiards_dwd.dim_member", "billiards_dwd.dim_member_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dim_member_card_account", "会员储值卡", "会员储值卡账户维度表",
|
||||
BusinessDomain.MEMBER, True,
|
||||
["billiards_dwd.dim_member_card_account", "billiards_dwd.dim_member_card_account_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dwd_member_balance_change", "余额变动", "会员余额变动事实表",
|
||||
BusinessDomain.MEMBER, False,
|
||||
["billiards_dwd.dwd_member_balance_change", "billiards_dwd.dwd_member_balance_change_ex"],
|
||||
),
|
||||
# ---- 结算/支付 ----
|
||||
DwdTableDefinition(
|
||||
"dwd_settlement_head", "结账记录", "结账/结算事实表",
|
||||
BusinessDomain.SETTLEMENT, False,
|
||||
["billiards_dwd.dwd_settlement_head", "billiards_dwd.dwd_settlement_head_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dwd_payment", "支付流水", "支付明细事实表",
|
||||
BusinessDomain.SETTLEMENT, False,
|
||||
["billiards_dwd.dwd_payment"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dwd_refund", "退款流水", "退款明细事实表",
|
||||
BusinessDomain.SETTLEMENT, False,
|
||||
["billiards_dwd.dwd_refund", "billiards_dwd.dwd_refund_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dwd_recharge_order", "充值订单", "充值结算事实表",
|
||||
BusinessDomain.SETTLEMENT, False,
|
||||
["billiards_dwd.dwd_recharge_order", "billiards_dwd.dwd_recharge_order_ex"],
|
||||
),
|
||||
# ---- 助教 ----
|
||||
DwdTableDefinition(
|
||||
"dim_assistant", "助教维度", "助教基本信息维度表",
|
||||
BusinessDomain.ASSISTANT, True,
|
||||
["billiards_dwd.dim_assistant", "billiards_dwd.dim_assistant_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dwd_assistant_service_log", "助教服务流水", "助教服务计费事实表",
|
||||
BusinessDomain.ASSISTANT, False,
|
||||
["billiards_dwd.dwd_assistant_service_log", "billiards_dwd.dwd_assistant_service_log_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dwd_assistant_trash_event", "助教作废", "助教作废事件事实表",
|
||||
BusinessDomain.ASSISTANT, False,
|
||||
["billiards_dwd.dwd_assistant_trash_event", "billiards_dwd.dwd_assistant_trash_event_ex"],
|
||||
),
|
||||
# ---- 商品/销售 ----
|
||||
DwdTableDefinition(
|
||||
"dim_tenant_goods", "租户商品", "租户商品维度表",
|
||||
BusinessDomain.GOODS, True,
|
||||
["billiards_dwd.dim_tenant_goods", "billiards_dwd.dim_tenant_goods_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dim_store_goods", "门店商品", "门店商品维度表",
|
||||
BusinessDomain.GOODS, True,
|
||||
["billiards_dwd.dim_store_goods", "billiards_dwd.dim_store_goods_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dim_goods_category", "商品分类", "商品分类维度表",
|
||||
BusinessDomain.GOODS, True,
|
||||
["billiards_dwd.dim_goods_category"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dwd_store_goods_sale", "商品销售", "商品销售事实表",
|
||||
BusinessDomain.GOODS, False,
|
||||
["billiards_dwd.dwd_store_goods_sale", "billiards_dwd.dwd_store_goods_sale_ex"],
|
||||
),
|
||||
# ---- 台桌 ----
|
||||
DwdTableDefinition(
|
||||
"dim_site", "门店维度", "门店基本信息维度表",
|
||||
BusinessDomain.TABLE, True,
|
||||
["billiards_dwd.dim_site", "billiards_dwd.dim_site_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dim_table", "台桌维度", "台桌基本信息维度表",
|
||||
BusinessDomain.TABLE, True,
|
||||
["billiards_dwd.dim_table", "billiards_dwd.dim_table_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dwd_table_fee_log", "台费流水", "台费计费事实表",
|
||||
BusinessDomain.TABLE, False,
|
||||
["billiards_dwd.dwd_table_fee_log", "billiards_dwd.dwd_table_fee_log_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dwd_table_fee_adjust", "台费折扣调账", "台费折扣调账事实表",
|
||||
BusinessDomain.TABLE, False,
|
||||
["billiards_dwd.dwd_table_fee_adjust", "billiards_dwd.dwd_table_fee_adjust_ex"],
|
||||
),
|
||||
# ---- 团购/优惠券 ----
|
||||
DwdTableDefinition(
|
||||
"dim_groupbuy_package", "团购套餐", "团购套餐维度表",
|
||||
BusinessDomain.PROMOTION, True,
|
||||
["billiards_dwd.dim_groupbuy_package", "billiards_dwd.dim_groupbuy_package_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dwd_groupbuy_redemption", "团购核销", "团购核销事实表",
|
||||
BusinessDomain.PROMOTION, False,
|
||||
["billiards_dwd.dwd_groupbuy_redemption", "billiards_dwd.dwd_groupbuy_redemption_ex"],
|
||||
),
|
||||
DwdTableDefinition(
|
||||
"dwd_platform_coupon_redemption", "平台券核销", "平台券核销事实表",
|
||||
BusinessDomain.PROMOTION, False,
|
||||
["billiards_dwd.dwd_platform_coupon_redemption", "billiards_dwd.dwd_platform_coupon_redemption_ex"],
|
||||
),
|
||||
]
|
||||
|
||||
# DWD 表按业务域显示顺序
|
||||
DWD_TABLE_DOMAIN_ORDER: List[BusinessDomain] = [
|
||||
BusinessDomain.MEMBER,
|
||||
BusinessDomain.SETTLEMENT,
|
||||
BusinessDomain.ASSISTANT,
|
||||
BusinessDomain.GOODS,
|
||||
BusinessDomain.TABLE,
|
||||
BusinessDomain.PROMOTION,
|
||||
]
|
||||
|
||||
|
||||
def get_dwd_tables_grouped() -> Dict[BusinessDomain, List[DwdTableDefinition]]:
|
||||
"""获取按业务域分组的 DWD 表定义"""
|
||||
grouped: Dict[BusinessDomain, List[DwdTableDefinition]] = {}
|
||||
for tbl in DWD_TABLE_DEFINITIONS:
|
||||
grouped.setdefault(tbl.domain, []).append(tbl)
|
||||
return grouped
|
||||
|
||||
|
||||
def get_all_dwd_table_codes() -> List[str]:
|
||||
"""获取所有 DWD 表编码"""
|
||||
return [t.code for t in DWD_TABLE_DEFINITIONS]
|
||||
|
||||
|
||||
def resolve_dwd_table_names(codes: Sequence[str]) -> List[str]:
|
||||
"""将 DWD 表编码解析为完整表名列表(含 _ex)"""
|
||||
code_set = {c.lower() for c in codes}
|
||||
result: List[str] = []
|
||||
for tbl in DWD_TABLE_DEFINITIONS:
|
||||
if tbl.code.lower() in code_set:
|
||||
result.extend(tbl.tables)
|
||||
return result
|
||||
|
||||
|
||||
# 非 ODS 任务定义
|
||||
NON_ODS_TASKS: List[TaskDefinition] = [
|
||||
# DWD 装载(保留为单一调度任务,表级选择通过 DWD_ONLY_TABLES 环境变量控制)
|
||||
TaskDefinition(
|
||||
code="DWD_LOAD_FROM_ODS",
|
||||
name="ODS→DWD 装载",
|
||||
description="从 ODS 增量装载到 DWD",
|
||||
domain=BusinessDomain.DWD,
|
||||
requires_window=True,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWD_QUALITY_CHECK",
|
||||
name="DWD 质量检查",
|
||||
description="执行 DWD 数据质量检查",
|
||||
domain=BusinessDomain.QUALITY,
|
||||
requires_window=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_BUILD_ORDER_SUMMARY",
|
||||
name="构建订单汇总",
|
||||
description="重算 DWS 订单汇总表",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=False,
|
||||
),
|
||||
# DWS 汇总任务
|
||||
TaskDefinition(
|
||||
code="DWS_ASSISTANT_DAILY",
|
||||
name="助教日度明细",
|
||||
description="汇总助教日度服务、时长与收入指标",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_ASSISTANT_MONTHLY",
|
||||
name="助教月度汇总",
|
||||
description="汇总助教月度绩效与服务指标",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_ASSISTANT_CUSTOMER",
|
||||
name="助教客户统计",
|
||||
description="统计助教与客户的服务关系与滚动窗口指标",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_ASSISTANT_SALARY",
|
||||
name="助教工资计算",
|
||||
description="计算助教月度工资与奖金明细",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
default_enabled=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_ASSISTANT_FINANCE",
|
||||
name="助教财务分析",
|
||||
description="汇总助教日度财务分析指标",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_MEMBER_CONSUMPTION",
|
||||
name="会员消费汇总",
|
||||
description="汇总会员消费行为与滚动窗口指标",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_MEMBER_VISIT",
|
||||
name="会员来店明细",
|
||||
description="记录会员来店消费明细与服务列表",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_FINANCE_DAILY",
|
||||
name="财务日度汇总",
|
||||
description="汇总当日财务发生额、优惠与现金流",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_FINANCE_RECHARGE",
|
||||
name="财务充值统计",
|
||||
description="统计充值笔数、金额与卡余额",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_FINANCE_INCOME_STRUCTURE",
|
||||
name="财务收入结构",
|
||||
description="统计收入结构分布",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_FINANCE_DISCOUNT_DETAIL",
|
||||
name="优惠明细分析",
|
||||
description="拆分优惠构成与占比",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_MV_REFRESH_FINANCE_DAILY",
|
||||
name="物化刷新-财务日汇总",
|
||||
description="刷新财务日汇总物化视图(L1-L4)",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=False,
|
||||
default_enabled=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_MV_REFRESH_ASSISTANT_DAILY",
|
||||
name="物化刷新-助教日明细",
|
||||
description="刷新助教日明细物化视图(L1-L4)",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=False,
|
||||
default_enabled=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_RETENTION_CLEANUP",
|
||||
name="时间分层清理",
|
||||
description="按配置清理历史 DWS 数据",
|
||||
domain=BusinessDomain.DWS,
|
||||
requires_window=True,
|
||||
default_enabled=False,
|
||||
),
|
||||
# DWS 指数计算
|
||||
TaskDefinition(
|
||||
code="DWS_WINBACK_INDEX",
|
||||
name="老客挽回指数(WBI)",
|
||||
description="计算老客挽回优先级,基于个人周期超期、降频、价值与充值压力",
|
||||
domain=BusinessDomain.INDEX,
|
||||
requires_window=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_NEWCONV_INDEX",
|
||||
name="新客转化指数(NCI)",
|
||||
description="计算新客二访/三访转化紧迫度与价值",
|
||||
domain=BusinessDomain.INDEX,
|
||||
requires_window=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_RELATION_INDEX",
|
||||
name="关系指数(RS/OS/MS/ML)",
|
||||
description="单任务计算关系强度、归属份额、升温动量、付费关联",
|
||||
domain=BusinessDomain.INDEX,
|
||||
requires_window=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DWS_ML_MANUAL_IMPORT",
|
||||
name="ML人工台账导入",
|
||||
description="导入人工台账并按日/30天批次覆盖写入 ML 归因明细",
|
||||
domain=BusinessDomain.INDEX,
|
||||
requires_window=False,
|
||||
default_enabled=False,
|
||||
),
|
||||
# Schema 初始化
|
||||
TaskDefinition(
|
||||
code="INIT_ODS_SCHEMA",
|
||||
name="初始化 ODS Schema",
|
||||
description="创建/重建 ODS 表结构",
|
||||
domain=BusinessDomain.SCHEMA,
|
||||
requires_window=False,
|
||||
default_enabled=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="INIT_DWD_SCHEMA",
|
||||
name="初始化 DWD Schema",
|
||||
description="创建/重建 DWD 表结构",
|
||||
domain=BusinessDomain.SCHEMA,
|
||||
requires_window=False,
|
||||
default_enabled=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="INIT_DWS_SCHEMA",
|
||||
name="初始化 DWS Schema",
|
||||
description="创建/重建 DWS 表结构",
|
||||
domain=BusinessDomain.SCHEMA,
|
||||
requires_window=False,
|
||||
default_enabled=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="SEED_DWS_CONFIG",
|
||||
name="初始化 DWS 配置",
|
||||
description="写入 DWS 配置表基础数据",
|
||||
domain=BusinessDomain.SCHEMA,
|
||||
requires_window=False,
|
||||
default_enabled=False,
|
||||
),
|
||||
# 其他
|
||||
TaskDefinition(
|
||||
code="MANUAL_INGEST",
|
||||
name="手工数据灌入",
|
||||
description="从本地 JSON 回放入库",
|
||||
domain=BusinessDomain.OTHER,
|
||||
requires_window=False,
|
||||
default_enabled=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="ODS_JSON_ARCHIVE",
|
||||
name="ODS JSON 归档",
|
||||
description="在线抓取 ODS 接口数据并落盘 JSON",
|
||||
domain=BusinessDomain.OTHER,
|
||||
requires_window=True,
|
||||
default_enabled=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="CHECK_CUTOFF",
|
||||
name="检查 Cutoff",
|
||||
description="查看各表数据截止时间",
|
||||
domain=BusinessDomain.QUALITY,
|
||||
requires_window=False,
|
||||
),
|
||||
TaskDefinition(
|
||||
code="DATA_INTEGRITY_CHECK",
|
||||
name="数据完整性检查",
|
||||
description="检查 ODS/DWD 数据完整性",
|
||||
domain=BusinessDomain.QUALITY,
|
||||
requires_window=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _build_ods_task_definition(code: str) -> TaskDefinition:
|
||||
"""根据 ODS 任务编码构建任务定义"""
|
||||
domain = ODS_DOMAIN_MAP.get(code, BusinessDomain.OTHER)
|
||||
name = ODS_DISPLAY_NAMES.get(code, code)
|
||||
is_dimension = code in DIMENSION_ODS_CODES
|
||||
|
||||
# 从后端获取描述(如果可用)
|
||||
description = f"抓取{name}到 ODS"
|
||||
if _HAS_BACKEND:
|
||||
for spec in ODS_TASK_SPECS:
|
||||
if spec.code == code:
|
||||
# 尝试解码描述(可能是乱码)
|
||||
desc = spec.description
|
||||
if desc and not any(ord(c) > 0x4e00 for c in desc[:10] if desc):
|
||||
description = f"抓取{name}到 ODS"
|
||||
break
|
||||
|
||||
return TaskDefinition(
|
||||
code=code,
|
||||
name=name,
|
||||
description=description,
|
||||
domain=domain,
|
||||
requires_window=code not in DIMENSION_ODS_CODES,
|
||||
is_ods=True,
|
||||
is_dimension=is_dimension,
|
||||
)
|
||||
|
||||
|
||||
class TaskRegistry:
|
||||
"""任务注册表:管理所有可用任务"""
|
||||
|
||||
_instance: Optional["TaskRegistry"] = None
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
cls._instance._initialized = False
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
if self._initialized:
|
||||
return
|
||||
self._initialized = True
|
||||
self._tasks: Dict[str, TaskDefinition] = {}
|
||||
self._load_tasks()
|
||||
|
||||
def _load_tasks(self):
|
||||
"""加载所有任务定义"""
|
||||
# 加载 ODS 任务
|
||||
ods_codes = ENABLED_ODS_CODES if _HAS_BACKEND else set(ODS_DOMAIN_MAP.keys())
|
||||
for code in ods_codes:
|
||||
self._tasks[code] = _build_ods_task_definition(code)
|
||||
|
||||
# 加载非 ODS 任务
|
||||
for task_def in NON_ODS_TASKS:
|
||||
self._tasks[task_def.code] = task_def
|
||||
|
||||
def get_task(self, code: str) -> Optional[TaskDefinition]:
|
||||
"""获取任务定义"""
|
||||
return self._tasks.get(code)
|
||||
|
||||
def get_all_tasks(self) -> List[TaskDefinition]:
|
||||
"""获取所有任务"""
|
||||
return list(self._tasks.values())
|
||||
|
||||
def get_ods_tasks(self) -> List[TaskDefinition]:
|
||||
"""获取所有 ODS 任务"""
|
||||
return [t for t in self._tasks.values() if t.is_ods]
|
||||
|
||||
def get_fact_ods_tasks(self) -> List[TaskDefinition]:
|
||||
"""获取事实类 ODS 任务(需要时间窗口)"""
|
||||
return [t for t in self._tasks.values() if t.is_ods and not t.is_dimension]
|
||||
|
||||
def get_dimension_ods_tasks(self) -> List[TaskDefinition]:
|
||||
"""获取维度类 ODS 任务"""
|
||||
return [t for t in self._tasks.values() if t.is_ods and t.is_dimension]
|
||||
|
||||
def get_tasks_by_domain(self, domain: BusinessDomain) -> List[TaskDefinition]:
|
||||
"""按业务域获取任务"""
|
||||
return [t for t in self._tasks.values() if t.domain == domain]
|
||||
|
||||
def get_ods_tasks_grouped(self) -> Dict[BusinessDomain, List[TaskDefinition]]:
|
||||
"""获取按业务域分组的 ODS 任务"""
|
||||
grouped: Dict[BusinessDomain, List[TaskDefinition]] = {}
|
||||
for task in self.get_ods_tasks():
|
||||
if task.domain not in grouped:
|
||||
grouped[task.domain] = []
|
||||
grouped[task.domain].append(task)
|
||||
return grouped
|
||||
|
||||
def get_non_ods_tasks(self) -> List[TaskDefinition]:
|
||||
"""获取非 ODS 任务"""
|
||||
return [t for t in self._tasks.values() if not t.is_ods]
|
||||
|
||||
|
||||
# 全局注册表实例
|
||||
task_registry = TaskRegistry()
|
||||
|
||||
|
||||
# 便捷函数
|
||||
def get_ods_task_codes() -> List[str]:
|
||||
"""获取所有 ODS 任务编码"""
|
||||
return [t.code for t in task_registry.get_ods_tasks()]
|
||||
|
||||
|
||||
def get_fact_ods_task_codes() -> List[str]:
|
||||
"""获取事实类 ODS 任务编码"""
|
||||
return [t.code for t in task_registry.get_fact_ods_tasks()]
|
||||
|
||||
|
||||
def get_dimension_ods_task_codes() -> List[str]:
|
||||
"""获取维度类 ODS 任务编码"""
|
||||
return [t.code for t in task_registry.get_dimension_ods_tasks()]
|
||||
|
||||
|
||||
def get_all_task_tuples() -> List[Tuple[str, str, str]]:
|
||||
"""获取所有任务的 (code, name, description) 元组列表"""
|
||||
return [(t.code, t.name, t.description) for t in task_registry.get_all_tasks()]
|
||||
|
||||
|
||||
def get_ods_tasks_for_ui() -> List[Tuple[str, str, BusinessDomain]]:
|
||||
"""获取 ODS 任务列表供 UI 使用:(code, display_name, domain)"""
|
||||
return [(t.code, t.name, t.domain) for t in task_registry.get_ods_tasks()]
|
||||
11
gui/pyproject.toml
Normal file
11
gui/pyproject.toml
Normal file
@@ -0,0 +1,11 @@
|
||||
[project]
|
||||
name = "etl-gui"
|
||||
version = "0.1.0"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"PySide6>=6.5.0",
|
||||
"neozqyy-shared",
|
||||
]
|
||||
|
||||
[tool.uv.sources]
|
||||
neozqyy-shared = { workspace = true }
|
||||
14
gui/resources/__init__.py
Normal file
14
gui/resources/__init__.py
Normal file
@@ -0,0 +1,14 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""GUI 资源模块"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
RESOURCES_DIR = Path(__file__).parent
|
||||
STYLES_PATH = RESOURCES_DIR / "styles.qss"
|
||||
|
||||
|
||||
def load_stylesheet() -> str:
|
||||
"""加载样式表"""
|
||||
if STYLES_PATH.exists():
|
||||
return STYLES_PATH.read_text(encoding="utf-8")
|
||||
return ""
|
||||
458
gui/resources/styles.qss
Normal file
458
gui/resources/styles.qss
Normal file
@@ -0,0 +1,458 @@
|
||||
/* ETL GUI 现代浅色主题样式表 */
|
||||
|
||||
/* ========== 全局样式 ========== */
|
||||
QWidget {
|
||||
font-family: "Microsoft YaHei", "Segoe UI", sans-serif;
|
||||
font-size: 13px;
|
||||
color: #333333;
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
|
||||
QMainWindow {
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
|
||||
/* ========== 菜单栏 ========== */
|
||||
QMenuBar {
|
||||
background-color: #ffffff;
|
||||
border-bottom: 1px solid #e0e0e0;
|
||||
padding: 4px;
|
||||
}
|
||||
|
||||
QMenuBar::item {
|
||||
padding: 6px 12px;
|
||||
background-color: transparent;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
QMenuBar::item:selected {
|
||||
background-color: #e8f0fe;
|
||||
}
|
||||
|
||||
QMenu {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #e0e0e0;
|
||||
border-radius: 8px;
|
||||
padding: 4px;
|
||||
}
|
||||
|
||||
QMenu::item {
|
||||
padding: 8px 24px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
QMenu::item:selected {
|
||||
background-color: #e8f0fe;
|
||||
}
|
||||
|
||||
/* ========== 工具栏 ========== */
|
||||
QToolBar {
|
||||
background-color: #ffffff;
|
||||
border-bottom: 1px solid #e0e0e0;
|
||||
padding: 4px;
|
||||
spacing: 4px;
|
||||
}
|
||||
|
||||
QToolButton {
|
||||
background-color: transparent;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
QToolButton:hover {
|
||||
background-color: #e8f0fe;
|
||||
}
|
||||
|
||||
QToolButton:pressed {
|
||||
background-color: #d2e3fc;
|
||||
}
|
||||
|
||||
/* ========== 按钮 ========== */
|
||||
QPushButton {
|
||||
background-color: #1a73e8;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
padding: 8px 16px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
QPushButton:hover {
|
||||
background-color: #1557b0;
|
||||
}
|
||||
|
||||
QPushButton:pressed {
|
||||
background-color: #104080;
|
||||
}
|
||||
|
||||
QPushButton:disabled {
|
||||
background-color: #dadce0;
|
||||
color: #9aa0a6;
|
||||
}
|
||||
|
||||
QPushButton[secondary="true"] {
|
||||
background-color: #ffffff;
|
||||
color: #1a73e8;
|
||||
border: 1px solid #dadce0;
|
||||
}
|
||||
|
||||
QPushButton[secondary="true"]:hover {
|
||||
background-color: #f8f9fa;
|
||||
border-color: #1a73e8;
|
||||
}
|
||||
|
||||
QPushButton[danger="true"] {
|
||||
background-color: #ea4335;
|
||||
}
|
||||
|
||||
QPushButton[danger="true"]:hover {
|
||||
background-color: #c5221f;
|
||||
}
|
||||
|
||||
/* ========== 输入框 ========== */
|
||||
QLineEdit, QTextEdit, QPlainTextEdit {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 6px;
|
||||
padding: 8px 12px;
|
||||
selection-background-color: #d2e3fc;
|
||||
}
|
||||
|
||||
QLineEdit:focus, QTextEdit:focus, QPlainTextEdit:focus {
|
||||
border-color: #1a73e8;
|
||||
border-width: 2px;
|
||||
padding: 7px 11px;
|
||||
}
|
||||
|
||||
QLineEdit:disabled, QTextEdit:disabled, QPlainTextEdit:disabled {
|
||||
background-color: #f1f3f4;
|
||||
color: #9aa0a6;
|
||||
}
|
||||
|
||||
/* ========== 下拉框 ========== */
|
||||
QComboBox {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 6px;
|
||||
padding: 8px 12px;
|
||||
padding-right: 30px;
|
||||
}
|
||||
|
||||
QComboBox:hover {
|
||||
border-color: #1a73e8;
|
||||
}
|
||||
|
||||
QComboBox:focus {
|
||||
border-color: #1a73e8;
|
||||
border-width: 2px;
|
||||
}
|
||||
|
||||
QComboBox::drop-down {
|
||||
border: none;
|
||||
width: 24px;
|
||||
}
|
||||
|
||||
QComboBox::down-arrow {
|
||||
image: none;
|
||||
border-left: 5px solid transparent;
|
||||
border-right: 5px solid transparent;
|
||||
border-top: 6px solid #5f6368;
|
||||
margin-right: 8px;
|
||||
}
|
||||
|
||||
QComboBox QAbstractItemView {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 8px;
|
||||
selection-background-color: #e8f0fe;
|
||||
}
|
||||
|
||||
/* ========== 复选框 ========== */
|
||||
QCheckBox {
|
||||
spacing: 8px;
|
||||
}
|
||||
|
||||
QCheckBox::indicator {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
border-radius: 4px;
|
||||
border: 2px solid #5f6368;
|
||||
}
|
||||
|
||||
QCheckBox::indicator:checked {
|
||||
background-color: #1a73e8;
|
||||
border-color: #1a73e8;
|
||||
}
|
||||
|
||||
QCheckBox::indicator:hover {
|
||||
border-color: #1a73e8;
|
||||
}
|
||||
|
||||
/* ========== 列表和树 ========== */
|
||||
QListWidget, QTreeWidget, QTableWidget {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 8px;
|
||||
outline: none;
|
||||
}
|
||||
|
||||
QListWidget::item, QTreeWidget::item {
|
||||
padding: 8px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
QListWidget::item:selected, QTreeWidget::item:selected {
|
||||
background-color: #e8f0fe;
|
||||
color: #1a73e8;
|
||||
}
|
||||
|
||||
QListWidget::item:hover, QTreeWidget::item:hover {
|
||||
background-color: #f8f9fa;
|
||||
}
|
||||
|
||||
QHeaderView::section {
|
||||
background-color: #f8f9fa;
|
||||
border: none;
|
||||
border-bottom: 1px solid #dadce0;
|
||||
padding: 10px 16px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
QTableWidget {
|
||||
gridline-color: #e8eaed;
|
||||
}
|
||||
|
||||
QTableWidget::item {
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
QTableWidget::item:selected {
|
||||
background-color: #e8f0fe;
|
||||
color: #1a73e8;
|
||||
}
|
||||
|
||||
/* ========== 滚动条 ========== */
|
||||
QScrollBar:vertical {
|
||||
background-color: transparent;
|
||||
width: 12px;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
QScrollBar::handle:vertical {
|
||||
background-color: #dadce0;
|
||||
border-radius: 6px;
|
||||
min-height: 30px;
|
||||
margin: 2px;
|
||||
}
|
||||
|
||||
QScrollBar::handle:vertical:hover {
|
||||
background-color: #bdc1c6;
|
||||
}
|
||||
|
||||
QScrollBar::add-line:vertical, QScrollBar::sub-line:vertical {
|
||||
height: 0;
|
||||
}
|
||||
|
||||
QScrollBar:horizontal {
|
||||
background-color: transparent;
|
||||
height: 12px;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
QScrollBar::handle:horizontal {
|
||||
background-color: #dadce0;
|
||||
border-radius: 6px;
|
||||
min-width: 30px;
|
||||
margin: 2px;
|
||||
}
|
||||
|
||||
QScrollBar::handle:horizontal:hover {
|
||||
background-color: #bdc1c6;
|
||||
}
|
||||
|
||||
QScrollBar::add-line:horizontal, QScrollBar::sub-line:horizontal {
|
||||
width: 0;
|
||||
}
|
||||
|
||||
/* ========== 选项卡 ========== */
|
||||
QTabWidget::pane {
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 8px;
|
||||
background-color: #ffffff;
|
||||
margin-top: -1px;
|
||||
}
|
||||
|
||||
QTabBar::tab {
|
||||
background-color: transparent;
|
||||
border: none;
|
||||
padding: 10px 20px;
|
||||
margin-right: 4px;
|
||||
color: #5f6368;
|
||||
}
|
||||
|
||||
QTabBar::tab:selected {
|
||||
color: #1a73e8;
|
||||
border-bottom: 2px solid #1a73e8;
|
||||
}
|
||||
|
||||
QTabBar::tab:hover:!selected {
|
||||
background-color: #f8f9fa;
|
||||
border-radius: 6px 6px 0 0;
|
||||
}
|
||||
|
||||
/* ========== 分组框 ========== */
|
||||
QGroupBox {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 8px;
|
||||
margin-top: 16px;
|
||||
padding: 16px;
|
||||
padding-top: 24px;
|
||||
}
|
||||
|
||||
QGroupBox::title {
|
||||
subcontrol-origin: margin;
|
||||
subcontrol-position: top left;
|
||||
left: 16px;
|
||||
padding: 0 8px;
|
||||
background-color: #ffffff;
|
||||
color: #5f6368;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* ========== 进度条 ========== */
|
||||
QProgressBar {
|
||||
background-color: #e8eaed;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
height: 8px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
QProgressBar::chunk {
|
||||
background-color: #1a73e8;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
/* ========== 分割器 ========== */
|
||||
QSplitter::handle {
|
||||
background-color: #e0e0e0;
|
||||
}
|
||||
|
||||
QSplitter::handle:horizontal {
|
||||
width: 2px;
|
||||
}
|
||||
|
||||
QSplitter::handle:vertical {
|
||||
height: 2px;
|
||||
}
|
||||
|
||||
QSplitter::handle:hover {
|
||||
background-color: #1a73e8;
|
||||
}
|
||||
|
||||
/* ========== 状态栏 ========== */
|
||||
QStatusBar {
|
||||
background-color: #ffffff;
|
||||
border-top: 1px solid #e0e0e0;
|
||||
padding: 4px;
|
||||
}
|
||||
|
||||
QStatusBar::item {
|
||||
border: none;
|
||||
}
|
||||
|
||||
/* ========== 提示框 ========== */
|
||||
QToolTip {
|
||||
background-color: #3c4043;
|
||||
color: #ffffff;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
padding: 8px 12px;
|
||||
}
|
||||
|
||||
/* ========== 消息框 ========== */
|
||||
QMessageBox {
|
||||
background-color: #ffffff;
|
||||
}
|
||||
|
||||
/* ========== 导航侧边栏 ========== */
|
||||
QListWidget#navList {
|
||||
background-color: #ffffff;
|
||||
border: none;
|
||||
border-right: 1px solid #e0e0e0;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
QListWidget#navList::item {
|
||||
padding: 12px 16px;
|
||||
border-radius: 8px;
|
||||
margin: 2px 0;
|
||||
}
|
||||
|
||||
QListWidget#navList::item:selected {
|
||||
background-color: #e8f0fe;
|
||||
color: #1a73e8;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* ========== 日志查看器 ========== */
|
||||
QPlainTextEdit#logViewer {
|
||||
font-family: "Consolas", "Courier New", monospace;
|
||||
font-size: 12px;
|
||||
background-color: #fafafa;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
/* ========== SQL 编辑器 ========== */
|
||||
QPlainTextEdit#sqlEditor {
|
||||
font-family: "Consolas", "Courier New", monospace;
|
||||
font-size: 13px;
|
||||
background-color: #ffffff;
|
||||
}
|
||||
|
||||
/* ========== 卡片样式 ========== */
|
||||
QFrame[card="true"] {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 12px;
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
QFrame[card="true"]:hover {
|
||||
border-color: #1a73e8;
|
||||
/* box-shadow not supported in Qt StyleSheets */
|
||||
}
|
||||
|
||||
/* ========== 标签 ========== */
|
||||
QLabel[heading="true"] {
|
||||
font-size: 18px;
|
||||
font-weight: 600;
|
||||
color: #202124;
|
||||
}
|
||||
|
||||
QLabel[subheading="true"] {
|
||||
font-size: 14px;
|
||||
color: #5f6368;
|
||||
}
|
||||
|
||||
QLabel[status="success"] {
|
||||
color: #1e8e3e;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
QLabel[status="error"] {
|
||||
color: #d93025;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
QLabel[status="warning"] {
|
||||
color: #f9ab00;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
QLabel[status="info"] {
|
||||
color: #1a73e8;
|
||||
font-weight: 500;
|
||||
}
|
||||
8
gui/utils/__init__.py
Normal file
8
gui/utils/__init__.py
Normal file
@@ -0,0 +1,8 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""工具模块"""
|
||||
|
||||
from .cli_builder import CLIBuilder
|
||||
from .config_helper import ConfigHelper
|
||||
from .app_settings import app_settings, AppSettings
|
||||
|
||||
__all__ = ["CLIBuilder", "ConfigHelper", "app_settings", "AppSettings"]
|
||||
837
gui/utils/app_settings.py
Normal file
837
gui/utils/app_settings.py
Normal file
@@ -0,0 +1,837 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# AI_CHANGELOG [2026-02-13] 移除 index_intimacy_check 属性
|
||||
"""应用程序设置管理"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
class AppSettings:
|
||||
"""应用程序设置单例"""
|
||||
|
||||
_instance: Optional["AppSettings"] = None
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
cls._instance._initialized = False
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
if self._initialized:
|
||||
return
|
||||
self._initialized = True
|
||||
|
||||
# 配置文件路径
|
||||
self._settings_file = self._get_settings_path()
|
||||
|
||||
# 默认设置
|
||||
self._settings = {
|
||||
"etl_project_path": "", # ETL 项目路径
|
||||
"env_file_path": "", # .env 文件路径
|
||||
# 窗口状态
|
||||
"window_state": {
|
||||
"geometry": None, # 窗口位置和大小 [x, y, width, height]
|
||||
"maximized": False, # 是否最大化
|
||||
"current_panel": 0, # 当前选中的面板索引
|
||||
"splitter_sizes": None, # 分割器大小
|
||||
},
|
||||
# 任务管理状态
|
||||
"task_manager_state": {
|
||||
"scheduler_enabled": False, # 调度器是否启用
|
||||
"auto_run_enabled": False, # 自动执行是否启用
|
||||
"current_tab": 0, # 当前选项卡索引
|
||||
},
|
||||
# 任务面板状态
|
||||
"task_panel_state": {
|
||||
"advanced_expanded": False, # 高级选项是否展开
|
||||
"current_tab": 0, # 当前选项卡
|
||||
"dwd_tasks": [], # DWD 任务选择
|
||||
"dws_tasks": [], # DWS 任务选择
|
||||
"build_tasks": [], # 数据建设任务选择
|
||||
"window_split": "day",
|
||||
"window_split_days": 10,
|
||||
"build_window_mode": "lookback",
|
||||
"build_lookback_hours": 24,
|
||||
"build_window_start": "",
|
||||
"build_window_end": "",
|
||||
"build_window_split": "day",
|
||||
"build_window_split_days": 10,
|
||||
"ml_manual_file_path": "",
|
||||
"index_relation_check": True,
|
||||
},
|
||||
# 自动更新配置
|
||||
"auto_update": {
|
||||
"hours": 24,
|
||||
"overlap_seconds": 600,
|
||||
"include_dwd": True,
|
||||
"auto_verify": False,
|
||||
"selected_tasks": [],
|
||||
},
|
||||
# 数据校验配置
|
||||
"integrity_check": {
|
||||
"mode": "history",
|
||||
"history_start": "",
|
||||
"history_end": "",
|
||||
"lookback_hours": 24,
|
||||
"include_dimensions": True,
|
||||
"auto_backfill": False,
|
||||
"ods_tasks": "",
|
||||
},
|
||||
# 高级配置
|
||||
"advanced": {
|
||||
"pipeline_flow": "FULL",
|
||||
"dry_run": False,
|
||||
"window_start": "",
|
||||
"window_end": "",
|
||||
"window_split": "none",
|
||||
"window_compensation": 0,
|
||||
"ingest_source": "",
|
||||
"store_id": "",
|
||||
"pg_dsn": "",
|
||||
"api_token": "",
|
||||
},
|
||||
}
|
||||
|
||||
# 加载设置
|
||||
self._load()
|
||||
|
||||
# 如果没有配置,尝试自动检测
|
||||
if not self._settings["etl_project_path"]:
|
||||
self._auto_detect_paths()
|
||||
|
||||
def _get_settings_path(self) -> Path:
|
||||
"""获取设置文件路径"""
|
||||
# 优先使用用户目录
|
||||
if sys.platform == "win32":
|
||||
app_data = os.environ.get("APPDATA", "")
|
||||
if app_data:
|
||||
settings_dir = Path(app_data) / "ETL管理系统"
|
||||
else:
|
||||
settings_dir = Path.home() / ".etl_gui"
|
||||
else:
|
||||
settings_dir = Path.home() / ".etl_gui"
|
||||
|
||||
settings_dir.mkdir(parents=True, exist_ok=True)
|
||||
return settings_dir / "settings.json"
|
||||
|
||||
def _auto_detect_paths(self):
|
||||
"""自动检测 ETL 项目路径"""
|
||||
# 方法1: 检查是否从源码目录运行
|
||||
try:
|
||||
source_dir = Path(__file__).resolve().parents[2]
|
||||
cli_main = source_dir / "cli" / "main.py"
|
||||
if cli_main.exists():
|
||||
rel_source = Path(os.path.relpath(source_dir, Path.cwd()))
|
||||
self._settings["etl_project_path"] = str(rel_source)
|
||||
env_file = rel_source / ".env"
|
||||
if env_file.exists():
|
||||
self._settings["env_file_path"] = str(env_file)
|
||||
self._save()
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 方法2: 检查常见位置
|
||||
common_paths = [
|
||||
Path("."),
|
||||
]
|
||||
|
||||
for path in common_paths:
|
||||
if path.exists() and (path / "cli" / "main.py").exists():
|
||||
self._settings["etl_project_path"] = str(path)
|
||||
env_file = path / ".env"
|
||||
if env_file.exists():
|
||||
self._settings["env_file_path"] = str(env_file)
|
||||
self._save()
|
||||
return
|
||||
|
||||
def _load(self):
|
||||
"""加载设置"""
|
||||
if self._settings_file.exists():
|
||||
try:
|
||||
data = json.loads(self._settings_file.read_text(encoding="utf-8"))
|
||||
self._settings.update(data)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _save(self):
|
||||
"""保存设置"""
|
||||
try:
|
||||
self._settings_file.write_text(
|
||||
json.dumps(self._settings, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@property
|
||||
def etl_project_path(self) -> str:
|
||||
"""获取 ETL 项目路径"""
|
||||
return self._settings.get("etl_project_path", "")
|
||||
|
||||
@etl_project_path.setter
|
||||
def etl_project_path(self, value: str):
|
||||
"""设置 ETL 项目路径"""
|
||||
self._settings["etl_project_path"] = value
|
||||
# 同时更新 .env 路径
|
||||
if value:
|
||||
env_path = Path(value) / ".env"
|
||||
if env_path.exists():
|
||||
self._settings["env_file_path"] = str(env_path)
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def env_file_path(self) -> str:
|
||||
"""获取 .env 文件路径"""
|
||||
path = self._settings.get("env_file_path", "")
|
||||
if not path and self.etl_project_path:
|
||||
path = str(Path(self.etl_project_path) / ".env")
|
||||
return path
|
||||
|
||||
@env_file_path.setter
|
||||
def env_file_path(self, value: str):
|
||||
"""设置 .env 文件路径"""
|
||||
self._settings["env_file_path"] = value
|
||||
self._save()
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
"""检查是否已配置"""
|
||||
path = self.etl_project_path
|
||||
if not path:
|
||||
return False
|
||||
return Path(path).exists() and (Path(path) / "cli" / "main.py").exists()
|
||||
|
||||
def validate(self) -> tuple[bool, str]:
|
||||
"""验证配置"""
|
||||
path = self.etl_project_path
|
||||
if not path:
|
||||
return False, "未配置 ETL 项目路径"
|
||||
|
||||
project_path = Path(path)
|
||||
if not project_path.exists():
|
||||
return False, f"ETL 项目路径不存在: {path}"
|
||||
|
||||
cli_main = project_path / "cli" / "main.py"
|
||||
if not cli_main.exists():
|
||||
return False, f"找不到 CLI 入口: {cli_main}"
|
||||
|
||||
return True, "配置有效"
|
||||
|
||||
# ==================== 自动更新配置 ====================
|
||||
|
||||
@property
|
||||
def auto_update_hours(self) -> int:
|
||||
return self._settings.get("auto_update", {}).get("hours", 24)
|
||||
|
||||
@auto_update_hours.setter
|
||||
def auto_update_hours(self, value: int):
|
||||
self._settings.setdefault("auto_update", {})["hours"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def auto_update_overlap_seconds(self) -> int:
|
||||
return self._settings.get("auto_update", {}).get("overlap_seconds", 3600)
|
||||
|
||||
@auto_update_overlap_seconds.setter
|
||||
def auto_update_overlap_seconds(self, value: int):
|
||||
self._settings.setdefault("auto_update", {})["overlap_seconds"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def auto_update_include_dwd(self) -> bool:
|
||||
return self._settings.get("auto_update", {}).get("include_dwd", True)
|
||||
|
||||
@auto_update_include_dwd.setter
|
||||
def auto_update_include_dwd(self, value: bool):
|
||||
self._settings.setdefault("auto_update", {})["include_dwd"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def auto_update_auto_verify(self) -> bool:
|
||||
return self._settings.get("auto_update", {}).get("auto_verify", False)
|
||||
|
||||
@auto_update_auto_verify.setter
|
||||
def auto_update_auto_verify(self, value: bool):
|
||||
self._settings.setdefault("auto_update", {})["auto_verify"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def auto_update_selected_tasks(self) -> list:
|
||||
return self._settings.get("auto_update", {}).get("selected_tasks", [])
|
||||
|
||||
@auto_update_selected_tasks.setter
|
||||
def auto_update_selected_tasks(self, value: list):
|
||||
self._settings.setdefault("auto_update", {})["selected_tasks"] = value
|
||||
self._save()
|
||||
|
||||
# ==================== 数据校验配置 ====================
|
||||
|
||||
@property
|
||||
def integrity_mode(self) -> str:
|
||||
return self._settings.get("integrity_check", {}).get("mode", "history")
|
||||
|
||||
@integrity_mode.setter
|
||||
def integrity_mode(self, value: str):
|
||||
self._settings.setdefault("integrity_check", {})["mode"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_history_start(self) -> str:
|
||||
return self._settings.get("integrity_check", {}).get("history_start", "")
|
||||
|
||||
@integrity_history_start.setter
|
||||
def integrity_history_start(self, value: str):
|
||||
self._settings.setdefault("integrity_check", {})["history_start"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_history_end(self) -> str:
|
||||
return self._settings.get("integrity_check", {}).get("history_end", "")
|
||||
|
||||
@integrity_history_end.setter
|
||||
def integrity_history_end(self, value: str):
|
||||
self._settings.setdefault("integrity_check", {})["history_end"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_lookback_hours(self) -> int:
|
||||
return self._settings.get("integrity_check", {}).get("lookback_hours", 24)
|
||||
|
||||
@integrity_lookback_hours.setter
|
||||
def integrity_lookback_hours(self, value: int):
|
||||
self._settings.setdefault("integrity_check", {})["lookback_hours"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_include_dimensions(self) -> bool:
|
||||
return self._settings.get("integrity_check", {}).get("include_dimensions", True)
|
||||
|
||||
@integrity_include_dimensions.setter
|
||||
def integrity_include_dimensions(self, value: bool):
|
||||
self._settings.setdefault("integrity_check", {})["include_dimensions"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_auto_backfill(self) -> bool:
|
||||
return self._settings.get("integrity_check", {}).get("auto_backfill", False)
|
||||
|
||||
@integrity_auto_backfill.setter
|
||||
def integrity_auto_backfill(self, value: bool):
|
||||
self._settings.setdefault("integrity_check", {})["auto_backfill"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_ods_tasks(self) -> str:
|
||||
return self._settings.get("integrity_check", {}).get("ods_tasks", "")
|
||||
|
||||
@integrity_ods_tasks.setter
|
||||
def integrity_ods_tasks(self, value: str):
|
||||
self._settings.setdefault("integrity_check", {})["ods_tasks"] = value
|
||||
self._save()
|
||||
|
||||
# ==================== 高级配置 ====================
|
||||
|
||||
@property
|
||||
def advanced_pipeline_flow(self) -> str:
|
||||
return self._settings.get("advanced", {}).get("pipeline_flow", "FULL")
|
||||
|
||||
@advanced_pipeline_flow.setter
|
||||
def advanced_pipeline_flow(self, value: str):
|
||||
self._settings.setdefault("advanced", {})["pipeline_flow"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_dry_run(self) -> bool:
|
||||
return self._settings.get("advanced", {}).get("dry_run", False)
|
||||
|
||||
@advanced_dry_run.setter
|
||||
def advanced_dry_run(self, value: bool):
|
||||
self._settings.setdefault("advanced", {})["dry_run"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_window_start(self) -> str:
|
||||
return self._settings.get("advanced", {}).get("window_start", "")
|
||||
|
||||
@advanced_window_start.setter
|
||||
def advanced_window_start(self, value: str):
|
||||
self._settings.setdefault("advanced", {})["window_start"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_window_end(self) -> str:
|
||||
return self._settings.get("advanced", {}).get("window_end", "")
|
||||
|
||||
@advanced_window_end.setter
|
||||
def advanced_window_end(self, value: str):
|
||||
self._settings.setdefault("advanced", {})["window_end"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_ingest_source(self) -> str:
|
||||
return self._settings.get("advanced", {}).get("ingest_source", "")
|
||||
|
||||
@advanced_ingest_source.setter
|
||||
def advanced_ingest_source(self, value: str):
|
||||
self._settings.setdefault("advanced", {})["ingest_source"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_window_split(self) -> str:
|
||||
return self._settings.get("advanced", {}).get("window_split", "none")
|
||||
|
||||
@advanced_window_split.setter
|
||||
def advanced_window_split(self, value: str):
|
||||
self._settings.setdefault("advanced", {})["window_split"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_window_compensation(self) -> int:
|
||||
return self._settings.get("advanced", {}).get("window_compensation", 0)
|
||||
|
||||
@advanced_window_compensation.setter
|
||||
def advanced_window_compensation(self, value: int):
|
||||
self._settings.setdefault("advanced", {})["window_compensation"] = value
|
||||
self._save()
|
||||
|
||||
def get_all_settings(self) -> Dict[str, Any]:
|
||||
"""获取所有设置(用于调试)"""
|
||||
return self._settings.copy()
|
||||
|
||||
def save_all(self):
|
||||
"""强制保存所有设置"""
|
||||
self._save()
|
||||
|
||||
# ==================== 窗口状态 ====================
|
||||
|
||||
@property
|
||||
def window_geometry(self) -> Optional[list]:
|
||||
"""获取窗口几何信息 [x, y, width, height]"""
|
||||
return self._settings.get("window_state", {}).get("geometry")
|
||||
|
||||
@window_geometry.setter
|
||||
def window_geometry(self, value: list):
|
||||
"""设置窗口几何信息"""
|
||||
self._settings.setdefault("window_state", {})["geometry"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def window_maximized(self) -> bool:
|
||||
"""获取窗口是否最大化"""
|
||||
return self._settings.get("window_state", {}).get("maximized", False)
|
||||
|
||||
@window_maximized.setter
|
||||
def window_maximized(self, value: bool):
|
||||
"""设置窗口是否最大化"""
|
||||
self._settings.setdefault("window_state", {})["maximized"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def current_panel(self) -> int:
|
||||
"""获取当前面板索引"""
|
||||
return self._settings.get("window_state", {}).get("current_panel", 0)
|
||||
|
||||
@current_panel.setter
|
||||
def current_panel(self, value: int):
|
||||
"""设置当前面板索引"""
|
||||
self._settings.setdefault("window_state", {})["current_panel"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def splitter_sizes(self) -> Optional[list]:
|
||||
"""获取分割器大小"""
|
||||
return self._settings.get("window_state", {}).get("splitter_sizes")
|
||||
|
||||
@splitter_sizes.setter
|
||||
def splitter_sizes(self, value: list):
|
||||
"""设置分割器大小"""
|
||||
self._settings.setdefault("window_state", {})["splitter_sizes"] = value
|
||||
self._save()
|
||||
|
||||
# ==================== 任务管理状态 ====================
|
||||
|
||||
@property
|
||||
def scheduler_enabled(self) -> bool:
|
||||
"""获取调度器是否启用"""
|
||||
return self._settings.get("task_manager_state", {}).get("scheduler_enabled", False)
|
||||
|
||||
@scheduler_enabled.setter
|
||||
def scheduler_enabled(self, value: bool):
|
||||
"""设置调度器是否启用"""
|
||||
self._settings.setdefault("task_manager_state", {})["scheduler_enabled"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def auto_run_enabled(self) -> bool:
|
||||
"""获取自动执行是否启用"""
|
||||
return self._settings.get("task_manager_state", {}).get("auto_run_enabled", False)
|
||||
|
||||
@auto_run_enabled.setter
|
||||
def auto_run_enabled(self, value: bool):
|
||||
"""设置自动执行是否启用"""
|
||||
self._settings.setdefault("task_manager_state", {})["auto_run_enabled"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def task_manager_tab(self) -> int:
|
||||
"""获取任务管理当前选项卡"""
|
||||
return self._settings.get("task_manager_state", {}).get("current_tab", 0)
|
||||
|
||||
@task_manager_tab.setter
|
||||
def task_manager_tab(self, value: int):
|
||||
"""设置任务管理当前选项卡"""
|
||||
self._settings.setdefault("task_manager_state", {})["current_tab"] = value
|
||||
self._save()
|
||||
|
||||
# ==================== 任务面板状态 ====================
|
||||
|
||||
@property
|
||||
def advanced_expanded(self) -> bool:
|
||||
"""获取高级选项是否展开"""
|
||||
return self._settings.get("task_panel_state", {}).get("advanced_expanded", False)
|
||||
|
||||
@advanced_expanded.setter
|
||||
def advanced_expanded(self, value: bool):
|
||||
"""设置高级选项是否展开"""
|
||||
self._settings.setdefault("task_panel_state", {})["advanced_expanded"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def task_panel_tab(self) -> int:
|
||||
"""获取任务面板当前选项卡"""
|
||||
return self._settings.get("task_panel_state", {}).get("current_tab", 0)
|
||||
|
||||
@task_panel_tab.setter
|
||||
def task_panel_tab(self, value: int):
|
||||
"""设置任务面板当前选项卡"""
|
||||
self._settings.setdefault("task_panel_state", {})["current_tab"] = value
|
||||
self._save()
|
||||
|
||||
# ==================== 统一任务配置状态 ====================
|
||||
|
||||
@property
|
||||
def unified_pipeline(self) -> str:
|
||||
"""获取管道类型"""
|
||||
return self._settings.get("task_panel_state", {}).get("pipeline", "api_ods_dwd")
|
||||
|
||||
@unified_pipeline.setter
|
||||
def unified_pipeline(self, value: str):
|
||||
"""设置管道类型"""
|
||||
self._settings.setdefault("task_panel_state", {})["pipeline"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def unified_processing_mode(self) -> str:
|
||||
"""获取处理模式"""
|
||||
return self._settings.get("task_panel_state", {}).get("processing_mode", "increment_only")
|
||||
|
||||
@unified_processing_mode.setter
|
||||
def unified_processing_mode(self, value: str):
|
||||
"""设置处理模式"""
|
||||
self._settings.setdefault("task_panel_state", {})["processing_mode"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def unified_fetch_before_verify(self) -> bool:
|
||||
"""获取校验前是否从 API 获取数据"""
|
||||
return self._settings.get("task_panel_state", {}).get("fetch_before_verify", False)
|
||||
|
||||
@unified_fetch_before_verify.setter
|
||||
def unified_fetch_before_verify(self, value: bool):
|
||||
"""设置校验前是否从 API 获取数据"""
|
||||
self._settings.setdefault("task_panel_state", {})["fetch_before_verify"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def unified_window_mode(self) -> str:
|
||||
"""获取时间窗口模式"""
|
||||
return self._settings.get("task_panel_state", {}).get("window_mode", "lookback")
|
||||
|
||||
@unified_window_mode.setter
|
||||
def unified_window_mode(self, value: str):
|
||||
"""设置时间窗口模式"""
|
||||
self._settings.setdefault("task_panel_state", {})["window_mode"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def unified_lookback_hours(self) -> int:
|
||||
"""获取回溯小时数"""
|
||||
return self._settings.get("task_panel_state", {}).get("lookback_hours", 24)
|
||||
|
||||
@unified_lookback_hours.setter
|
||||
def unified_lookback_hours(self, value: int):
|
||||
"""设置回溯小时数"""
|
||||
self._settings.setdefault("task_panel_state", {})["lookback_hours"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def unified_overlap_seconds(self) -> int:
|
||||
"""获取冗余秒数"""
|
||||
return self._settings.get("task_panel_state", {}).get("overlap_seconds", 600)
|
||||
|
||||
@unified_overlap_seconds.setter
|
||||
def unified_overlap_seconds(self, value: int):
|
||||
"""设置冗余秒数"""
|
||||
self._settings.setdefault("task_panel_state", {})["overlap_seconds"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def unified_window_split(self) -> str:
|
||||
"""获取窗口切分方式"""
|
||||
return self._settings.get("task_panel_state", {}).get("window_split", "day")
|
||||
|
||||
@unified_window_split.setter
|
||||
def unified_window_split(self, value: str):
|
||||
"""设置窗口切分方式"""
|
||||
self._settings.setdefault("task_panel_state", {})["window_split"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def unified_window_split_days(self) -> int:
|
||||
"""获取窗口切分天数(按天时生效)"""
|
||||
return self._settings.get("task_panel_state", {}).get("window_split_days", 10)
|
||||
|
||||
@unified_window_split_days.setter
|
||||
def unified_window_split_days(self, value: int):
|
||||
"""设置窗口切分天数(按天时生效)"""
|
||||
self._settings.setdefault("task_panel_state", {})["window_split_days"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def unified_ods_tasks(self) -> list:
|
||||
"""获取 ODS 任务选择"""
|
||||
return self._settings.get("task_panel_state", {}).get("ods_tasks", [])
|
||||
|
||||
@unified_ods_tasks.setter
|
||||
def unified_ods_tasks(self, value: list):
|
||||
"""设置 ODS 任务选择"""
|
||||
self._settings.setdefault("task_panel_state", {})["ods_tasks"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def unified_dws_tasks(self) -> list:
|
||||
"""获取 DWS 任务选择"""
|
||||
return self._settings.get("task_panel_state", {}).get("dws_tasks", [])
|
||||
|
||||
@unified_dws_tasks.setter
|
||||
def unified_dws_tasks(self, value: list):
|
||||
"""设置 DWS 任务选择"""
|
||||
self._settings.setdefault("task_panel_state", {})["dws_tasks"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def unified_dwd_tasks(self) -> list:
|
||||
"""获取 DWD 任务选择"""
|
||||
return self._settings.get("task_panel_state", {}).get("dwd_tasks", [])
|
||||
|
||||
@unified_dwd_tasks.setter
|
||||
def unified_dwd_tasks(self, value: list):
|
||||
"""设置 DWD 任务选择"""
|
||||
self._settings.setdefault("task_panel_state", {})["dwd_tasks"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def build_tasks(self) -> list:
|
||||
"""获取数据建设任务选择"""
|
||||
return self._settings.get("task_panel_state", {}).get("build_tasks", [])
|
||||
|
||||
@build_tasks.setter
|
||||
def build_tasks(self, value: list):
|
||||
"""设置数据建设任务选择"""
|
||||
self._settings.setdefault("task_panel_state", {})["build_tasks"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def build_window_mode(self) -> str:
|
||||
"""获取数据建设时间窗口模式"""
|
||||
return self._settings.get("task_panel_state", {}).get("build_window_mode", "lookback")
|
||||
|
||||
@build_window_mode.setter
|
||||
def build_window_mode(self, value: str):
|
||||
"""设置数据建设时间窗口模式"""
|
||||
self._settings.setdefault("task_panel_state", {})["build_window_mode"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def build_lookback_hours(self) -> int:
|
||||
"""获取数据建设回溯小时数"""
|
||||
return self._settings.get("task_panel_state", {}).get("build_lookback_hours", 24)
|
||||
|
||||
@build_lookback_hours.setter
|
||||
def build_lookback_hours(self, value: int):
|
||||
"""设置数据建设回溯小时数"""
|
||||
self._settings.setdefault("task_panel_state", {})["build_lookback_hours"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def build_window_start(self) -> str:
|
||||
"""获取数据建设窗口开始"""
|
||||
return self._settings.get("task_panel_state", {}).get("build_window_start", "")
|
||||
|
||||
@build_window_start.setter
|
||||
def build_window_start(self, value: str):
|
||||
"""设置数据建设窗口开始"""
|
||||
self._settings.setdefault("task_panel_state", {})["build_window_start"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def build_window_end(self) -> str:
|
||||
"""获取数据建设窗口结束"""
|
||||
return self._settings.get("task_panel_state", {}).get("build_window_end", "")
|
||||
|
||||
@build_window_end.setter
|
||||
def build_window_end(self, value: str):
|
||||
"""设置数据建设窗口结束"""
|
||||
self._settings.setdefault("task_panel_state", {})["build_window_end"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def build_window_split(self) -> str:
|
||||
"""获取数据建设窗口切分方式"""
|
||||
return self._settings.get("task_panel_state", {}).get("build_window_split", "day")
|
||||
|
||||
@build_window_split.setter
|
||||
def build_window_split(self, value: str):
|
||||
"""设置数据建设窗口切分方式"""
|
||||
self._settings.setdefault("task_panel_state", {})["build_window_split"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def build_window_split_days(self) -> int:
|
||||
"""获取数据建设窗口切分天数(按天时生效)"""
|
||||
return self._settings.get("task_panel_state", {}).get("build_window_split_days", 10)
|
||||
|
||||
@build_window_split_days.setter
|
||||
def build_window_split_days(self, value: int):
|
||||
"""设置数据建设窗口切分天数(按天时生效)"""
|
||||
self._settings.setdefault("task_panel_state", {})["build_window_split_days"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def index_recall_check(self) -> bool:
|
||||
"""获取召回指数复选框状态"""
|
||||
return self._settings.get("task_panel_state", {}).get("index_recall_check", False)
|
||||
|
||||
@index_recall_check.setter
|
||||
def index_recall_check(self, value: bool):
|
||||
"""设置召回指数复选框状态"""
|
||||
self._settings.setdefault("task_panel_state", {})["index_recall_check"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def index_winback_check(self) -> bool:
|
||||
"""获取老客挽回指数复选框状态"""
|
||||
return self._settings.get("task_panel_state", {}).get("index_winback_check", True)
|
||||
|
||||
@index_winback_check.setter
|
||||
def index_winback_check(self, value: bool):
|
||||
"""设置老客挽回指数复选框状态"""
|
||||
self._settings.setdefault("task_panel_state", {})["index_winback_check"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def index_newconv_check(self) -> bool:
|
||||
"""获取新客转化指数复选框状态"""
|
||||
return self._settings.get("task_panel_state", {}).get("index_newconv_check", True)
|
||||
|
||||
@index_newconv_check.setter
|
||||
def index_newconv_check(self, value: bool):
|
||||
"""设置新客转化指数复选框状态"""
|
||||
self._settings.setdefault("task_panel_state", {})["index_newconv_check"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def index_relation_check(self) -> bool:
|
||||
"""获取关系指数复选框状态"""
|
||||
return self._settings.get("task_panel_state", {}).get("index_relation_check", True)
|
||||
|
||||
@index_relation_check.setter
|
||||
def index_relation_check(self, value: bool):
|
||||
"""设置关系指数复选框状态"""
|
||||
self._settings.setdefault("task_panel_state", {})["index_relation_check"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def ml_manual_file_path(self) -> str:
|
||||
"""获取 ML 人工台账文件路径"""
|
||||
return self._settings.get("task_panel_state", {}).get("ml_manual_file_path", "")
|
||||
|
||||
@ml_manual_file_path.setter
|
||||
def ml_manual_file_path(self, value: str):
|
||||
"""设置 ML 人工台账文件路径"""
|
||||
self._settings.setdefault("task_panel_state", {})["ml_manual_file_path"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def index_lookback_days(self) -> int:
|
||||
"""获取指数回溯天数"""
|
||||
return self._settings.get("task_panel_state", {}).get("index_lookback_days", 60)
|
||||
|
||||
@index_lookback_days.setter
|
||||
def index_lookback_days(self, value: int):
|
||||
"""设置指数回溯天数"""
|
||||
self._settings.setdefault("task_panel_state", {})["index_lookback_days"] = value
|
||||
self._save()
|
||||
|
||||
# ==================== 任务历史存储 ====================
|
||||
|
||||
def _get_history_path(self) -> Path:
|
||||
"""获取任务历史文件路径"""
|
||||
return self._settings_file.parent / "task_history.json"
|
||||
|
||||
def save_task_history(self, history_list: list):
|
||||
"""保存任务历史到文件"""
|
||||
try:
|
||||
history_path = self._get_history_path()
|
||||
|
||||
# 序列化任务历史
|
||||
serialized = []
|
||||
for task in history_list[:100]: # 最多保存100条
|
||||
try:
|
||||
task_data = {
|
||||
"id": task.id,
|
||||
"tasks": task.config.tasks if hasattr(task, 'config') else [],
|
||||
"status": task.status.value if hasattr(task.status, 'value') else str(task.status),
|
||||
"created_at": task.created_at.isoformat() if task.created_at else None,
|
||||
"started_at": task.started_at.isoformat() if task.started_at else None,
|
||||
"finished_at": task.finished_at.isoformat() if task.finished_at else None,
|
||||
"exit_code": task.exit_code,
|
||||
"error": task.error[:500] if task.error else "", # 限制长度
|
||||
"output_preview": task.output[:1000] if task.output else "", # 输出预览
|
||||
# 保存配置信息
|
||||
"pipeline_flow": task.config.pipeline_flow if hasattr(task, 'config') else "FULL",
|
||||
"window_start": task.config.window_start if hasattr(task, 'config') else None,
|
||||
"window_end": task.config.window_end if hasattr(task, 'config') else None,
|
||||
}
|
||||
serialized.append(task_data)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
history_path.write_text(
|
||||
json.dumps(serialized, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8"
|
||||
)
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).warning("保存任务历史失败: %s", e)
|
||||
|
||||
def load_task_history(self) -> list:
|
||||
"""从文件加载任务历史"""
|
||||
try:
|
||||
history_path = self._get_history_path()
|
||||
if not history_path.exists():
|
||||
return []
|
||||
|
||||
data = json.loads(history_path.read_text(encoding="utf-8"))
|
||||
return data
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).warning("加载任务历史失败: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
# 全局单例
|
||||
app_settings = AppSettings()
|
||||
198
gui/utils/cli_builder.py
Normal file
198
gui/utils/cli_builder.py
Normal file
@@ -0,0 +1,198 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""CLI 命令构建器
|
||||
|
||||
支持两种模式:
|
||||
1. 传统模式:--tasks 参数指定任务列表
|
||||
2. 管道模式:--pipeline 参数指定管道类型,支持后置校验
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any, Optional
|
||||
from ..models.task_model import TaskConfig
|
||||
|
||||
|
||||
# CLI 支持的命令行参数(来自 cli/main.py)
|
||||
CLI_SUPPORTED_ARGS = {
|
||||
# 值类型参数
|
||||
"store_id", "tasks", "pg_dsn", "pg_host", "pg_port", "pg_name",
|
||||
"pg_user", "pg_password", "api_base", "api_token", "api_timeout",
|
||||
"api_page_size", "api_retry_max", "window_start", "window_end",
|
||||
"export_root", "log_root", "pipeline_flow", "fetch_root",
|
||||
"ingest_source", "idle_start", "idle_end",
|
||||
# 新增:管道模式参数
|
||||
"pipeline", "processing_mode", "window_split", "window_split_unit", "window_split_days",
|
||||
"lookback_hours", "overlap_seconds",
|
||||
# 布尔类型参数
|
||||
"dry_run", "force_window_override", "write_pretty_json", "allow_empty_advance",
|
||||
}
|
||||
|
||||
|
||||
class CLIBuilder:
|
||||
"""构建 CLI 命令行参数"""
|
||||
|
||||
def __init__(self, python_executable: str = "python"):
|
||||
self.python_executable = python_executable
|
||||
|
||||
def build_command(self, config: TaskConfig) -> List[str]:
|
||||
"""
|
||||
根据任务配置构建命令行参数列表
|
||||
|
||||
支持两种模式:
|
||||
1. 管道模式(优先):使用 --pipeline 参数
|
||||
2. 传统模式:使用 --tasks 参数
|
||||
|
||||
Args:
|
||||
config: 任务配置对象
|
||||
|
||||
Returns:
|
||||
命令行参数列表
|
||||
"""
|
||||
cmd = [self.python_executable, "-m", "cli.main"]
|
||||
|
||||
# 判断使用管道模式还是传统模式
|
||||
use_pipeline_mode = bool(config.pipeline and config.pipeline != "legacy")
|
||||
|
||||
if use_pipeline_mode:
|
||||
# 管道模式
|
||||
cmd.extend(["--pipeline", config.pipeline])
|
||||
|
||||
# 处理模式
|
||||
if config.processing_mode:
|
||||
cmd.extend(["--processing-mode", config.processing_mode])
|
||||
|
||||
# 校验前从 API 获取数据(仅 verify_only 模式有效)
|
||||
if config.fetch_before_verify and config.processing_mode == "verify_only":
|
||||
cmd.append("--fetch-before-verify")
|
||||
|
||||
# 时间窗口模式
|
||||
if config.window_mode == "lookback":
|
||||
# 回溯模式:使用 lookback_hours 和 overlap_seconds
|
||||
if config.lookback_hours:
|
||||
cmd.extend(["--lookback-hours", str(config.lookback_hours)])
|
||||
if config.overlap_seconds:
|
||||
cmd.extend(["--overlap-seconds", str(config.overlap_seconds)])
|
||||
else:
|
||||
# 自定义时间窗口
|
||||
if config.window_start:
|
||||
cmd.extend(["--window-start", config.window_start])
|
||||
if config.window_end:
|
||||
cmd.extend(["--window-end", config.window_end])
|
||||
|
||||
# 时间窗口切分(管道层拆分 + 任务层拆分)
|
||||
if config.window_split and config.window_split != "none":
|
||||
cmd.extend(["--window-split", config.window_split])
|
||||
cmd.extend(["--window-split-unit", config.window_split])
|
||||
if config.window_split_days:
|
||||
cmd.extend(["--window-split-days", str(config.window_split_days)])
|
||||
|
||||
# 如果同时指定了任务列表,也传递(用于过滤)
|
||||
if config.tasks:
|
||||
cmd.extend(["--tasks", ",".join(config.tasks)])
|
||||
else:
|
||||
# 传统模式
|
||||
if config.tasks:
|
||||
cmd.extend(["--tasks", ",".join(config.tasks)])
|
||||
|
||||
# Pipeline 流程
|
||||
if config.pipeline_flow:
|
||||
cmd.extend(["--pipeline-flow", config.pipeline_flow])
|
||||
|
||||
# 时间窗口
|
||||
if config.window_start:
|
||||
cmd.extend(["--window-start", config.window_start])
|
||||
if config.window_end:
|
||||
cmd.extend(["--window-end", config.window_end])
|
||||
|
||||
# 时间窗口切分(任务层拆分)
|
||||
if config.window_split and config.window_split != "none":
|
||||
cmd.extend(["--window-split-unit", config.window_split])
|
||||
if config.window_split_days:
|
||||
cmd.extend(["--window-split-days", str(config.window_split_days)])
|
||||
|
||||
# Dry-run 模式
|
||||
if config.dry_run:
|
||||
cmd.append("--dry-run")
|
||||
|
||||
# 数据源目录(传统模式)
|
||||
if config.ingest_source:
|
||||
cmd.extend(["--ingest-source", config.ingest_source])
|
||||
|
||||
# 门店 ID
|
||||
if config.store_id is not None:
|
||||
cmd.extend(["--store-id", str(config.store_id)])
|
||||
|
||||
# 数据库 DSN
|
||||
if config.pg_dsn:
|
||||
cmd.extend(["--pg-dsn", config.pg_dsn])
|
||||
|
||||
# API Token
|
||||
if config.api_token:
|
||||
cmd.extend(["--api-token", config.api_token])
|
||||
|
||||
# 额外参数(只传递 CLI 支持的参数)
|
||||
for key, value in config.extra_args.items():
|
||||
if value is not None and key in CLI_SUPPORTED_ARGS:
|
||||
arg_name = f"--{key.replace('_', '-')}"
|
||||
if isinstance(value, bool):
|
||||
if value:
|
||||
cmd.append(arg_name)
|
||||
else:
|
||||
cmd.extend([arg_name, str(value)])
|
||||
|
||||
return cmd
|
||||
|
||||
def build_command_string(self, config: TaskConfig) -> str:
|
||||
"""
|
||||
构建命令行字符串(用于显示)
|
||||
|
||||
Args:
|
||||
config: 任务配置对象
|
||||
|
||||
Returns:
|
||||
命令行字符串
|
||||
"""
|
||||
cmd = self.build_command(config)
|
||||
# 对包含空格的参数添加引号
|
||||
quoted_cmd = []
|
||||
for arg in cmd:
|
||||
if ' ' in arg or '"' in arg:
|
||||
quoted_cmd.append(f'"{arg}"')
|
||||
else:
|
||||
quoted_cmd.append(arg)
|
||||
return " ".join(quoted_cmd)
|
||||
|
||||
def build_from_dict(self, params: Dict[str, Any]) -> List[str]:
|
||||
"""
|
||||
从字典构建命令行参数
|
||||
|
||||
Args:
|
||||
params: 参数字典
|
||||
|
||||
Returns:
|
||||
命令行参数列表
|
||||
"""
|
||||
config = TaskConfig(
|
||||
tasks=params.get("tasks", []),
|
||||
pipeline_flow=params.get("pipeline_flow", "FULL"),
|
||||
dry_run=params.get("dry_run", False),
|
||||
window_start=params.get("window_start"),
|
||||
window_end=params.get("window_end"),
|
||||
window_split=params.get("window_split"),
|
||||
window_split_days=params.get("window_split_days"),
|
||||
ingest_source=params.get("ingest_source"),
|
||||
store_id=params.get("store_id"),
|
||||
pg_dsn=params.get("pg_dsn"),
|
||||
api_token=params.get("api_token"),
|
||||
extra_args=params.get("extra_args", {}),
|
||||
# 新增管道参数
|
||||
pipeline=params.get("pipeline", ""),
|
||||
processing_mode=params.get("processing_mode", "increment_only"),
|
||||
fetch_before_verify=params.get("fetch_before_verify", False),
|
||||
window_mode=params.get("window_mode", "lookback"),
|
||||
lookback_hours=params.get("lookback_hours", 24),
|
||||
overlap_seconds=params.get("overlap_seconds", 600),
|
||||
)
|
||||
return self.build_command(config)
|
||||
|
||||
|
||||
# 全局实例
|
||||
cli_builder = CLIBuilder()
|
||||
324
gui/utils/config_helper.py
Normal file
324
gui/utils/config_helper.py
Normal file
@@ -0,0 +1,324 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""配置辅助工具"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple, Optional, Any
|
||||
|
||||
|
||||
# 环境变量分组
|
||||
ENV_GROUPS = {
|
||||
"database": {
|
||||
"title": "数据库配置",
|
||||
"keys": ["PG_DSN", "PG_HOST", "PG_PORT", "PG_NAME", "PG_USER", "PG_PASSWORD", "PG_CONNECT_TIMEOUT"],
|
||||
"sensitive": ["PG_PASSWORD"],
|
||||
},
|
||||
"api": {
|
||||
"title": "API 配置",
|
||||
"keys": ["API_BASE", "API_TOKEN", "FICOO_TOKEN", "API_TIMEOUT", "API_PAGE_SIZE", "API_RETRY_MAX"],
|
||||
"sensitive": ["API_TOKEN", "FICOO_TOKEN"],
|
||||
},
|
||||
"store": {
|
||||
"title": "门店配置",
|
||||
"keys": ["STORE_ID", "TIMEZONE", "SCHEMA_OLTP", "SCHEMA_ETL"],
|
||||
"sensitive": [],
|
||||
},
|
||||
"paths": {
|
||||
"title": "路径配置",
|
||||
"keys": ["EXPORT_ROOT", "LOG_ROOT", "FETCH_ROOT", "INGEST_SOURCE_DIR", "JSON_FETCH_ROOT", "JSON_SOURCE_DIR"],
|
||||
"sensitive": [],
|
||||
},
|
||||
"pipeline": {
|
||||
"title": "流水线配置",
|
||||
"keys": ["PIPELINE_FLOW", "RUN_TASKS", "OVERLAP_SECONDS"],
|
||||
"sensitive": [],
|
||||
},
|
||||
"window": {
|
||||
"title": "时间窗口配置",
|
||||
"keys": [
|
||||
"WINDOW_START", "WINDOW_END", "WINDOW_BUSY_MIN", "WINDOW_IDLE_MIN",
|
||||
"WINDOW_SPLIT_UNIT", "WINDOW_SPLIT_DAYS",
|
||||
"IDLE_START", "IDLE_END",
|
||||
],
|
||||
"sensitive": [],
|
||||
},
|
||||
"integrity": {
|
||||
"title": "数据完整性配置",
|
||||
"keys": [
|
||||
"INTEGRITY_MODE",
|
||||
"INTEGRITY_HISTORY_START",
|
||||
"INTEGRITY_HISTORY_END",
|
||||
"INTEGRITY_INCLUDE_DIMENSIONS",
|
||||
"INTEGRITY_AUTO_CHECK",
|
||||
"INTEGRITY_AUTO_BACKFILL",
|
||||
"INTEGRITY_COMPARE_CONTENT",
|
||||
"INTEGRITY_CONTENT_SAMPLE_LIMIT",
|
||||
"INTEGRITY_BACKFILL_MISMATCH",
|
||||
"INTEGRITY_RECHECK_AFTER_BACKFILL",
|
||||
"INTEGRITY_ODS_TASK_CODES",
|
||||
],
|
||||
"sensitive": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ConfigHelper:
|
||||
"""配置文件辅助类"""
|
||||
|
||||
def __init__(self, env_path: Optional[Path] = None):
|
||||
"""
|
||||
初始化配置辅助器
|
||||
|
||||
Args:
|
||||
env_path: .env 文件路径,默认使用 AppSettings 中的路径
|
||||
"""
|
||||
if env_path is not None:
|
||||
self.env_path = Path(env_path)
|
||||
else:
|
||||
# 从 AppSettings 获取路径
|
||||
from .app_settings import app_settings
|
||||
settings_path = app_settings.env_file_path
|
||||
if settings_path:
|
||||
self.env_path = Path(settings_path)
|
||||
else:
|
||||
# 回退到源码目录
|
||||
self.env_path = Path(__file__).resolve().parents[2] / ".env"
|
||||
|
||||
def load_env(self) -> Dict[str, str]:
|
||||
"""
|
||||
加载 .env 文件内容
|
||||
|
||||
Returns:
|
||||
环境变量字典
|
||||
"""
|
||||
env_vars = {}
|
||||
if not self.env_path.exists():
|
||||
return env_vars
|
||||
|
||||
try:
|
||||
content = self.env_path.read_text(encoding="utf-8", errors="ignore")
|
||||
for line in content.splitlines():
|
||||
parsed = self._parse_line(line)
|
||||
if parsed:
|
||||
key, value = parsed
|
||||
env_vars[key] = value
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return env_vars
|
||||
|
||||
def save_env(self, env_vars: Dict[str, str]) -> bool:
|
||||
"""
|
||||
保存环境变量到 .env 文件
|
||||
|
||||
Args:
|
||||
env_vars: 环境变量字典
|
||||
|
||||
Returns:
|
||||
是否保存成功
|
||||
"""
|
||||
try:
|
||||
lines = []
|
||||
# 按分组输出
|
||||
written_keys = set()
|
||||
|
||||
for group_id, group_info in ENV_GROUPS.items():
|
||||
group_lines = []
|
||||
for key in group_info["keys"]:
|
||||
if key in env_vars:
|
||||
value = env_vars[key]
|
||||
group_lines.append(self._format_line(key, value))
|
||||
written_keys.add(key)
|
||||
|
||||
if group_lines:
|
||||
lines.append(f"\n# {group_info['title']}")
|
||||
lines.extend(group_lines)
|
||||
|
||||
# 写入未分组的变量
|
||||
other_lines = []
|
||||
for key, value in env_vars.items():
|
||||
if key not in written_keys:
|
||||
other_lines.append(self._format_line(key, value))
|
||||
|
||||
if other_lines:
|
||||
lines.append("\n# 其他配置")
|
||||
lines.extend(other_lines)
|
||||
|
||||
content = "\n".join(lines).strip() + "\n"
|
||||
self.env_path.write_text(content, encoding="utf-8")
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def get_grouped_env(self) -> Dict[str, List[Tuple[str, str, bool]]]:
|
||||
"""
|
||||
获取分组的环境变量
|
||||
|
||||
Returns:
|
||||
分组字典 {group_id: [(key, value, is_sensitive), ...]}
|
||||
"""
|
||||
env_vars = self.load_env()
|
||||
result = {}
|
||||
used_keys = set()
|
||||
|
||||
for group_id, group_info in ENV_GROUPS.items():
|
||||
items = []
|
||||
for key in group_info["keys"]:
|
||||
value = env_vars.get(key, "")
|
||||
is_sensitive = key in group_info.get("sensitive", [])
|
||||
items.append((key, value, is_sensitive))
|
||||
if key in env_vars:
|
||||
used_keys.add(key)
|
||||
result[group_id] = items
|
||||
|
||||
# 添加未分组的变量到 "other" 组
|
||||
other_items = []
|
||||
for key, value in env_vars.items():
|
||||
if key not in used_keys:
|
||||
other_items.append((key, value, False))
|
||||
if other_items:
|
||||
result["other"] = other_items
|
||||
|
||||
return result
|
||||
|
||||
def validate_env(self, env_vars: Dict[str, str]) -> List[str]:
|
||||
"""
|
||||
验证环境变量
|
||||
|
||||
Args:
|
||||
env_vars: 环境变量字典
|
||||
|
||||
Returns:
|
||||
错误消息列表
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# 验证 PG_DSN 格式
|
||||
pg_dsn = env_vars.get("PG_DSN", "")
|
||||
if pg_dsn and not pg_dsn.startswith("postgresql://"):
|
||||
errors.append("PG_DSN 应以 'postgresql://' 开头")
|
||||
|
||||
# 验证端口号
|
||||
pg_port = env_vars.get("PG_PORT", "")
|
||||
if pg_port:
|
||||
try:
|
||||
port = int(pg_port)
|
||||
if port < 1 or port > 65535:
|
||||
errors.append("PG_PORT 应在 1-65535 范围内")
|
||||
except ValueError:
|
||||
errors.append("PG_PORT 应为数字")
|
||||
|
||||
# 验证 STORE_ID
|
||||
store_id = env_vars.get("STORE_ID", "")
|
||||
if store_id:
|
||||
try:
|
||||
int(store_id)
|
||||
except ValueError:
|
||||
errors.append("STORE_ID 应为数字")
|
||||
|
||||
# 验证路径存在性(可选)
|
||||
for key in ["EXPORT_ROOT", "LOG_ROOT", "FETCH_ROOT"]:
|
||||
path = env_vars.get(key, "")
|
||||
if path and not os.path.isabs(path):
|
||||
errors.append(f"{key} 建议使用绝对路径")
|
||||
|
||||
return errors
|
||||
|
||||
def mask_sensitive(self, value: str, visible_chars: int = 4) -> str:
|
||||
"""
|
||||
脱敏敏感值
|
||||
|
||||
Args:
|
||||
value: 原始值
|
||||
visible_chars: 可见字符数
|
||||
|
||||
Returns:
|
||||
脱敏后的值
|
||||
"""
|
||||
if not value or len(value) <= visible_chars:
|
||||
return "*" * len(value) if value else ""
|
||||
return value[:visible_chars] + "*" * (len(value) - visible_chars)
|
||||
|
||||
def _parse_line(self, line: str) -> Optional[Tuple[str, str]]:
|
||||
"""解析 .env 文件的一行"""
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
return None
|
||||
if stripped.startswith("export "):
|
||||
stripped = stripped[7:].strip()
|
||||
if "=" not in stripped:
|
||||
return None
|
||||
|
||||
key, value = stripped.split("=", 1)
|
||||
key = key.strip()
|
||||
value = self._unquote_value(value)
|
||||
return key, value
|
||||
|
||||
def _unquote_value(self, value: str) -> str:
|
||||
"""处理引号和注释"""
|
||||
# 去除内联注释
|
||||
value = self._strip_inline_comment(value)
|
||||
value = value.rstrip(",").strip()
|
||||
|
||||
if not value:
|
||||
return value
|
||||
|
||||
# 去除引号
|
||||
if len(value) >= 2 and value[0] in ("'", '"') and value[-1] == value[0]:
|
||||
return value[1:-1]
|
||||
if len(value) >= 3 and value[0] in ("r", "R") and value[1] in ("'", '"') and value[-1] == value[1]:
|
||||
return value[2:-1]
|
||||
|
||||
return value
|
||||
|
||||
def _strip_inline_comment(self, value: str) -> str:
|
||||
"""去除内联注释"""
|
||||
result = []
|
||||
in_quote = False
|
||||
quote_char = ""
|
||||
escape = False
|
||||
|
||||
for ch in value:
|
||||
if escape:
|
||||
result.append(ch)
|
||||
escape = False
|
||||
continue
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
result.append(ch)
|
||||
continue
|
||||
if ch in ("'", '"'):
|
||||
if not in_quote:
|
||||
in_quote = True
|
||||
quote_char = ch
|
||||
elif quote_char == ch:
|
||||
in_quote = False
|
||||
quote_char = ""
|
||||
result.append(ch)
|
||||
continue
|
||||
if ch == "#" and not in_quote:
|
||||
break
|
||||
result.append(ch)
|
||||
|
||||
return "".join(result).rstrip()
|
||||
|
||||
def _format_line(self, key: str, value: str) -> str:
|
||||
"""格式化为 .env 行"""
|
||||
# 如果值包含特殊字符,使用引号包裹
|
||||
if any(c in value for c in [' ', '"', "'", '#', '\n', '\r']):
|
||||
# 使用双引号,转义内部的双引号
|
||||
escaped = value.replace('\\', '\\\\').replace('"', '\\"')
|
||||
return f'{key}="{escaped}"'
|
||||
return f"{key}={value}"
|
||||
|
||||
@staticmethod
|
||||
def get_group_title(group_id: str) -> str:
|
||||
"""获取分组标题"""
|
||||
if group_id in ENV_GROUPS:
|
||||
return ENV_GROUPS[group_id]["title"]
|
||||
return "其他配置"
|
||||
|
||||
|
||||
# 全局实例
|
||||
config_helper = ConfigHelper()
|
||||
21
gui/widgets/__init__.py
Normal file
21
gui/widgets/__init__.py
Normal file
@@ -0,0 +1,21 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""GUI 组件模块"""
|
||||
|
||||
from .task_panel import TaskPanel
|
||||
from .env_editor import EnvEditor
|
||||
from .log_viewer import LogViewer
|
||||
from .db_viewer import DBViewer
|
||||
from .status_panel import StatusPanel
|
||||
from .task_manager import TaskManager
|
||||
from .task_selector import TaskSelectorWidget, CompactTaskSelector
|
||||
|
||||
__all__ = [
|
||||
"TaskPanel",
|
||||
"EnvEditor",
|
||||
"LogViewer",
|
||||
"DBViewer",
|
||||
"StatusPanel",
|
||||
"TaskManager",
|
||||
"TaskSelectorWidget",
|
||||
"CompactTaskSelector",
|
||||
]
|
||||
390
gui/widgets/db_viewer.py
Normal file
390
gui/widgets/db_viewer.py
Normal file
@@ -0,0 +1,390 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""数据库查看器"""
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QWidget, QVBoxLayout, QHBoxLayout, QSplitter,
|
||||
QGroupBox, QLabel, QPushButton, QLineEdit, QPlainTextEdit,
|
||||
QTableWidget, QTableWidgetItem, QTreeWidget, QTreeWidgetItem,
|
||||
QHeaderView, QComboBox, QTabWidget, QMessageBox, QFrame
|
||||
)
|
||||
from PySide6.QtCore import Qt, Signal
|
||||
from PySide6.QtGui import QFont
|
||||
|
||||
from ..workers.db_worker import DBWorker
|
||||
from ..utils.config_helper import ConfigHelper
|
||||
|
||||
|
||||
# 常用查询模板
|
||||
QUERY_TEMPLATES = {
|
||||
"ODS 行数统计": """
|
||||
SELECT
|
||||
table_name,
|
||||
(xpath('/row/cnt/text()',
|
||||
query_to_xml('SELECT COUNT(*) AS cnt FROM ' || table_schema || '.' || table_name, false, false, ''))
|
||||
)[1]::text::bigint AS row_count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'billiards_ods'
|
||||
ORDER BY table_name;
|
||||
""",
|
||||
"DWD 行数统计": """
|
||||
SELECT
|
||||
table_name,
|
||||
(xpath('/row/cnt/text()',
|
||||
query_to_xml('SELECT COUNT(*) AS cnt FROM ' || table_schema || '.' || table_name, false, false, ''))
|
||||
)[1]::text::bigint AS row_count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'billiards_dwd'
|
||||
ORDER BY table_name;
|
||||
""",
|
||||
"ETL 游标状态": """
|
||||
SELECT
|
||||
task_code,
|
||||
last_start,
|
||||
last_end,
|
||||
last_run_id,
|
||||
updated_at
|
||||
FROM etl_admin.etl_cursor
|
||||
ORDER BY task_code;
|
||||
""",
|
||||
"最近运行记录": """
|
||||
SELECT
|
||||
run_id,
|
||||
task_code,
|
||||
status,
|
||||
started_at,
|
||||
finished_at,
|
||||
EXTRACT(EPOCH FROM (finished_at - started_at))::int AS duration_sec,
|
||||
rows_affected
|
||||
FROM etl_admin.run_tracker
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 50;
|
||||
""",
|
||||
"ODS 最新入库时间": """
|
||||
SELECT
|
||||
'payment_transactions' AS table_name, MAX(fetched_at) AS max_fetched_at FROM billiards_ods.payment_transactions
|
||||
UNION ALL
|
||||
SELECT 'member_profiles', MAX(fetched_at) FROM billiards_ods.member_profiles
|
||||
UNION ALL
|
||||
SELECT 'settlement_records', MAX(fetched_at) FROM billiards_ods.settlement_records
|
||||
UNION ALL
|
||||
SELECT 'recharge_settlements', MAX(fetched_at) FROM billiards_ods.recharge_settlements
|
||||
ORDER BY table_name;
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
class DBViewer(QWidget):
|
||||
"""数据库查看器"""
|
||||
|
||||
# 信号
|
||||
connection_changed = Signal(bool, str) # 连接状态变化
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.config_helper = ConfigHelper()
|
||||
self.db_worker = DBWorker(self)
|
||||
self._connected = False
|
||||
|
||||
self._init_ui()
|
||||
self._connect_signals()
|
||||
self._load_dsn_from_env()
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(16, 16, 16, 16)
|
||||
layout.setSpacing(16)
|
||||
|
||||
# 标题
|
||||
title = QLabel("数据库查看器")
|
||||
title.setProperty("heading", True)
|
||||
layout.addWidget(title)
|
||||
|
||||
# 连接配置
|
||||
conn_group = QGroupBox("数据库连接")
|
||||
conn_layout = QHBoxLayout(conn_group)
|
||||
|
||||
conn_layout.addWidget(QLabel("DSN:"))
|
||||
self.dsn_edit = QLineEdit()
|
||||
self.dsn_edit.setPlaceholderText("postgresql://user:password@host:5432/dbname")
|
||||
self.dsn_edit.setEchoMode(QLineEdit.Password)
|
||||
conn_layout.addWidget(self.dsn_edit, 1)
|
||||
|
||||
self.show_dsn_btn = QPushButton("显示")
|
||||
self.show_dsn_btn.setProperty("secondary", True)
|
||||
self.show_dsn_btn.setCheckable(True)
|
||||
self.show_dsn_btn.setFixedWidth(60)
|
||||
conn_layout.addWidget(self.show_dsn_btn)
|
||||
|
||||
self.connect_btn = QPushButton("连接")
|
||||
self.connect_btn.setFixedWidth(80)
|
||||
conn_layout.addWidget(self.connect_btn)
|
||||
|
||||
self.disconnect_btn = QPushButton("断开")
|
||||
self.disconnect_btn.setProperty("secondary", True)
|
||||
self.disconnect_btn.setFixedWidth(80)
|
||||
self.disconnect_btn.setEnabled(False)
|
||||
conn_layout.addWidget(self.disconnect_btn)
|
||||
|
||||
layout.addWidget(conn_group)
|
||||
|
||||
# 主分割器
|
||||
main_splitter = QSplitter(Qt.Horizontal)
|
||||
layout.addWidget(main_splitter, 1)
|
||||
|
||||
# 左侧:表浏览器
|
||||
left_widget = self._create_table_browser()
|
||||
main_splitter.addWidget(left_widget)
|
||||
|
||||
# 右侧:查询和结果
|
||||
right_widget = self._create_query_area()
|
||||
main_splitter.addWidget(right_widget)
|
||||
|
||||
# 设置分割比例
|
||||
main_splitter.setSizes([300, 700])
|
||||
|
||||
def _create_table_browser(self) -> QWidget:
|
||||
"""创建表浏览器"""
|
||||
widget = QWidget()
|
||||
layout = QVBoxLayout(widget)
|
||||
layout.setContentsMargins(0, 0, 8, 0)
|
||||
|
||||
# 标题和刷新按钮
|
||||
header_layout = QHBoxLayout()
|
||||
header_layout.addWidget(QLabel("表结构"))
|
||||
self.refresh_tables_btn = QPushButton("刷新")
|
||||
self.refresh_tables_btn.setProperty("secondary", True)
|
||||
self.refresh_tables_btn.setEnabled(False)
|
||||
header_layout.addWidget(self.refresh_tables_btn)
|
||||
layout.addLayout(header_layout)
|
||||
|
||||
# 表树形视图
|
||||
self.table_tree = QTreeWidget()
|
||||
self.table_tree.setHeaderLabels(["名称", "行数", "最后更新"])
|
||||
self.table_tree.header().setSectionResizeMode(0, QHeaderView.Stretch)
|
||||
self.table_tree.setColumnWidth(1, 80)
|
||||
self.table_tree.setColumnWidth(2, 130)
|
||||
layout.addWidget(self.table_tree, 1)
|
||||
|
||||
return widget
|
||||
|
||||
def _create_query_area(self) -> QWidget:
|
||||
"""创建查询区域"""
|
||||
widget = QWidget()
|
||||
layout = QVBoxLayout(widget)
|
||||
layout.setContentsMargins(8, 0, 0, 0)
|
||||
|
||||
# 查询输入区
|
||||
query_group = QGroupBox("SQL 查询")
|
||||
query_layout = QVBoxLayout(query_group)
|
||||
|
||||
# 模板选择
|
||||
template_layout = QHBoxLayout()
|
||||
template_layout.addWidget(QLabel("常用查询:"))
|
||||
self.template_combo = QComboBox()
|
||||
self.template_combo.addItem("-- 选择模板 --")
|
||||
for name in QUERY_TEMPLATES.keys():
|
||||
self.template_combo.addItem(name)
|
||||
template_layout.addWidget(self.template_combo, 1)
|
||||
query_layout.addLayout(template_layout)
|
||||
|
||||
# SQL 编辑器
|
||||
self.sql_editor = QPlainTextEdit()
|
||||
self.sql_editor.setObjectName("sqlEditor")
|
||||
self.sql_editor.setPlaceholderText("输入 SQL 查询语句...")
|
||||
self.sql_editor.setFont(QFont("Consolas", 11))
|
||||
self.sql_editor.setMaximumHeight(150)
|
||||
query_layout.addWidget(self.sql_editor)
|
||||
|
||||
# 执行按钮
|
||||
exec_layout = QHBoxLayout()
|
||||
exec_layout.addStretch()
|
||||
|
||||
self.exec_btn = QPushButton("执行查询 (Ctrl+Enter)")
|
||||
self.exec_btn.setEnabled(False)
|
||||
exec_layout.addWidget(self.exec_btn)
|
||||
|
||||
query_layout.addLayout(exec_layout)
|
||||
layout.addWidget(query_group)
|
||||
|
||||
# 结果区域
|
||||
result_group = QGroupBox("查询结果")
|
||||
result_layout = QVBoxLayout(result_group)
|
||||
|
||||
# 结果表格
|
||||
self.result_table = QTableWidget()
|
||||
self.result_table.setAlternatingRowColors(True)
|
||||
self.result_table.horizontalHeader().setStretchLastSection(True)
|
||||
result_layout.addWidget(self.result_table, 1)
|
||||
|
||||
# 结果统计
|
||||
self.result_label = QLabel("就绪")
|
||||
self.result_label.setProperty("subheading", True)
|
||||
result_layout.addWidget(self.result_label)
|
||||
|
||||
layout.addWidget(result_group, 1)
|
||||
|
||||
return widget
|
||||
|
||||
def _connect_signals(self):
|
||||
"""连接信号"""
|
||||
# 连接按钮
|
||||
self.show_dsn_btn.toggled.connect(self._toggle_dsn_visibility)
|
||||
self.connect_btn.clicked.connect(self._connect_db)
|
||||
self.disconnect_btn.clicked.connect(self._disconnect_db)
|
||||
self.refresh_tables_btn.clicked.connect(self._refresh_tables)
|
||||
|
||||
# 模板选择
|
||||
self.template_combo.currentIndexChanged.connect(self._on_template_selected)
|
||||
|
||||
# 执行查询
|
||||
self.exec_btn.clicked.connect(self._execute_query)
|
||||
|
||||
# 表双击
|
||||
self.table_tree.itemDoubleClicked.connect(self._on_table_double_clicked)
|
||||
|
||||
# 工作线程信号
|
||||
self.db_worker.connection_status.connect(self._on_connection_status)
|
||||
self.db_worker.tables_loaded.connect(self._on_tables_loaded)
|
||||
self.db_worker.query_finished.connect(self._on_query_finished)
|
||||
self.db_worker.query_error.connect(self._on_query_error)
|
||||
|
||||
def _load_dsn_from_env(self):
|
||||
"""从环境变量加载 DSN"""
|
||||
env_vars = self.config_helper.load_env()
|
||||
dsn = env_vars.get("PG_DSN", "")
|
||||
if dsn:
|
||||
self.dsn_edit.setText(dsn)
|
||||
|
||||
def _toggle_dsn_visibility(self, checked: bool):
|
||||
"""切换 DSN 可见性"""
|
||||
self.dsn_edit.setEchoMode(
|
||||
QLineEdit.Normal if checked else QLineEdit.Password
|
||||
)
|
||||
self.show_dsn_btn.setText("隐藏" if checked else "显示")
|
||||
|
||||
def _connect_db(self):
|
||||
"""连接数据库"""
|
||||
dsn = self.dsn_edit.text().strip()
|
||||
if not dsn:
|
||||
QMessageBox.warning(self, "提示", "请输入数据库连接字符串")
|
||||
return
|
||||
|
||||
self.connect_btn.setEnabled(False)
|
||||
self.connect_btn.setText("连接中...")
|
||||
self.db_worker.connect_db(dsn)
|
||||
|
||||
def _disconnect_db(self):
|
||||
"""断开数据库连接"""
|
||||
self.db_worker.disconnect_db()
|
||||
|
||||
def _refresh_tables(self):
|
||||
"""刷新表列表"""
|
||||
self.db_worker.load_tables()
|
||||
|
||||
def _on_connection_status(self, connected: bool, message: str):
|
||||
"""处理连接状态变化"""
|
||||
self._connected = connected
|
||||
self.connect_btn.setEnabled(not connected)
|
||||
self.connect_btn.setText("连接")
|
||||
self.disconnect_btn.setEnabled(connected)
|
||||
self.refresh_tables_btn.setEnabled(connected)
|
||||
self.exec_btn.setEnabled(connected)
|
||||
|
||||
self.connection_changed.emit(connected, message)
|
||||
|
||||
if connected:
|
||||
# 自动加载表列表
|
||||
self._refresh_tables()
|
||||
|
||||
def _on_tables_loaded(self, tables_dict: dict):
|
||||
"""处理表列表加载完成"""
|
||||
self.table_tree.clear()
|
||||
|
||||
for schema, tables in tables_dict.items():
|
||||
schema_item = QTreeWidgetItem([schema, "", ""])
|
||||
schema_item.setExpanded(True)
|
||||
|
||||
for table_name, row_count, updated_at in tables:
|
||||
table_item = QTreeWidgetItem([table_name, str(row_count), updated_at])
|
||||
table_item.setData(0, Qt.UserRole, f"{schema}.{table_name}")
|
||||
schema_item.addChild(table_item)
|
||||
|
||||
self.table_tree.addTopLevelItem(schema_item)
|
||||
|
||||
def _on_template_selected(self, index: int):
|
||||
"""模板选择变化"""
|
||||
if index <= 0:
|
||||
return
|
||||
|
||||
template_name = self.template_combo.currentText()
|
||||
if template_name in QUERY_TEMPLATES:
|
||||
self.sql_editor.setPlainText(QUERY_TEMPLATES[template_name].strip())
|
||||
|
||||
# 重置选择
|
||||
self.template_combo.setCurrentIndex(0)
|
||||
|
||||
def _on_table_double_clicked(self, item: QTreeWidgetItem, column: int):
|
||||
"""表双击事件"""
|
||||
full_name = item.data(0, Qt.UserRole)
|
||||
if full_name:
|
||||
# 生成预览查询
|
||||
sql = f"SELECT * FROM {full_name} LIMIT 100;"
|
||||
self.sql_editor.setPlainText(sql)
|
||||
self._execute_query()
|
||||
|
||||
def _execute_query(self):
|
||||
"""执行查询"""
|
||||
sql = self.sql_editor.toPlainText().strip()
|
||||
if not sql:
|
||||
QMessageBox.warning(self, "提示", "请输入 SQL 语句")
|
||||
return
|
||||
|
||||
self.exec_btn.setEnabled(False)
|
||||
self.exec_btn.setText("执行中...")
|
||||
self.result_label.setText("正在查询...")
|
||||
|
||||
self.db_worker.execute_query(sql)
|
||||
|
||||
def _on_query_finished(self, columns: list, rows: list):
|
||||
"""查询完成"""
|
||||
self.exec_btn.setEnabled(True)
|
||||
self.exec_btn.setText("执行查询 (Ctrl+Enter)")
|
||||
|
||||
# 更新结果表格
|
||||
self.result_table.clear()
|
||||
self.result_table.setColumnCount(len(columns))
|
||||
self.result_table.setRowCount(len(rows))
|
||||
self.result_table.setHorizontalHeaderLabels(columns)
|
||||
|
||||
for row_idx, row_data in enumerate(rows):
|
||||
for col_idx, col_name in enumerate(columns):
|
||||
value = row_data.get(col_name, "")
|
||||
item = QTableWidgetItem(str(value) if value is not None else "NULL")
|
||||
if value is None:
|
||||
item.setForeground(Qt.gray)
|
||||
self.result_table.setItem(row_idx, col_idx, item)
|
||||
|
||||
# 更新统计
|
||||
self.result_label.setText(f"返回 {len(rows)} 行, {len(columns)} 列")
|
||||
|
||||
def _on_query_error(self, error: str):
|
||||
"""查询错误"""
|
||||
self.exec_btn.setEnabled(True)
|
||||
self.exec_btn.setText("执行查询 (Ctrl+Enter)")
|
||||
self.result_label.setText(f"错误: {error}")
|
||||
QMessageBox.critical(self, "查询错误", error)
|
||||
|
||||
def close_connection(self):
|
||||
"""关闭连接"""
|
||||
if self._connected:
|
||||
self.db_worker.disconnect_db()
|
||||
|
||||
def keyPressEvent(self, event):
|
||||
"""键盘事件"""
|
||||
# Ctrl+Enter 执行查询
|
||||
if event.modifiers() == Qt.ControlModifier and event.key() == Qt.Key_Return:
|
||||
if self._connected:
|
||||
self._execute_query()
|
||||
else:
|
||||
super().keyPressEvent(event)
|
||||
318
gui/widgets/env_editor.py
Normal file
318
gui/widgets/env_editor.py
Normal file
@@ -0,0 +1,318 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""环境变量编辑器"""
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||||
QGroupBox, QLabel, QLineEdit, QPushButton, QScrollArea,
|
||||
QFrame, QMessageBox, QFileDialog, QCheckBox
|
||||
)
|
||||
from PySide6.QtCore import Qt, Signal
|
||||
|
||||
from ..utils.config_helper import ConfigHelper, ENV_GROUPS
|
||||
|
||||
|
||||
class EnvEditor(QWidget):
|
||||
"""环境变量编辑器"""
|
||||
|
||||
# 信号
|
||||
config_saved = Signal() # 配置保存成功
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.config_helper = ConfigHelper()
|
||||
self.field_widgets = {} # 存储字段控件
|
||||
self.show_sensitive = False
|
||||
|
||||
self._init_ui()
|
||||
self.load_config()
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(16, 16, 16, 16)
|
||||
layout.setSpacing(16)
|
||||
|
||||
# 标题和按钮
|
||||
header_layout = QHBoxLayout()
|
||||
|
||||
title = QLabel("环境配置")
|
||||
title.setProperty("heading", True)
|
||||
header_layout.addWidget(title)
|
||||
|
||||
header_layout.addStretch()
|
||||
|
||||
self.show_sensitive_check = QCheckBox("显示敏感信息")
|
||||
self.show_sensitive_check.stateChanged.connect(self._toggle_sensitive)
|
||||
header_layout.addWidget(self.show_sensitive_check)
|
||||
|
||||
self.import_btn = QPushButton("导入")
|
||||
self.import_btn.setProperty("secondary", True)
|
||||
self.import_btn.clicked.connect(self._import_config)
|
||||
header_layout.addWidget(self.import_btn)
|
||||
|
||||
self.export_btn = QPushButton("导出")
|
||||
self.export_btn.setProperty("secondary", True)
|
||||
self.export_btn.clicked.connect(self._export_config)
|
||||
header_layout.addWidget(self.export_btn)
|
||||
|
||||
self.reload_btn = QPushButton("重新加载")
|
||||
self.reload_btn.setProperty("secondary", True)
|
||||
self.reload_btn.clicked.connect(self.load_config)
|
||||
header_layout.addWidget(self.reload_btn)
|
||||
|
||||
self.save_btn = QPushButton("保存")
|
||||
self.save_btn.clicked.connect(self._save_config)
|
||||
header_layout.addWidget(self.save_btn)
|
||||
|
||||
layout.addLayout(header_layout)
|
||||
|
||||
# 配置文件路径
|
||||
path_layout = QHBoxLayout()
|
||||
path_layout.addWidget(QLabel("配置文件:"))
|
||||
self.path_label = QLabel(str(self.config_helper.env_path))
|
||||
self.path_label.setProperty("subheading", True)
|
||||
path_layout.addWidget(self.path_label, 1)
|
||||
layout.addLayout(path_layout)
|
||||
|
||||
# 滚动区域
|
||||
scroll_area = QScrollArea()
|
||||
scroll_area.setWidgetResizable(True)
|
||||
scroll_area.setFrameShape(QFrame.NoFrame)
|
||||
layout.addWidget(scroll_area, 1)
|
||||
|
||||
# 配置组容器
|
||||
config_widget = QWidget()
|
||||
self.config_layout = QVBoxLayout(config_widget)
|
||||
self.config_layout.setSpacing(16)
|
||||
|
||||
# 创建各配置组
|
||||
self._create_config_groups()
|
||||
|
||||
# 弹性空间
|
||||
self.config_layout.addStretch()
|
||||
|
||||
scroll_area.setWidget(config_widget)
|
||||
|
||||
# 验证结果
|
||||
self.validation_label = QLabel()
|
||||
self.validation_label.setWordWrap(True)
|
||||
layout.addWidget(self.validation_label)
|
||||
|
||||
def _create_config_groups(self):
|
||||
"""创建配置分组"""
|
||||
for group_id, group_info in ENV_GROUPS.items():
|
||||
group = QGroupBox(group_info["title"])
|
||||
grid_layout = QGridLayout(group)
|
||||
|
||||
for row, key in enumerate(group_info["keys"]):
|
||||
# 标签
|
||||
label = QLabel(f"{key}:")
|
||||
label.setMinimumWidth(180)
|
||||
grid_layout.addWidget(label, row, 0)
|
||||
|
||||
# 输入框
|
||||
edit = QLineEdit()
|
||||
edit.setPlaceholderText(self._get_placeholder(key))
|
||||
|
||||
# 敏感字段处理
|
||||
if key in group_info.get("sensitive", []):
|
||||
edit.setEchoMode(QLineEdit.Password)
|
||||
edit.setProperty("sensitive", True)
|
||||
|
||||
edit.textChanged.connect(self._on_value_changed)
|
||||
grid_layout.addWidget(edit, row, 1)
|
||||
|
||||
# 存储控件引用
|
||||
self.field_widgets[key] = edit
|
||||
|
||||
self.config_layout.addWidget(group)
|
||||
|
||||
# 其他配置组(动态添加)
|
||||
self.other_group = QGroupBox("其他配置")
|
||||
self.other_layout = QGridLayout(self.other_group)
|
||||
self.other_group.setVisible(False)
|
||||
self.config_layout.addWidget(self.other_group)
|
||||
|
||||
def load_config(self):
|
||||
"""加载配置"""
|
||||
env_vars = self.config_helper.load_env()
|
||||
|
||||
# 更新已知字段
|
||||
for key, edit in self.field_widgets.items():
|
||||
value = env_vars.get(key, "")
|
||||
edit.blockSignals(True)
|
||||
edit.setText(value)
|
||||
edit.blockSignals(False)
|
||||
|
||||
# 处理其他字段
|
||||
known_keys = set(self.field_widgets.keys())
|
||||
other_keys = [k for k in env_vars.keys() if k not in known_keys]
|
||||
|
||||
# 清除旧的其他字段
|
||||
while self.other_layout.count():
|
||||
item = self.other_layout.takeAt(0)
|
||||
if item.widget():
|
||||
item.widget().deleteLater()
|
||||
|
||||
# 添加其他字段
|
||||
if other_keys:
|
||||
self.other_group.setVisible(True)
|
||||
for row, key in enumerate(sorted(other_keys)):
|
||||
label = QLabel(f"{key}:")
|
||||
self.other_layout.addWidget(label, row, 0)
|
||||
|
||||
edit = QLineEdit(env_vars[key])
|
||||
edit.textChanged.connect(self._on_value_changed)
|
||||
self.other_layout.addWidget(edit, row, 1)
|
||||
|
||||
self.field_widgets[key] = edit
|
||||
else:
|
||||
self.other_group.setVisible(False)
|
||||
|
||||
self._validate()
|
||||
|
||||
def _save_config(self):
|
||||
"""保存配置"""
|
||||
# 收集所有值
|
||||
env_vars = {}
|
||||
for key, edit in self.field_widgets.items():
|
||||
value = edit.text().strip()
|
||||
if value:
|
||||
env_vars[key] = value
|
||||
|
||||
# 验证
|
||||
errors = self.config_helper.validate_env(env_vars)
|
||||
if errors:
|
||||
reply = QMessageBox.question(
|
||||
self,
|
||||
"验证警告",
|
||||
"配置存在以下问题:\n\n" + "\n".join(f"• {e}" for e in errors) + "\n\n是否仍要保存?",
|
||||
QMessageBox.Yes | QMessageBox.No,
|
||||
QMessageBox.No
|
||||
)
|
||||
if reply == QMessageBox.No:
|
||||
return
|
||||
|
||||
# 保存
|
||||
if self.config_helper.save_env(env_vars):
|
||||
QMessageBox.information(self, "成功", "配置已保存")
|
||||
self.config_saved.emit()
|
||||
else:
|
||||
QMessageBox.critical(self, "错误", "保存配置失败")
|
||||
|
||||
def _import_config(self):
|
||||
"""导入配置"""
|
||||
file_path, _ = QFileDialog.getOpenFileName(
|
||||
self,
|
||||
"导入配置文件",
|
||||
"",
|
||||
"环境文件 (*.env);;所有文件 (*.*)"
|
||||
)
|
||||
if not file_path:
|
||||
return
|
||||
|
||||
try:
|
||||
from pathlib import Path
|
||||
temp_helper = ConfigHelper(Path(file_path))
|
||||
env_vars = temp_helper.load_env()
|
||||
|
||||
# 更新字段
|
||||
for key, value in env_vars.items():
|
||||
if key in self.field_widgets:
|
||||
self.field_widgets[key].setText(value)
|
||||
|
||||
QMessageBox.information(self, "成功", f"已导入 {len(env_vars)} 个配置项")
|
||||
except Exception as e:
|
||||
QMessageBox.critical(self, "错误", f"导入失败: {e}")
|
||||
|
||||
def _export_config(self):
|
||||
"""导出配置"""
|
||||
file_path, _ = QFileDialog.getSaveFileName(
|
||||
self,
|
||||
"导出配置文件",
|
||||
".env.backup",
|
||||
"环境文件 (*.env);;所有文件 (*.*)"
|
||||
)
|
||||
if not file_path:
|
||||
return
|
||||
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
# 收集当前值
|
||||
env_vars = {}
|
||||
for key, edit in self.field_widgets.items():
|
||||
value = edit.text().strip()
|
||||
if value:
|
||||
env_vars[key] = value
|
||||
|
||||
# 保存到指定路径
|
||||
temp_helper = ConfigHelper(Path(file_path))
|
||||
if temp_helper.save_env(env_vars):
|
||||
QMessageBox.information(self, "成功", f"配置已导出到:\n{file_path}")
|
||||
else:
|
||||
QMessageBox.critical(self, "错误", "导出失败")
|
||||
except Exception as e:
|
||||
QMessageBox.critical(self, "错误", f"导出失败: {e}")
|
||||
|
||||
def _toggle_sensitive(self, state: int):
|
||||
"""切换敏感信息显示"""
|
||||
self.show_sensitive = state == Qt.Checked
|
||||
|
||||
for key, edit in self.field_widgets.items():
|
||||
if edit.property("sensitive"):
|
||||
edit.setEchoMode(
|
||||
QLineEdit.Normal if self.show_sensitive else QLineEdit.Password
|
||||
)
|
||||
|
||||
def _on_value_changed(self):
|
||||
"""值变化时验证"""
|
||||
self._validate()
|
||||
|
||||
def _validate(self):
|
||||
"""验证配置"""
|
||||
env_vars = {}
|
||||
for key, edit in self.field_widgets.items():
|
||||
value = edit.text().strip()
|
||||
if value:
|
||||
env_vars[key] = value
|
||||
|
||||
errors = self.config_helper.validate_env(env_vars)
|
||||
|
||||
if errors:
|
||||
self.validation_label.setText("⚠ " + "; ".join(errors))
|
||||
self.validation_label.setProperty("status", "warning")
|
||||
else:
|
||||
self.validation_label.setText("✓ 配置验证通过")
|
||||
self.validation_label.setProperty("status", "success")
|
||||
|
||||
self.validation_label.style().unpolish(self.validation_label)
|
||||
self.validation_label.style().polish(self.validation_label)
|
||||
|
||||
@staticmethod
|
||||
def _get_placeholder(key: str) -> str:
|
||||
"""获取占位符提示"""
|
||||
placeholders = {
|
||||
"PG_DSN": "postgresql://user:password@host:5432/dbname",
|
||||
"PG_HOST": "localhost",
|
||||
"PG_PORT": "5432",
|
||||
"PG_NAME": "billiards",
|
||||
"PG_USER": "postgres",
|
||||
"PG_PASSWORD": "密码",
|
||||
"API_BASE": "https://pc.ficoo.vip/apiprod/admin/v1",
|
||||
"API_TOKEN": "Bearer token",
|
||||
"API_TIMEOUT": "20",
|
||||
"API_PAGE_SIZE": "200",
|
||||
"STORE_ID": "门店ID (数字)",
|
||||
"TIMEZONE": "Asia/Shanghai",
|
||||
"EXPORT_ROOT": "export/JSON",
|
||||
"LOG_ROOT": "export/LOG",
|
||||
"FETCH_ROOT": "JSON 抓取输出目录",
|
||||
"INGEST_SOURCE_DIR": "本地 JSON 输入目录",
|
||||
"PIPELINE_FLOW": "FULL / FETCH_ONLY / INGEST_ONLY",
|
||||
"RUN_TASKS": "任务列表,逗号分隔",
|
||||
"OVERLAP_SECONDS": "3600",
|
||||
"WINDOW_START": "2025-07-01 00:00:00",
|
||||
"WINDOW_END": "2025-08-01 00:00:00",
|
||||
}
|
||||
return placeholders.get(key, "")
|
||||
247
gui/widgets/log_viewer.py
Normal file
247
gui/widgets/log_viewer.py
Normal file
@@ -0,0 +1,247 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""日志查看器"""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QWidget, QVBoxLayout, QHBoxLayout,
|
||||
QPlainTextEdit, QPushButton, QLineEdit, QLabel,
|
||||
QComboBox, QCheckBox, QFileDialog, QMessageBox
|
||||
)
|
||||
from PySide6.QtCore import Qt, Signal, Slot
|
||||
from PySide6.QtGui import QTextCharFormat, QColor, QFont, QTextCursor
|
||||
|
||||
|
||||
class LogViewer(QWidget):
|
||||
"""日志查看器"""
|
||||
|
||||
# 信号
|
||||
log_cleared = Signal()
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.max_lines = 10000
|
||||
self.auto_scroll = True
|
||||
self.filter_text = ""
|
||||
self.filter_level = "ALL"
|
||||
self._all_logs = [] # 存储所有日志
|
||||
|
||||
self._init_ui()
|
||||
self._connect_signals()
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(16, 16, 16, 16)
|
||||
layout.setSpacing(8)
|
||||
|
||||
# 标题和工具栏
|
||||
header_layout = QHBoxLayout()
|
||||
|
||||
title = QLabel("执行日志")
|
||||
title.setProperty("heading", True)
|
||||
header_layout.addWidget(title)
|
||||
|
||||
header_layout.addStretch()
|
||||
|
||||
# 日志级别过滤
|
||||
header_layout.addWidget(QLabel("级别:"))
|
||||
self.level_combo = QComboBox()
|
||||
self.level_combo.addItems(["ALL", "INFO", "WARNING", "ERROR", "DEBUG"])
|
||||
self.level_combo.setFixedWidth(100)
|
||||
header_layout.addWidget(self.level_combo)
|
||||
|
||||
# 搜索框
|
||||
header_layout.addWidget(QLabel("搜索:"))
|
||||
self.search_edit = QLineEdit()
|
||||
self.search_edit.setPlaceholderText("输入关键字...")
|
||||
self.search_edit.setFixedWidth(200)
|
||||
header_layout.addWidget(self.search_edit)
|
||||
|
||||
# 自动滚动
|
||||
self.auto_scroll_check = QCheckBox("自动滚动")
|
||||
self.auto_scroll_check.setChecked(True)
|
||||
header_layout.addWidget(self.auto_scroll_check)
|
||||
|
||||
layout.addLayout(header_layout)
|
||||
|
||||
# 日志文本区域
|
||||
self.log_text = QPlainTextEdit()
|
||||
self.log_text.setObjectName("logViewer")
|
||||
self.log_text.setReadOnly(True)
|
||||
self.log_text.setFont(QFont("Consolas", 10))
|
||||
self.log_text.setLineWrapMode(QPlainTextEdit.NoWrap)
|
||||
layout.addWidget(self.log_text, 1)
|
||||
|
||||
# 底部工具栏
|
||||
footer_layout = QHBoxLayout()
|
||||
|
||||
self.line_count_label = QLabel("0 行")
|
||||
self.line_count_label.setProperty("subheading", True)
|
||||
footer_layout.addWidget(self.line_count_label)
|
||||
|
||||
footer_layout.addStretch()
|
||||
|
||||
self.copy_btn = QPushButton("复制全部")
|
||||
self.copy_btn.setProperty("secondary", True)
|
||||
footer_layout.addWidget(self.copy_btn)
|
||||
|
||||
self.export_btn = QPushButton("导出")
|
||||
self.export_btn.setProperty("secondary", True)
|
||||
footer_layout.addWidget(self.export_btn)
|
||||
|
||||
self.clear_btn = QPushButton("清空")
|
||||
self.clear_btn.setProperty("secondary", True)
|
||||
footer_layout.addWidget(self.clear_btn)
|
||||
|
||||
layout.addLayout(footer_layout)
|
||||
|
||||
def _connect_signals(self):
|
||||
"""连接信号"""
|
||||
self.level_combo.currentTextChanged.connect(self._apply_filter)
|
||||
self.search_edit.textChanged.connect(self._apply_filter)
|
||||
self.auto_scroll_check.stateChanged.connect(self._toggle_auto_scroll)
|
||||
self.copy_btn.clicked.connect(self._copy_all)
|
||||
self.export_btn.clicked.connect(self._export_log)
|
||||
self.clear_btn.clicked.connect(self._clear_log)
|
||||
|
||||
@Slot(str)
|
||||
def append_log(self, text: str):
|
||||
"""追加日志"""
|
||||
# 添加时间戳(如果没有)
|
||||
if not re.match(r'^\d{4}-\d{2}-\d{2}', text) and not text.startswith('['):
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
text = f"[{timestamp}] {text}"
|
||||
|
||||
# 存储到全部日志
|
||||
self._all_logs.append(text)
|
||||
|
||||
# 限制日志行数
|
||||
if len(self._all_logs) > self.max_lines:
|
||||
self._all_logs = self._all_logs[-self.max_lines:]
|
||||
|
||||
# 检查是否通过过滤器
|
||||
if self._matches_filter(text):
|
||||
self._append_formatted_line(text)
|
||||
|
||||
# 更新行数
|
||||
self._update_line_count()
|
||||
|
||||
def _append_formatted_line(self, text: str):
|
||||
"""追加格式化的行"""
|
||||
cursor = self.log_text.textCursor()
|
||||
cursor.movePosition(QTextCursor.End)
|
||||
|
||||
# 设置格式
|
||||
fmt = QTextCharFormat()
|
||||
|
||||
text_lower = text.lower()
|
||||
if "[error]" in text_lower or "错误" in text or "失败" in text:
|
||||
fmt.setForeground(QColor("#d93025"))
|
||||
fmt.setFontWeight(QFont.Bold)
|
||||
elif "[warning]" in text_lower or "警告" in text or "warn" in text_lower:
|
||||
fmt.setForeground(QColor("#f9ab00"))
|
||||
elif "[info]" in text_lower:
|
||||
fmt.setForeground(QColor("#1a73e8"))
|
||||
elif "[debug]" in text_lower:
|
||||
fmt.setForeground(QColor("#9aa0a6"))
|
||||
elif "[gui]" in text_lower:
|
||||
fmt.setForeground(QColor("#1e8e3e"))
|
||||
else:
|
||||
fmt.setForeground(QColor("#333333"))
|
||||
|
||||
cursor.insertText(text + "\n", fmt)
|
||||
|
||||
# 自动滚动
|
||||
if self.auto_scroll:
|
||||
self.log_text.verticalScrollBar().setValue(
|
||||
self.log_text.verticalScrollBar().maximum()
|
||||
)
|
||||
|
||||
def _matches_filter(self, text: str) -> bool:
|
||||
"""检查是否匹配过滤器"""
|
||||
# 级别过滤
|
||||
if self.filter_level != "ALL":
|
||||
level_marker = f"[{self.filter_level}]"
|
||||
if level_marker.lower() not in text.lower():
|
||||
return False
|
||||
|
||||
# 文本过滤
|
||||
if self.filter_text:
|
||||
if self.filter_text.lower() not in text.lower():
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _apply_filter(self):
|
||||
"""应用过滤器"""
|
||||
self.filter_level = self.level_combo.currentText()
|
||||
self.filter_text = self.search_edit.text().strip()
|
||||
|
||||
# 重新显示日志
|
||||
self.log_text.clear()
|
||||
for line in self._all_logs:
|
||||
if self._matches_filter(line):
|
||||
self._append_formatted_line(line)
|
||||
|
||||
self._update_line_count()
|
||||
|
||||
def _toggle_auto_scroll(self, state: int):
|
||||
"""切换自动滚动"""
|
||||
self.auto_scroll = state == Qt.Checked
|
||||
|
||||
def _copy_all(self):
|
||||
"""复制全部日志"""
|
||||
from PySide6.QtWidgets import QApplication
|
||||
text = self.log_text.toPlainText()
|
||||
QApplication.clipboard().setText(text)
|
||||
QMessageBox.information(self, "提示", "日志已复制到剪贴板")
|
||||
|
||||
def _export_log(self):
|
||||
"""导出日志"""
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
default_name = f"etl_log_{timestamp}.txt"
|
||||
|
||||
file_path, _ = QFileDialog.getSaveFileName(
|
||||
self,
|
||||
"导出日志",
|
||||
default_name,
|
||||
"文本文件 (*.txt);;日志文件 (*.log);;所有文件 (*.*)"
|
||||
)
|
||||
|
||||
if not file_path:
|
||||
return
|
||||
|
||||
try:
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
f.write(self.log_text.toPlainText())
|
||||
QMessageBox.information(self, "成功", f"日志已导出到:\n{file_path}")
|
||||
except Exception as e:
|
||||
QMessageBox.critical(self, "错误", f"导出失败: {e}")
|
||||
|
||||
def _clear_log(self):
|
||||
"""清空日志"""
|
||||
reply = QMessageBox.question(
|
||||
self,
|
||||
"确认",
|
||||
"确定要清空所有日志吗?",
|
||||
QMessageBox.Yes | QMessageBox.No,
|
||||
QMessageBox.No
|
||||
)
|
||||
|
||||
if reply == QMessageBox.Yes:
|
||||
self._all_logs.clear()
|
||||
self.log_text.clear()
|
||||
self._update_line_count()
|
||||
self.log_cleared.emit()
|
||||
|
||||
def _update_line_count(self):
|
||||
"""更新行数显示"""
|
||||
visible_count = self.log_text.document().blockCount() - 1
|
||||
total_count = len(self._all_logs)
|
||||
|
||||
if visible_count < total_count:
|
||||
self.line_count_label.setText(f"{visible_count} / {total_count} 行")
|
||||
else:
|
||||
self.line_count_label.setText(f"{total_count} 行")
|
||||
604
gui/widgets/pipeline_selector.py
Normal file
604
gui/widgets/pipeline_selector.py
Normal file
@@ -0,0 +1,604 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""管道选择组件:统一的 ETL 管道配置界面。"""
|
||||
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QWidget, QVBoxLayout, QHBoxLayout, QGroupBox,
|
||||
QRadioButton, QButtonGroup, QLabel, QSpinBox,
|
||||
QDateTimeEdit, QComboBox, QCheckBox, QPushButton,
|
||||
QScrollArea, QFrame
|
||||
)
|
||||
from PySide6.QtCore import Signal, Qt, QDateTime
|
||||
|
||||
from ..models.task_registry import (
|
||||
TaskRegistry, TaskDefinition, BusinessDomain, DOMAIN_LABELS,
|
||||
task_registry, get_fact_ods_task_codes, get_dimension_ods_task_codes
|
||||
)
|
||||
|
||||
|
||||
# 管道选项定义:(id, 显示名称, 包含的层)
|
||||
PIPELINE_OPTIONS: List[Tuple[str, str, List[str]]] = [
|
||||
("api_ods", "API → ODS", ["ODS"]),
|
||||
("api_ods_dwd", "API → ODS → DWD", ["ODS", "DWD"]),
|
||||
("api_full", "API → ODS → DWD → DWS汇总 → DWS指数", ["ODS", "DWD", "DWS", "INDEX"]),
|
||||
("ods_dwd", "ODS → DWD", ["DWD"]),
|
||||
("dwd_dws", "DWD → DWS汇总", ["DWS"]),
|
||||
("dwd_dws_index", "DWD → DWS汇总 → DWS指数", ["DWS", "INDEX"]),
|
||||
("dwd_index", "DWD → DWS指数", ["INDEX"]),
|
||||
]
|
||||
|
||||
# 数据处理模式
|
||||
PROCESSING_MODES: List[Tuple[str, str, str]] = [
|
||||
("increment_only", "仅增量", "仅执行增量数据处理,不进行校验"),
|
||||
("verify_only", "校验并修复", "跳过增量处理,直接校验数据一致性并自动补齐缺失/不一致数据"),
|
||||
("increment_verify", "增量 + 校验并修复", "先执行增量处理,再校验并修复缺失/不一致数据"),
|
||||
]
|
||||
|
||||
# 校验模式附加选项
|
||||
VERIFY_MODE_OPTIONS = {
|
||||
"fetch_before_verify": "校验前先从 API 获取数据",
|
||||
"skip_ods_when_fetch_before_verify": "跳过 ODS 校验(仅在校验前获取时)",
|
||||
"ods_use_local_json": "ODS 校验使用本地 JSON(不请求 API)",
|
||||
}
|
||||
|
||||
# 时间窗口模式
|
||||
WINDOW_MODES: List[Tuple[str, str]] = [
|
||||
("lookback", "回溯 + 冗余"),
|
||||
("custom", "自定义时间范围"),
|
||||
]
|
||||
|
||||
# 时间窗口切分选项
|
||||
WINDOW_SPLIT_OPTIONS: List[Tuple[str, str]] = [
|
||||
("none", "不切分"),
|
||||
("day", "按天"),
|
||||
]
|
||||
|
||||
# 时间窗口切分天数(按天时生效)
|
||||
WINDOW_SPLIT_DAY_OPTIONS: List[Tuple[int, str]] = [
|
||||
(1, "1 天"),
|
||||
(10, "10 天"),
|
||||
(30, "30 天"),
|
||||
]
|
||||
|
||||
|
||||
def get_pipeline_layers(pipeline_id: str) -> List[str]:
|
||||
"""获取管道包含的层"""
|
||||
for pid, _, layers in PIPELINE_OPTIONS:
|
||||
if pid == pipeline_id:
|
||||
return layers
|
||||
return []
|
||||
|
||||
|
||||
def get_pipeline_display_name(pipeline_id: str) -> str:
|
||||
"""获取管道显示名称"""
|
||||
for pid, name, _ in PIPELINE_OPTIONS:
|
||||
if pid == pipeline_id:
|
||||
return name
|
||||
return pipeline_id
|
||||
|
||||
|
||||
class PipelineSelectorWidget(QWidget):
|
||||
"""管道选择组件"""
|
||||
|
||||
# 信号
|
||||
pipeline_changed = Signal(str) # 管道ID
|
||||
processing_mode_changed = Signal(str) # 处理模式
|
||||
window_mode_changed = Signal(str) # 时间窗口模式
|
||||
config_changed = Signal() # 任意配置变化
|
||||
|
||||
def __init__(self, parent: Optional[QWidget] = None):
|
||||
super().__init__(parent)
|
||||
|
||||
# 当前选择
|
||||
self._pipeline_id = "api_ods_dwd"
|
||||
self._processing_mode = "increment_only"
|
||||
self._window_mode = "lookback"
|
||||
self._fetch_before_verify = False
|
||||
self._skip_ods_when_fetch_before_verify = True
|
||||
self._ods_use_local_json = True
|
||||
|
||||
self._init_ui()
|
||||
self._connect_signals()
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(0, 0, 0, 0)
|
||||
layout.setSpacing(12)
|
||||
|
||||
# 1. 管道选择
|
||||
pipeline_group = self._create_pipeline_group()
|
||||
layout.addWidget(pipeline_group)
|
||||
|
||||
# 2. 数据处理模式
|
||||
processing_group = self._create_processing_mode_group()
|
||||
layout.addWidget(processing_group)
|
||||
|
||||
# 3. 时间窗口配置
|
||||
window_group = self._create_window_group()
|
||||
layout.addWidget(window_group)
|
||||
|
||||
layout.addStretch()
|
||||
|
||||
def _create_pipeline_group(self) -> QGroupBox:
|
||||
"""创建管道选择分组"""
|
||||
group = QGroupBox("管道选择 (Pipeline)")
|
||||
layout = QVBoxLayout(group)
|
||||
layout.setSpacing(4)
|
||||
|
||||
self._pipeline_button_group = QButtonGroup(self)
|
||||
|
||||
for i, (pid, name, layers) in enumerate(PIPELINE_OPTIONS):
|
||||
radio = QRadioButton(name)
|
||||
radio.setProperty("pipeline_id", pid)
|
||||
radio.setToolTip(f"包含层: {' → '.join(layers)}")
|
||||
|
||||
if pid == self._pipeline_id:
|
||||
radio.setChecked(True)
|
||||
|
||||
self._pipeline_button_group.addButton(radio, i)
|
||||
layout.addWidget(radio)
|
||||
|
||||
return group
|
||||
|
||||
def _create_processing_mode_group(self) -> QGroupBox:
|
||||
"""创建数据处理模式分组"""
|
||||
group = QGroupBox("数据处理模式")
|
||||
layout = QVBoxLayout(group)
|
||||
layout.setSpacing(4)
|
||||
|
||||
self._processing_button_group = QButtonGroup(self)
|
||||
|
||||
for i, (mode_id, name, tooltip) in enumerate(PROCESSING_MODES):
|
||||
radio = QRadioButton(name)
|
||||
radio.setProperty("mode_id", mode_id)
|
||||
radio.setToolTip(tooltip)
|
||||
|
||||
if mode_id == self._processing_mode:
|
||||
radio.setChecked(True)
|
||||
|
||||
self._processing_button_group.addButton(radio, i)
|
||||
layout.addWidget(radio)
|
||||
|
||||
# 校验模式附加选项:校验前从 API 获取数据
|
||||
option_layout = QHBoxLayout()
|
||||
option_layout.setContentsMargins(20, 4, 0, 0) # 缩进以表示从属关系
|
||||
|
||||
self._fetch_before_verify_checkbox = QCheckBox(
|
||||
VERIFY_MODE_OPTIONS["fetch_before_verify"]
|
||||
)
|
||||
self._fetch_before_verify_checkbox.setToolTip(
|
||||
"勾选后,在执行校验前会先从 API 获取最新数据到 ODS 层。\n"
|
||||
"适用于需要同时获取新数据并校验修复的场景。"
|
||||
)
|
||||
self._fetch_before_verify_checkbox.setChecked(self._fetch_before_verify)
|
||||
# 默认禁用,仅在 verify_only 模式下启用
|
||||
self._fetch_before_verify_checkbox.setEnabled(
|
||||
self._processing_mode == "verify_only"
|
||||
)
|
||||
option_layout.addWidget(self._fetch_before_verify_checkbox)
|
||||
option_layout.addStretch()
|
||||
layout.addLayout(option_layout)
|
||||
|
||||
# 仅在 fetch_before_verify 时生效的附加选项
|
||||
skip_ods_layout = QHBoxLayout()
|
||||
skip_ods_layout.setContentsMargins(40, 2, 0, 0)
|
||||
self._skip_ods_when_fetch_before_verify_checkbox = QCheckBox(
|
||||
VERIFY_MODE_OPTIONS["skip_ods_when_fetch_before_verify"]
|
||||
)
|
||||
self._skip_ods_when_fetch_before_verify_checkbox.setToolTip(
|
||||
"勾选后,在校验前先抓取数据的场景下跳过 ODS 校验。\n"
|
||||
"适用于仅关心 ODS 入库统计或避免重复校验的场景。"
|
||||
)
|
||||
self._skip_ods_when_fetch_before_verify_checkbox.setChecked(
|
||||
self._skip_ods_when_fetch_before_verify
|
||||
)
|
||||
skip_ods_layout.addWidget(self._skip_ods_when_fetch_before_verify_checkbox)
|
||||
skip_ods_layout.addStretch()
|
||||
layout.addLayout(skip_ods_layout)
|
||||
|
||||
local_json_layout = QHBoxLayout()
|
||||
local_json_layout.setContentsMargins(40, 2, 0, 0)
|
||||
self._ods_use_local_json_checkbox = QCheckBox(
|
||||
VERIFY_MODE_OPTIONS["ods_use_local_json"]
|
||||
)
|
||||
self._ods_use_local_json_checkbox.setToolTip(
|
||||
"勾选后,ODS 校验将完全基于落盘 JSON 进行,不再请求 API。\n"
|
||||
"需要先执行“校验前先从 API 获取数据”以生成 JSON。"
|
||||
)
|
||||
self._ods_use_local_json_checkbox.setChecked(self._ods_use_local_json)
|
||||
local_json_layout.addWidget(self._ods_use_local_json_checkbox)
|
||||
local_json_layout.addStretch()
|
||||
layout.addLayout(local_json_layout)
|
||||
|
||||
self._update_verify_option_states()
|
||||
|
||||
return group
|
||||
|
||||
def _create_window_group(self) -> QGroupBox:
|
||||
"""创建时间窗口配置分组"""
|
||||
group = QGroupBox("时间窗口")
|
||||
layout = QVBoxLayout(group)
|
||||
layout.setSpacing(8)
|
||||
|
||||
# 时间窗口模式选择
|
||||
self._window_button_group = QButtonGroup(self)
|
||||
|
||||
# 回溯模式
|
||||
lookback_layout = QHBoxLayout()
|
||||
self._lookback_radio = QRadioButton("回溯 + 冗余:")
|
||||
self._lookback_radio.setProperty("mode_id", "lookback")
|
||||
self._lookback_radio.setChecked(True)
|
||||
self._window_button_group.addButton(self._lookback_radio, 0)
|
||||
lookback_layout.addWidget(self._lookback_radio)
|
||||
|
||||
self._lookback_hours_spin = QSpinBox()
|
||||
self._lookback_hours_spin.setRange(1, 720)
|
||||
self._lookback_hours_spin.setValue(24)
|
||||
self._lookback_hours_spin.setSuffix(" 小时")
|
||||
self._lookback_hours_spin.setToolTip("回溯时间长度")
|
||||
self._lookback_hours_spin.setFixedWidth(100)
|
||||
lookback_layout.addWidget(self._lookback_hours_spin)
|
||||
|
||||
lookback_layout.addWidget(QLabel("冗余:"))
|
||||
|
||||
self._overlap_seconds_spin = QSpinBox()
|
||||
self._overlap_seconds_spin.setRange(0, 7200)
|
||||
self._overlap_seconds_spin.setValue(600)
|
||||
self._overlap_seconds_spin.setSuffix(" 秒")
|
||||
self._overlap_seconds_spin.setToolTip("时间窗口前后的重叠冗余")
|
||||
self._overlap_seconds_spin.setFixedWidth(100)
|
||||
lookback_layout.addWidget(self._overlap_seconds_spin)
|
||||
|
||||
lookback_layout.addStretch()
|
||||
layout.addLayout(lookback_layout)
|
||||
|
||||
# 自定义模式
|
||||
custom_layout = QHBoxLayout()
|
||||
self._custom_radio = QRadioButton("自定义:")
|
||||
self._custom_radio.setProperty("mode_id", "custom")
|
||||
self._window_button_group.addButton(self._custom_radio, 1)
|
||||
custom_layout.addWidget(self._custom_radio)
|
||||
|
||||
self._start_datetime = QDateTimeEdit()
|
||||
self._start_datetime.setCalendarPopup(True)
|
||||
self._start_datetime.setDisplayFormat("yyyy-MM-dd HH:mm:ss")
|
||||
self._start_datetime.setDateTime(QDateTime.currentDateTime().addDays(-1))
|
||||
self._start_datetime.setFixedWidth(160)
|
||||
self._start_datetime.setEnabled(False)
|
||||
custom_layout.addWidget(self._start_datetime)
|
||||
|
||||
custom_layout.addWidget(QLabel("至"))
|
||||
|
||||
self._end_datetime = QDateTimeEdit()
|
||||
self._end_datetime.setCalendarPopup(True)
|
||||
self._end_datetime.setDisplayFormat("yyyy-MM-dd HH:mm:ss")
|
||||
self._end_datetime.setDateTime(QDateTime.currentDateTime())
|
||||
self._end_datetime.setFixedWidth(160)
|
||||
self._end_datetime.setEnabled(False)
|
||||
custom_layout.addWidget(self._end_datetime)
|
||||
|
||||
custom_layout.addStretch()
|
||||
layout.addLayout(custom_layout)
|
||||
|
||||
# 时间窗口切分
|
||||
split_layout = QHBoxLayout()
|
||||
split_layout.addWidget(QLabel("时间窗口切分:"))
|
||||
|
||||
self._split_combo = QComboBox()
|
||||
for split_id, split_name in WINDOW_SPLIT_OPTIONS:
|
||||
self._split_combo.addItem(split_name, split_id)
|
||||
default_split_index = self._split_combo.findData("day")
|
||||
if default_split_index >= 0:
|
||||
self._split_combo.setCurrentIndex(default_split_index)
|
||||
self._split_combo.setFixedWidth(100)
|
||||
split_layout.addWidget(self._split_combo)
|
||||
|
||||
split_layout.addWidget(QLabel("切分天数:"))
|
||||
self._split_days_combo = QComboBox()
|
||||
for days, label in WINDOW_SPLIT_DAY_OPTIONS:
|
||||
self._split_days_combo.addItem(label, days)
|
||||
default_days_index = self._split_days_combo.findData(10)
|
||||
if default_days_index >= 0:
|
||||
self._split_days_combo.setCurrentIndex(default_days_index)
|
||||
self._split_days_combo.setFixedWidth(90)
|
||||
split_layout.addWidget(self._split_days_combo)
|
||||
|
||||
split_layout.addStretch()
|
||||
layout.addLayout(split_layout)
|
||||
|
||||
self._update_split_days_state()
|
||||
|
||||
return group
|
||||
|
||||
def _connect_signals(self):
|
||||
"""连接信号"""
|
||||
# 管道选择变化
|
||||
self._pipeline_button_group.buttonClicked.connect(self._on_pipeline_changed)
|
||||
|
||||
# 处理模式变化
|
||||
self._processing_button_group.buttonClicked.connect(self._on_processing_mode_changed)
|
||||
|
||||
# 时间窗口模式变化
|
||||
self._window_button_group.buttonClicked.connect(self._on_window_mode_changed)
|
||||
|
||||
# 其他配置变化
|
||||
self._lookback_hours_spin.valueChanged.connect(self._emit_config_changed)
|
||||
self._overlap_seconds_spin.valueChanged.connect(self._emit_config_changed)
|
||||
self._start_datetime.dateTimeChanged.connect(self._emit_config_changed)
|
||||
self._end_datetime.dateTimeChanged.connect(self._emit_config_changed)
|
||||
self._split_combo.currentIndexChanged.connect(self._on_split_changed)
|
||||
self._split_days_combo.currentIndexChanged.connect(self._emit_config_changed)
|
||||
|
||||
# 校验模式附加选项变化
|
||||
self._fetch_before_verify_checkbox.stateChanged.connect(self._on_fetch_before_verify_changed)
|
||||
self._skip_ods_when_fetch_before_verify_checkbox.stateChanged.connect(
|
||||
self._on_skip_ods_when_fetch_before_verify_changed
|
||||
)
|
||||
self._ods_use_local_json_checkbox.stateChanged.connect(
|
||||
self._on_ods_use_local_json_changed
|
||||
)
|
||||
|
||||
def _on_pipeline_changed(self, button: QRadioButton):
|
||||
"""管道选择变化"""
|
||||
pipeline_id = button.property("pipeline_id")
|
||||
if pipeline_id and pipeline_id != self._pipeline_id:
|
||||
self._pipeline_id = pipeline_id
|
||||
self.pipeline_changed.emit(pipeline_id)
|
||||
self.config_changed.emit()
|
||||
|
||||
def _on_processing_mode_changed(self, button: QRadioButton):
|
||||
"""处理模式变化"""
|
||||
mode_id = button.property("mode_id")
|
||||
if mode_id and mode_id != self._processing_mode:
|
||||
self._processing_mode = mode_id
|
||||
|
||||
# 更新 "校验前获取数据" 选项的启用状态
|
||||
# 仅在 verify_only 模式下可用
|
||||
is_verify_only = mode_id == "verify_only"
|
||||
self._fetch_before_verify_checkbox.setEnabled(is_verify_only)
|
||||
if not is_verify_only:
|
||||
# 非 verify_only 模式时,自动取消勾选
|
||||
self._fetch_before_verify_checkbox.setChecked(False)
|
||||
|
||||
self._update_verify_option_states()
|
||||
|
||||
self.processing_mode_changed.emit(mode_id)
|
||||
self.config_changed.emit()
|
||||
|
||||
def _on_fetch_before_verify_changed(self, state: int):
|
||||
"""校验前获取数据选项变化"""
|
||||
from PySide6.QtCore import Qt
|
||||
self._fetch_before_verify = state == Qt.Checked.value
|
||||
self._update_verify_option_states()
|
||||
self.config_changed.emit()
|
||||
|
||||
def _on_skip_ods_when_fetch_before_verify_changed(self, state: int):
|
||||
"""跳过 ODS 校验选项变化"""
|
||||
from PySide6.QtCore import Qt
|
||||
self._skip_ods_when_fetch_before_verify = state == Qt.Checked.value
|
||||
self.config_changed.emit()
|
||||
|
||||
def _on_ods_use_local_json_changed(self, state: int):
|
||||
"""ODS 校验使用本地 JSON 选项变化"""
|
||||
from PySide6.QtCore import Qt
|
||||
self._ods_use_local_json = state == Qt.Checked.value
|
||||
self.config_changed.emit()
|
||||
|
||||
def _update_verify_option_states(self):
|
||||
"""更新校验附加选项的启用状态"""
|
||||
enable_suboptions = self._processing_mode == "verify_only" and self._fetch_before_verify
|
||||
self._skip_ods_when_fetch_before_verify_checkbox.setEnabled(enable_suboptions)
|
||||
self._ods_use_local_json_checkbox.setEnabled(enable_suboptions)
|
||||
|
||||
def _on_window_mode_changed(self, button: QRadioButton):
|
||||
"""时间窗口模式变化"""
|
||||
mode_id = button.property("mode_id")
|
||||
if mode_id and mode_id != self._window_mode:
|
||||
self._window_mode = mode_id
|
||||
|
||||
# 更新控件启用状态
|
||||
is_lookback = mode_id == "lookback"
|
||||
self._lookback_hours_spin.setEnabled(is_lookback)
|
||||
self._overlap_seconds_spin.setEnabled(is_lookback)
|
||||
self._start_datetime.setEnabled(not is_lookback)
|
||||
self._end_datetime.setEnabled(not is_lookback)
|
||||
|
||||
self.window_mode_changed.emit(mode_id)
|
||||
self.config_changed.emit()
|
||||
|
||||
def _on_split_changed(self):
|
||||
"""时间窗口切分方式变化"""
|
||||
self._update_split_days_state()
|
||||
self.config_changed.emit()
|
||||
|
||||
def _update_split_days_state(self):
|
||||
"""按天切分才允许选择天数"""
|
||||
is_day_split = self.get_window_split() == "day"
|
||||
self._split_days_combo.setEnabled(is_day_split)
|
||||
|
||||
def _emit_config_changed(self):
|
||||
"""发出配置变化信号"""
|
||||
self.config_changed.emit()
|
||||
|
||||
# === 公共接口 ===
|
||||
|
||||
def get_pipeline_id(self) -> str:
|
||||
"""获取当前管道ID"""
|
||||
return self._pipeline_id
|
||||
|
||||
def set_pipeline_id(self, pipeline_id: str):
|
||||
"""设置管道ID"""
|
||||
for button in self._pipeline_button_group.buttons():
|
||||
if button.property("pipeline_id") == pipeline_id:
|
||||
button.setChecked(True)
|
||||
self._pipeline_id = pipeline_id
|
||||
break
|
||||
|
||||
def get_pipeline_layers(self) -> List[str]:
|
||||
"""获取当前管道包含的层"""
|
||||
return get_pipeline_layers(self._pipeline_id)
|
||||
|
||||
def get_processing_mode(self) -> str:
|
||||
"""获取数据处理模式"""
|
||||
return self._processing_mode
|
||||
|
||||
def set_processing_mode(self, mode: str):
|
||||
"""设置数据处理模式"""
|
||||
for button in self._processing_button_group.buttons():
|
||||
if button.property("mode_id") == mode:
|
||||
button.setChecked(True)
|
||||
self._processing_mode = mode
|
||||
# 更新复选框启用状态
|
||||
is_verify_only = mode == "verify_only"
|
||||
self._fetch_before_verify_checkbox.setEnabled(is_verify_only)
|
||||
if not is_verify_only:
|
||||
self._fetch_before_verify_checkbox.setChecked(False)
|
||||
self._update_verify_option_states()
|
||||
break
|
||||
|
||||
def get_fetch_before_verify(self) -> bool:
|
||||
"""获取是否在校验前从 API 获取数据"""
|
||||
return self._fetch_before_verify
|
||||
|
||||
def set_fetch_before_verify(self, enabled: bool):
|
||||
"""设置是否在校验前从 API 获取数据"""
|
||||
self._fetch_before_verify = enabled
|
||||
self._fetch_before_verify_checkbox.setChecked(enabled)
|
||||
self._update_verify_option_states()
|
||||
|
||||
def get_skip_ods_when_fetch_before_verify(self) -> bool:
|
||||
"""获取是否跳过 ODS 校验(仅校验前获取时生效)"""
|
||||
return self._skip_ods_when_fetch_before_verify
|
||||
|
||||
def set_skip_ods_when_fetch_before_verify(self, enabled: bool):
|
||||
"""设置是否跳过 ODS 校验(仅校验前获取时生效)"""
|
||||
self._skip_ods_when_fetch_before_verify = enabled
|
||||
self._skip_ods_when_fetch_before_verify_checkbox.setChecked(enabled)
|
||||
self._update_verify_option_states()
|
||||
|
||||
def get_ods_use_local_json(self) -> bool:
|
||||
"""获取是否使用本地 JSON 进行 ODS 校验"""
|
||||
return self._ods_use_local_json
|
||||
|
||||
def set_ods_use_local_json(self, enabled: bool):
|
||||
"""设置是否使用本地 JSON 进行 ODS 校验"""
|
||||
self._ods_use_local_json = enabled
|
||||
self._ods_use_local_json_checkbox.setChecked(enabled)
|
||||
self._update_verify_option_states()
|
||||
|
||||
def get_window_mode(self) -> str:
|
||||
"""获取时间窗口模式"""
|
||||
return self._window_mode
|
||||
|
||||
def set_window_mode(self, mode: str):
|
||||
"""设置时间窗口模式"""
|
||||
for button in self._window_button_group.buttons():
|
||||
if button.property("mode_id") == mode:
|
||||
button.setChecked(True)
|
||||
self._on_window_mode_changed(button)
|
||||
break
|
||||
|
||||
def get_lookback_hours(self) -> int:
|
||||
"""获取回溯小时数"""
|
||||
return self._lookback_hours_spin.value()
|
||||
|
||||
def set_lookback_hours(self, hours: int):
|
||||
"""设置回溯小时数"""
|
||||
self._lookback_hours_spin.setValue(hours)
|
||||
|
||||
def get_overlap_seconds(self) -> int:
|
||||
"""获取冗余秒数"""
|
||||
return self._overlap_seconds_spin.value()
|
||||
|
||||
def set_overlap_seconds(self, seconds: int):
|
||||
"""设置冗余秒数"""
|
||||
self._overlap_seconds_spin.setValue(seconds)
|
||||
|
||||
def get_window_start(self) -> str:
|
||||
"""获取开始时间(ISO格式)"""
|
||||
return self._start_datetime.dateTime().toString("yyyy-MM-dd HH:mm:ss")
|
||||
|
||||
def set_window_start(self, dt_str: str):
|
||||
"""设置开始时间"""
|
||||
dt = QDateTime.fromString(dt_str, "yyyy-MM-dd HH:mm:ss")
|
||||
if dt.isValid():
|
||||
self._start_datetime.setDateTime(dt)
|
||||
|
||||
def get_window_end(self) -> str:
|
||||
"""获取结束时间(ISO格式)"""
|
||||
return self._end_datetime.dateTime().toString("yyyy-MM-dd HH:mm:ss")
|
||||
|
||||
def set_window_end(self, dt_str: str):
|
||||
"""设置结束时间"""
|
||||
dt = QDateTime.fromString(dt_str, "yyyy-MM-dd HH:mm:ss")
|
||||
if dt.isValid():
|
||||
self._end_datetime.setDateTime(dt)
|
||||
|
||||
def get_window_split(self) -> str:
|
||||
"""获取窗口切分模式"""
|
||||
return self._split_combo.currentData()
|
||||
|
||||
def get_window_split_days(self) -> int:
|
||||
"""获取按天切分天数"""
|
||||
return int(self._split_days_combo.currentData())
|
||||
|
||||
def set_window_split(self, split: str):
|
||||
"""设置窗口切分模式"""
|
||||
index = self._split_combo.findData(split)
|
||||
if index >= 0:
|
||||
self._split_combo.setCurrentIndex(index)
|
||||
self._update_split_days_state()
|
||||
|
||||
def set_window_split_days(self, days: int):
|
||||
"""设置按天切分天数"""
|
||||
index = self._split_days_combo.findData(days)
|
||||
if index >= 0:
|
||||
self._split_days_combo.setCurrentIndex(index)
|
||||
|
||||
def get_config(self) -> dict:
|
||||
"""获取完整配置字典"""
|
||||
split_unit = self.get_window_split()
|
||||
split_days = self.get_window_split_days()
|
||||
return {
|
||||
"pipeline": self._pipeline_id,
|
||||
"processing_mode": self._processing_mode,
|
||||
"fetch_before_verify": self._fetch_before_verify,
|
||||
"skip_ods_when_fetch_before_verify": self._skip_ods_when_fetch_before_verify,
|
||||
"ods_use_local_json": self._ods_use_local_json,
|
||||
"window_mode": self._window_mode,
|
||||
"lookback_hours": self.get_lookback_hours(),
|
||||
"overlap_seconds": self.get_overlap_seconds(),
|
||||
"window_start": self.get_window_start(),
|
||||
"window_end": self.get_window_end(),
|
||||
"window_split": split_unit,
|
||||
"window_split_days": split_days,
|
||||
}
|
||||
|
||||
def set_config(self, config: dict):
|
||||
"""从配置字典恢复设置"""
|
||||
if "pipeline" in config:
|
||||
self.set_pipeline_id(config["pipeline"])
|
||||
if "processing_mode" in config:
|
||||
self.set_processing_mode(config["processing_mode"])
|
||||
if "fetch_before_verify" in config:
|
||||
self.set_fetch_before_verify(config["fetch_before_verify"])
|
||||
if "skip_ods_when_fetch_before_verify" in config:
|
||||
self.set_skip_ods_when_fetch_before_verify(config["skip_ods_when_fetch_before_verify"])
|
||||
if "ods_use_local_json" in config:
|
||||
self.set_ods_use_local_json(config["ods_use_local_json"])
|
||||
if "window_mode" in config:
|
||||
self.set_window_mode(config["window_mode"])
|
||||
if "lookback_hours" in config:
|
||||
self.set_lookback_hours(config["lookback_hours"])
|
||||
if "overlap_seconds" in config:
|
||||
self.set_overlap_seconds(config["overlap_seconds"])
|
||||
if "window_start" in config:
|
||||
self.set_window_start(config["window_start"])
|
||||
if "window_end" in config:
|
||||
self.set_window_end(config["window_end"])
|
||||
if "window_split" in config:
|
||||
self.set_window_split(config["window_split"])
|
||||
if "window_split_days" in config and config["window_split_days"]:
|
||||
self.set_window_split_days(config["window_split_days"])
|
||||
166
gui/widgets/settings_dialog.py
Normal file
166
gui/widgets/settings_dialog.py
Normal file
@@ -0,0 +1,166 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""应用程序设置对话框"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QDialog, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||||
QGroupBox, QLabel, QLineEdit, QPushButton,
|
||||
QFileDialog, QMessageBox, QDialogButtonBox
|
||||
)
|
||||
from PySide6.QtCore import Qt
|
||||
|
||||
from ..utils.app_settings import app_settings
|
||||
|
||||
|
||||
class SettingsDialog(QDialog):
|
||||
"""设置对话框"""
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.setWindowTitle("应用程序设置")
|
||||
self.setMinimumWidth(600)
|
||||
self._init_ui()
|
||||
self._load_settings()
|
||||
|
||||
def _init_ui(self):
|
||||
layout = QVBoxLayout(self)
|
||||
|
||||
# ETL 项目路径
|
||||
project_group = QGroupBox("ETL 项目配置")
|
||||
project_layout = QGridLayout(project_group)
|
||||
|
||||
project_layout.addWidget(QLabel("ETL 项目路径:"), 0, 0)
|
||||
self.project_path_edit = QLineEdit()
|
||||
self.project_path_edit.setPlaceholderText("例: C:\\ZQYY\\FQ-ETL")
|
||||
project_layout.addWidget(self.project_path_edit, 0, 1)
|
||||
|
||||
browse_project_btn = QPushButton("浏览...")
|
||||
browse_project_btn.clicked.connect(self._browse_project_path)
|
||||
project_layout.addWidget(browse_project_btn, 0, 2)
|
||||
|
||||
project_layout.addWidget(QLabel(".env 文件路径:"), 1, 0)
|
||||
self.env_path_edit = QLineEdit()
|
||||
self.env_path_edit.setPlaceholderText("例: .env")
|
||||
project_layout.addWidget(self.env_path_edit, 1, 1)
|
||||
|
||||
browse_env_btn = QPushButton("浏览...")
|
||||
browse_env_btn.clicked.connect(self._browse_env_path)
|
||||
project_layout.addWidget(browse_env_btn, 1, 2)
|
||||
|
||||
# 验证按钮
|
||||
validate_btn = QPushButton("验证配置")
|
||||
validate_btn.clicked.connect(self._validate_config)
|
||||
project_layout.addWidget(validate_btn, 2, 1)
|
||||
|
||||
# 验证结果
|
||||
self.validation_label = QLabel()
|
||||
self.validation_label.setWordWrap(True)
|
||||
project_layout.addWidget(self.validation_label, 3, 0, 1, 3)
|
||||
|
||||
layout.addWidget(project_group)
|
||||
|
||||
# 说明
|
||||
note = QLabel(
|
||||
"说明:\n"
|
||||
"• ETL 项目路径:包含 cli/main.py 的目录\n"
|
||||
"• .env 文件路径:环境变量配置文件\n"
|
||||
"• 配置后才能正常执行 ETL 任务"
|
||||
)
|
||||
note.setProperty("subheading", True)
|
||||
note.setWordWrap(True)
|
||||
layout.addWidget(note)
|
||||
|
||||
layout.addStretch()
|
||||
|
||||
# 按钮
|
||||
btn_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
|
||||
btn_box.accepted.connect(self._save_and_accept)
|
||||
btn_box.rejected.connect(self.reject)
|
||||
layout.addWidget(btn_box)
|
||||
|
||||
def _load_settings(self):
|
||||
"""加载设置"""
|
||||
self.project_path_edit.setText(app_settings.etl_project_path)
|
||||
self.env_path_edit.setText(app_settings.env_file_path)
|
||||
self._validate_config()
|
||||
|
||||
def _browse_project_path(self):
|
||||
"""浏览项目路径"""
|
||||
path = QFileDialog.getExistingDirectory(
|
||||
self, "选择 ETL 项目目录",
|
||||
self.project_path_edit.text() or str(Path.home())
|
||||
)
|
||||
if path:
|
||||
self.project_path_edit.setText(path)
|
||||
# 自动填充 .env 路径
|
||||
env_path = Path(path) / ".env"
|
||||
if env_path.exists():
|
||||
self.env_path_edit.setText(str(env_path))
|
||||
self._validate_config()
|
||||
|
||||
def _browse_env_path(self):
|
||||
"""浏览 .env 文件"""
|
||||
path, _ = QFileDialog.getOpenFileName(
|
||||
self, "选择 .env 文件",
|
||||
self.env_path_edit.text() or str(Path.home()),
|
||||
"环境变量文件 (*.env);;所有文件 (*.*)"
|
||||
)
|
||||
if path:
|
||||
self.env_path_edit.setText(path)
|
||||
self._validate_config()
|
||||
|
||||
def _validate_config(self):
|
||||
"""验证配置"""
|
||||
project_path = self.project_path_edit.text().strip()
|
||||
env_path = self.env_path_edit.text().strip()
|
||||
|
||||
issues = []
|
||||
|
||||
if not project_path:
|
||||
issues.append("• 未设置 ETL 项目路径")
|
||||
else:
|
||||
p = Path(project_path)
|
||||
if not p.exists():
|
||||
issues.append(f"• ETL 项目路径不存在")
|
||||
elif not (p / "cli" / "main.py").exists():
|
||||
issues.append(f"• 找不到 cli/main.py")
|
||||
|
||||
if not env_path:
|
||||
issues.append("• 未设置 .env 文件路径")
|
||||
elif not Path(env_path).exists():
|
||||
issues.append("• .env 文件不存在")
|
||||
|
||||
if issues:
|
||||
self.validation_label.setText("❌ 配置问题:\n" + "\n".join(issues))
|
||||
self.validation_label.setStyleSheet("color: #d93025;")
|
||||
else:
|
||||
self.validation_label.setText("✅ 配置有效")
|
||||
self.validation_label.setStyleSheet("color: #1e8e3e;")
|
||||
|
||||
def _save_and_accept(self):
|
||||
"""保存并关闭"""
|
||||
project_path = self.project_path_edit.text().strip()
|
||||
env_path = self.env_path_edit.text().strip()
|
||||
|
||||
# 简单验证
|
||||
if project_path:
|
||||
p = Path(project_path)
|
||||
if not p.exists():
|
||||
QMessageBox.warning(self, "警告", "ETL 项目路径不存在")
|
||||
return
|
||||
if not (p / "cli" / "main.py").exists():
|
||||
reply = QMessageBox.question(
|
||||
self, "确认",
|
||||
"找不到 cli/main.py,确定要使用此路径吗?",
|
||||
QMessageBox.Yes | QMessageBox.No
|
||||
)
|
||||
if reply == QMessageBox.No:
|
||||
return
|
||||
|
||||
# 保存设置
|
||||
app_settings.etl_project_path = project_path
|
||||
if env_path:
|
||||
app_settings.env_file_path = env_path
|
||||
|
||||
self.accept()
|
||||
406
gui/widgets/status_panel.py
Normal file
406
gui/widgets/status_panel.py
Normal file
@@ -0,0 +1,406 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""ETL 状态面板"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||||
QGroupBox, QLabel, QPushButton, QTableWidget, QTableWidgetItem,
|
||||
QHeaderView, QFrame, QScrollArea, QMessageBox
|
||||
)
|
||||
from PySide6.QtCore import Qt, Signal, QTimer
|
||||
from PySide6.QtGui import QColor
|
||||
|
||||
from ..workers.db_worker import DBWorker
|
||||
from ..utils.config_helper import ConfigHelper
|
||||
|
||||
|
||||
class StatusCard(QFrame):
|
||||
"""状态卡片"""
|
||||
|
||||
def __init__(self, title: str, parent=None):
|
||||
super().__init__(parent)
|
||||
self.setProperty("card", True)
|
||||
self.setFrameShape(QFrame.StyledPanel)
|
||||
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(16, 12, 16, 12)
|
||||
layout.setSpacing(8)
|
||||
|
||||
# 标题
|
||||
self.title_label = QLabel(title)
|
||||
self.title_label.setProperty("subheading", True)
|
||||
layout.addWidget(self.title_label)
|
||||
|
||||
# 值
|
||||
self.value_label = QLabel("-")
|
||||
self.value_label.setStyleSheet("font-size: 24px; font-weight: bold;")
|
||||
layout.addWidget(self.value_label)
|
||||
|
||||
# 描述
|
||||
self.desc_label = QLabel("")
|
||||
self.desc_label.setProperty("subheading", True)
|
||||
layout.addWidget(self.desc_label)
|
||||
|
||||
def set_value(self, value: str, description: str = "", status: str = ""):
|
||||
"""设置值"""
|
||||
self.value_label.setText(value)
|
||||
self.desc_label.setText(description)
|
||||
|
||||
if status:
|
||||
self.value_label.setProperty("status", status)
|
||||
self.value_label.style().unpolish(self.value_label)
|
||||
self.value_label.style().polish(self.value_label)
|
||||
|
||||
|
||||
class StatusPanel(QWidget):
|
||||
"""ETL 状态面板"""
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.config_helper = ConfigHelper()
|
||||
self.db_worker = DBWorker(self)
|
||||
self._connected = False
|
||||
|
||||
self._init_ui()
|
||||
self._connect_signals()
|
||||
|
||||
# 定时刷新
|
||||
self.refresh_timer = QTimer(self)
|
||||
self.refresh_timer.timeout.connect(self._auto_refresh)
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(16, 16, 16, 16)
|
||||
layout.setSpacing(16)
|
||||
|
||||
# 标题和按钮
|
||||
header_layout = QHBoxLayout()
|
||||
|
||||
title = QLabel("ETL 状态")
|
||||
title.setProperty("heading", True)
|
||||
header_layout.addWidget(title)
|
||||
|
||||
header_layout.addStretch()
|
||||
|
||||
self.auto_refresh_btn = QPushButton("自动刷新: 关")
|
||||
self.auto_refresh_btn.setProperty("secondary", True)
|
||||
self.auto_refresh_btn.setCheckable(True)
|
||||
header_layout.addWidget(self.auto_refresh_btn)
|
||||
|
||||
self.refresh_btn = QPushButton("刷新")
|
||||
self.refresh_btn.clicked.connect(self._refresh_all)
|
||||
header_layout.addWidget(self.refresh_btn)
|
||||
|
||||
layout.addLayout(header_layout)
|
||||
|
||||
# 连接状态
|
||||
self.conn_status_label = QLabel("数据库: 未连接")
|
||||
self.conn_status_label.setProperty("status", "warning")
|
||||
layout.addWidget(self.conn_status_label)
|
||||
|
||||
# 滚动区域
|
||||
scroll_area = QScrollArea()
|
||||
scroll_area.setWidgetResizable(True)
|
||||
scroll_area.setFrameShape(QFrame.NoFrame)
|
||||
layout.addWidget(scroll_area, 1)
|
||||
|
||||
# 内容容器
|
||||
content_widget = QWidget()
|
||||
content_layout = QVBoxLayout(content_widget)
|
||||
content_layout.setSpacing(16)
|
||||
|
||||
# 概览卡片
|
||||
cards_layout = QHBoxLayout()
|
||||
|
||||
self.ods_card = StatusCard("ODS 表数量")
|
||||
cards_layout.addWidget(self.ods_card)
|
||||
|
||||
self.dwd_card = StatusCard("DWD 表数量")
|
||||
cards_layout.addWidget(self.dwd_card)
|
||||
|
||||
self.last_update_card = StatusCard("最后更新")
|
||||
cards_layout.addWidget(self.last_update_card)
|
||||
|
||||
self.task_count_card = StatusCard("今日任务")
|
||||
cards_layout.addWidget(self.task_count_card)
|
||||
|
||||
content_layout.addLayout(cards_layout)
|
||||
|
||||
# ODS Cutoff 状态
|
||||
cutoff_group = QGroupBox("ODS Cutoff 状态")
|
||||
cutoff_layout = QVBoxLayout(cutoff_group)
|
||||
|
||||
self.cutoff_table = QTableWidget()
|
||||
self.cutoff_table.setColumnCount(4)
|
||||
self.cutoff_table.setHorizontalHeaderLabels(["表名", "最新 fetched_at", "行数", "状态"])
|
||||
self.cutoff_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.Stretch)
|
||||
self.cutoff_table.setMaximumHeight(250)
|
||||
cutoff_layout.addWidget(self.cutoff_table)
|
||||
|
||||
content_layout.addWidget(cutoff_group)
|
||||
|
||||
# 最近运行记录
|
||||
history_group = QGroupBox("最近运行记录")
|
||||
history_layout = QVBoxLayout(history_group)
|
||||
|
||||
self.history_table = QTableWidget()
|
||||
self.history_table.setColumnCount(6)
|
||||
self.history_table.setHorizontalHeaderLabels(["运行ID", "任务", "状态", "开始时间", "耗时", "影响行数"])
|
||||
self.history_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.Stretch)
|
||||
self.history_table.setMaximumHeight(250)
|
||||
history_layout.addWidget(self.history_table)
|
||||
|
||||
content_layout.addWidget(history_group)
|
||||
|
||||
# 弹性空间
|
||||
content_layout.addStretch()
|
||||
|
||||
scroll_area.setWidget(content_widget)
|
||||
|
||||
def _connect_signals(self):
|
||||
"""连接信号"""
|
||||
self.auto_refresh_btn.toggled.connect(self._toggle_auto_refresh)
|
||||
self.db_worker.connection_status.connect(self._on_connection_status)
|
||||
self.db_worker.query_finished.connect(self._on_query_finished)
|
||||
self.db_worker.query_error.connect(self._on_query_error)
|
||||
|
||||
def _toggle_auto_refresh(self, checked: bool):
|
||||
"""切换自动刷新"""
|
||||
if checked:
|
||||
self.auto_refresh_btn.setText("自动刷新: 开")
|
||||
self.refresh_timer.start(30000) # 30秒刷新一次
|
||||
self._refresh_all()
|
||||
else:
|
||||
self.auto_refresh_btn.setText("自动刷新: 关")
|
||||
self.refresh_timer.stop()
|
||||
|
||||
def _auto_refresh(self):
|
||||
"""自动刷新"""
|
||||
if self._connected:
|
||||
self._refresh_all()
|
||||
|
||||
def _refresh_all(self):
|
||||
"""刷新所有数据"""
|
||||
# 尝试连接数据库
|
||||
if not self._connected:
|
||||
env_vars = self.config_helper.load_env()
|
||||
dsn = env_vars.get("PG_DSN", "")
|
||||
if dsn:
|
||||
self.db_worker.connect_db(dsn)
|
||||
else:
|
||||
self.conn_status_label.setText("数据库: 未配置 DSN")
|
||||
return
|
||||
else:
|
||||
self._load_status_data()
|
||||
|
||||
def _on_connection_status(self, connected: bool, message: str):
|
||||
"""处理连接状态"""
|
||||
self._connected = connected
|
||||
|
||||
if connected:
|
||||
self.conn_status_label.setText(f"数据库: 已连接")
|
||||
self.conn_status_label.setProperty("status", "success")
|
||||
self._load_status_data()
|
||||
else:
|
||||
self.conn_status_label.setText(f"数据库: {message}")
|
||||
self.conn_status_label.setProperty("status", "error")
|
||||
|
||||
self.conn_status_label.style().unpolish(self.conn_status_label)
|
||||
self.conn_status_label.style().polish(self.conn_status_label)
|
||||
|
||||
def _load_status_data(self):
|
||||
"""加载状态数据"""
|
||||
# 加载表统计
|
||||
self._current_query = "table_count"
|
||||
self.db_worker.execute_query("""
|
||||
SELECT
|
||||
table_schema,
|
||||
COUNT(*) as table_count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema IN ('billiards_ods', 'billiards_dwd', 'billiards_dws')
|
||||
GROUP BY table_schema
|
||||
""")
|
||||
|
||||
def _on_query_finished(self, columns: list, rows: list):
|
||||
"""处理查询结果"""
|
||||
query_type = getattr(self, '_current_query', '')
|
||||
|
||||
if query_type == "table_count":
|
||||
self._process_table_count(rows)
|
||||
# 继续加载 cutoff 数据
|
||||
self._current_query = "cutoff"
|
||||
self.db_worker.execute_query("""
|
||||
SELECT
|
||||
'payment_transactions' AS table_name,
|
||||
MAX(fetched_at) AS max_fetched_at,
|
||||
COUNT(*) AS row_count
|
||||
FROM billiards_ods.payment_transactions
|
||||
UNION ALL
|
||||
SELECT 'member_profiles', MAX(fetched_at), COUNT(*)
|
||||
FROM billiards_ods.member_profiles
|
||||
UNION ALL
|
||||
SELECT 'settlement_records', MAX(fetched_at), COUNT(*)
|
||||
FROM billiards_ods.settlement_records
|
||||
UNION ALL
|
||||
SELECT 'recharge_settlements', MAX(fetched_at), COUNT(*)
|
||||
FROM billiards_ods.recharge_settlements
|
||||
UNION ALL
|
||||
SELECT 'assistant_service_records', MAX(fetched_at), COUNT(*)
|
||||
FROM billiards_ods.assistant_service_records
|
||||
ORDER BY table_name
|
||||
""")
|
||||
elif query_type == "cutoff":
|
||||
self._process_cutoff_data(rows)
|
||||
# 继续加载运行历史
|
||||
self._current_query = "history"
|
||||
self.db_worker.execute_query("""
|
||||
SELECT
|
||||
run_id,
|
||||
task_code,
|
||||
status,
|
||||
started_at,
|
||||
finished_at,
|
||||
rows_affected
|
||||
FROM etl_admin.run_tracker
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 20
|
||||
""")
|
||||
elif query_type == "history":
|
||||
self._process_history_data(rows)
|
||||
self._current_query = ""
|
||||
|
||||
def _process_table_count(self, rows: list):
|
||||
"""处理表数量数据"""
|
||||
ods_count = 0
|
||||
dwd_count = 0
|
||||
|
||||
for row in rows:
|
||||
schema = row.get("table_schema", "")
|
||||
count = row.get("table_count", 0)
|
||||
|
||||
if schema == "billiards_ods":
|
||||
ods_count = count
|
||||
elif schema == "billiards_dwd":
|
||||
dwd_count = count
|
||||
|
||||
self.ods_card.set_value(str(ods_count), "个表")
|
||||
self.dwd_card.set_value(str(dwd_count), "个表")
|
||||
|
||||
def _process_cutoff_data(self, rows: list):
|
||||
"""处理 Cutoff 数据"""
|
||||
self.cutoff_table.setRowCount(len(rows))
|
||||
|
||||
latest_time = None
|
||||
now = datetime.now()
|
||||
|
||||
for row_idx, row in enumerate(rows):
|
||||
table_name = row.get("table_name", "")
|
||||
max_fetched = row.get("max_fetched_at")
|
||||
row_count = row.get("row_count", 0)
|
||||
|
||||
self.cutoff_table.setItem(row_idx, 0, QTableWidgetItem(table_name))
|
||||
|
||||
if max_fetched:
|
||||
time_str = str(max_fetched)[:19]
|
||||
self.cutoff_table.setItem(row_idx, 1, QTableWidgetItem(time_str))
|
||||
|
||||
# 更新最新时间
|
||||
if latest_time is None or max_fetched > latest_time:
|
||||
latest_time = max_fetched
|
||||
|
||||
# 计算状态
|
||||
if isinstance(max_fetched, datetime):
|
||||
hours_ago = (now - max_fetched).total_seconds() / 3600
|
||||
if hours_ago < 1:
|
||||
status = "正常"
|
||||
status_color = QColor("#1e8e3e")
|
||||
elif hours_ago < 24:
|
||||
status = "较新"
|
||||
status_color = QColor("#1a73e8")
|
||||
else:
|
||||
status = f"落后 {int(hours_ago)}h"
|
||||
status_color = QColor("#f9ab00")
|
||||
else:
|
||||
status = "-"
|
||||
status_color = QColor("#9aa0a6")
|
||||
else:
|
||||
self.cutoff_table.setItem(row_idx, 1, QTableWidgetItem("-"))
|
||||
status = "无数据"
|
||||
status_color = QColor("#d93025")
|
||||
|
||||
self.cutoff_table.setItem(row_idx, 2, QTableWidgetItem(str(row_count)))
|
||||
|
||||
status_item = QTableWidgetItem(status)
|
||||
status_item.setForeground(status_color)
|
||||
self.cutoff_table.setItem(row_idx, 3, status_item)
|
||||
|
||||
# 更新最后更新时间卡片
|
||||
if latest_time:
|
||||
time_str = str(latest_time)[:16]
|
||||
self.last_update_card.set_value(time_str, "")
|
||||
else:
|
||||
self.last_update_card.set_value("-", "无数据")
|
||||
|
||||
def _process_history_data(self, rows: list):
|
||||
"""处理运行历史数据"""
|
||||
self.history_table.setRowCount(len(rows))
|
||||
|
||||
today_count = 0
|
||||
today = datetime.now().date()
|
||||
|
||||
for row_idx, row in enumerate(rows):
|
||||
run_id = row.get("run_id", "")
|
||||
task_code = row.get("task_code", "")
|
||||
status = row.get("status", "")
|
||||
started_at = row.get("started_at")
|
||||
finished_at = row.get("finished_at")
|
||||
rows_affected = row.get("rows_affected", 0)
|
||||
|
||||
# 统计今日任务
|
||||
if started_at and isinstance(started_at, datetime):
|
||||
if started_at.date() == today:
|
||||
today_count += 1
|
||||
|
||||
self.history_table.setItem(row_idx, 0, QTableWidgetItem(str(run_id)[:8] if run_id else "-"))
|
||||
self.history_table.setItem(row_idx, 1, QTableWidgetItem(task_code))
|
||||
|
||||
# 状态
|
||||
status_item = QTableWidgetItem(status)
|
||||
if status and "success" in status.lower():
|
||||
status_item.setForeground(QColor("#1e8e3e"))
|
||||
elif status and ("fail" in status.lower() or "error" in status.lower()):
|
||||
status_item.setForeground(QColor("#d93025"))
|
||||
self.history_table.setItem(row_idx, 2, status_item)
|
||||
|
||||
# 开始时间
|
||||
time_str = str(started_at)[:19] if started_at else "-"
|
||||
self.history_table.setItem(row_idx, 3, QTableWidgetItem(time_str))
|
||||
|
||||
# 耗时
|
||||
if started_at and finished_at:
|
||||
try:
|
||||
duration = (finished_at - started_at).total_seconds()
|
||||
if duration < 60:
|
||||
duration_str = f"{duration:.1f}秒"
|
||||
else:
|
||||
duration_str = f"{int(duration // 60)}分{int(duration % 60)}秒"
|
||||
except:
|
||||
duration_str = "-"
|
||||
else:
|
||||
duration_str = "-"
|
||||
self.history_table.setItem(row_idx, 4, QTableWidgetItem(duration_str))
|
||||
|
||||
# 影响行数
|
||||
self.history_table.setItem(row_idx, 5, QTableWidgetItem(str(rows_affected or 0)))
|
||||
|
||||
# 更新今日任务卡片
|
||||
self.task_count_card.set_value(str(today_count), "次执行")
|
||||
|
||||
def _on_query_error(self, error: str):
|
||||
"""处理查询错误"""
|
||||
self._current_query = ""
|
||||
# 可能是表不存在,忽略错误继续
|
||||
pass
|
||||
1989
gui/widgets/task_manager.py
Normal file
1989
gui/widgets/task_manager.py
Normal file
File diff suppressed because it is too large
Load Diff
1218
gui/widgets/task_panel.py
Normal file
1218
gui/widgets/task_panel.py
Normal file
File diff suppressed because it is too large
Load Diff
550
gui/widgets/task_selector.py
Normal file
550
gui/widgets/task_selector.py
Normal file
@@ -0,0 +1,550 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""可复用的任务选择组件:按业务域分组显示,支持全选/反选。"""
|
||||
|
||||
from typing import Dict, List, Optional, Set
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QWidget, QVBoxLayout, QHBoxLayout, QGroupBox,
|
||||
QCheckBox, QPushButton, QScrollArea, QFrame,
|
||||
QLabel, QSizePolicy
|
||||
)
|
||||
from PySide6.QtCore import Signal, Qt
|
||||
|
||||
from ..models.task_registry import (
|
||||
TaskRegistry, TaskDefinition, BusinessDomain, DOMAIN_LABELS,
|
||||
task_registry, get_fact_ods_task_codes, get_dimension_ods_task_codes,
|
||||
DwdTableDefinition, DWD_TABLE_DEFINITIONS, DWD_TABLE_DOMAIN_ORDER,
|
||||
get_dwd_tables_grouped, get_all_dwd_table_codes,
|
||||
)
|
||||
|
||||
|
||||
class TaskSelectorWidget(QWidget):
|
||||
"""ODS 任务选择组件:按业务域分组显示"""
|
||||
|
||||
# 选择变化信号
|
||||
selection_changed = Signal(list) # 选中的任务编码列表
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
parent: Optional[QWidget] = None,
|
||||
show_dimensions: bool = True,
|
||||
show_facts: bool = True,
|
||||
default_select_facts: bool = True,
|
||||
default_select_dimensions: bool = False,
|
||||
compact: bool = False,
|
||||
max_height: int = 0,
|
||||
):
|
||||
"""
|
||||
初始化任务选择器
|
||||
|
||||
Args:
|
||||
parent: 父组件
|
||||
show_dimensions: 是否显示维度类任务
|
||||
show_facts: 是否显示事实类任务
|
||||
default_select_facts: 默认选中事实类任务
|
||||
default_select_dimensions: 默认选中维度类任务
|
||||
compact: 紧凑模式(更小的间距)
|
||||
max_height: 最大高度(0 表示不限制)
|
||||
"""
|
||||
super().__init__(parent)
|
||||
self.show_dimensions = show_dimensions
|
||||
self.show_facts = show_facts
|
||||
self.default_select_facts = default_select_facts
|
||||
self.default_select_dimensions = default_select_dimensions
|
||||
self.compact = compact
|
||||
self.max_height = max_height
|
||||
|
||||
# 任务复选框映射:code -> QCheckBox
|
||||
self._checkboxes: Dict[str, QCheckBox] = {}
|
||||
# 业务域分组框映射:domain -> QGroupBox
|
||||
self._domain_groups: Dict[BusinessDomain, QGroupBox] = {}
|
||||
|
||||
self._init_ui()
|
||||
self._apply_default_selection()
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(0, 0, 0, 0)
|
||||
spacing = 4 if self.compact else 8
|
||||
layout.setSpacing(spacing)
|
||||
|
||||
# 顶部工具栏
|
||||
toolbar = QHBoxLayout()
|
||||
toolbar.setSpacing(8)
|
||||
|
||||
self.select_all_btn = QPushButton("全选")
|
||||
self.select_all_btn.setProperty("secondary", True)
|
||||
self.select_all_btn.setFixedWidth(60)
|
||||
self.select_all_btn.clicked.connect(self._select_all)
|
||||
toolbar.addWidget(self.select_all_btn)
|
||||
|
||||
self.deselect_all_btn = QPushButton("全不选")
|
||||
self.deselect_all_btn.setProperty("secondary", True)
|
||||
self.deselect_all_btn.setFixedWidth(60)
|
||||
self.deselect_all_btn.clicked.connect(self._deselect_all)
|
||||
toolbar.addWidget(self.deselect_all_btn)
|
||||
|
||||
self.select_facts_btn = QPushButton("选事实表")
|
||||
self.select_facts_btn.setProperty("secondary", True)
|
||||
self.select_facts_btn.setFixedWidth(70)
|
||||
self.select_facts_btn.setToolTip("选中所有事实类任务(需要时间窗口的任务)")
|
||||
self.select_facts_btn.clicked.connect(self._select_facts_only)
|
||||
toolbar.addWidget(self.select_facts_btn)
|
||||
|
||||
toolbar.addStretch()
|
||||
|
||||
self.selected_count_label = QLabel("已选: 0")
|
||||
self.selected_count_label.setProperty("subheading", True)
|
||||
toolbar.addWidget(self.selected_count_label)
|
||||
|
||||
layout.addLayout(toolbar)
|
||||
|
||||
# 内容容器
|
||||
content_widget = QWidget()
|
||||
content_layout = QVBoxLayout(content_widget)
|
||||
content_layout.setContentsMargins(0, 0, 0, 0)
|
||||
content_layout.setSpacing(spacing)
|
||||
|
||||
# 按业务域分组创建复选框
|
||||
grouped_tasks = task_registry.get_ods_tasks_grouped()
|
||||
|
||||
# 定义业务域显示顺序
|
||||
domain_order = [
|
||||
BusinessDomain.MEMBER,
|
||||
BusinessDomain.SETTLEMENT,
|
||||
BusinessDomain.ASSISTANT,
|
||||
BusinessDomain.GOODS,
|
||||
BusinessDomain.TABLE,
|
||||
BusinessDomain.PROMOTION,
|
||||
BusinessDomain.INVENTORY,
|
||||
]
|
||||
|
||||
for domain in domain_order:
|
||||
if domain not in grouped_tasks:
|
||||
continue
|
||||
|
||||
tasks = grouped_tasks[domain]
|
||||
# 过滤任务
|
||||
filtered_tasks = []
|
||||
for task in tasks:
|
||||
if task.is_dimension and not self.show_dimensions:
|
||||
continue
|
||||
if not task.is_dimension and not self.show_facts:
|
||||
continue
|
||||
filtered_tasks.append(task)
|
||||
|
||||
if not filtered_tasks:
|
||||
continue
|
||||
|
||||
# 创建业务域分组
|
||||
group_box = self._create_domain_group(domain, filtered_tasks)
|
||||
self._domain_groups[domain] = group_box
|
||||
content_layout.addWidget(group_box)
|
||||
|
||||
content_layout.addStretch()
|
||||
|
||||
if self.max_height > 0:
|
||||
# 需要限制高度时启用滚动区域
|
||||
scroll_area = QScrollArea()
|
||||
scroll_area.setWidgetResizable(True)
|
||||
scroll_area.setFrameShape(QFrame.NoFrame)
|
||||
scroll_area.setMaximumHeight(self.max_height)
|
||||
scroll_area.setWidget(content_widget)
|
||||
layout.addWidget(scroll_area, 1)
|
||||
else:
|
||||
# 全量展示,不使用内部滚动
|
||||
layout.addWidget(content_widget)
|
||||
|
||||
def _create_domain_group(self, domain: BusinessDomain, tasks: List[TaskDefinition]) -> QGroupBox:
|
||||
"""创建业务域分组框"""
|
||||
group_box = QGroupBox(DOMAIN_LABELS.get(domain, str(domain.value)))
|
||||
group_layout = QVBoxLayout(group_box)
|
||||
group_layout.setContentsMargins(8, 4, 8, 4)
|
||||
group_layout.setSpacing(2)
|
||||
|
||||
for task in tasks:
|
||||
checkbox = QCheckBox(f"{task.name}")
|
||||
checkbox.setToolTip(f"{task.code}: {task.description}")
|
||||
checkbox.setProperty("task_code", task.code)
|
||||
checkbox.setProperty("is_dimension", task.is_dimension)
|
||||
checkbox.stateChanged.connect(self._on_selection_changed)
|
||||
|
||||
self._checkboxes[task.code] = checkbox
|
||||
group_layout.addWidget(checkbox)
|
||||
|
||||
return group_box
|
||||
|
||||
def _apply_default_selection(self):
|
||||
"""应用默认选择"""
|
||||
for code, checkbox in self._checkboxes.items():
|
||||
is_dimension = checkbox.property("is_dimension")
|
||||
if is_dimension:
|
||||
checkbox.setChecked(self.default_select_dimensions)
|
||||
else:
|
||||
checkbox.setChecked(self.default_select_facts)
|
||||
|
||||
self._update_count_label()
|
||||
|
||||
def _on_selection_changed(self):
|
||||
"""选择变化时"""
|
||||
self._update_count_label()
|
||||
self.selection_changed.emit(self.get_selected_codes())
|
||||
|
||||
def _update_count_label(self):
|
||||
"""更新选中计数标签"""
|
||||
count = len(self.get_selected_codes())
|
||||
total = len(self._checkboxes)
|
||||
self.selected_count_label.setText(f"已选: {count}/{total}")
|
||||
|
||||
def _select_all(self):
|
||||
"""全选"""
|
||||
for checkbox in self._checkboxes.values():
|
||||
checkbox.blockSignals(True)
|
||||
checkbox.setChecked(True)
|
||||
checkbox.blockSignals(False)
|
||||
self._on_selection_changed()
|
||||
|
||||
def _deselect_all(self):
|
||||
"""全不选"""
|
||||
for checkbox in self._checkboxes.values():
|
||||
checkbox.blockSignals(True)
|
||||
checkbox.setChecked(False)
|
||||
checkbox.blockSignals(False)
|
||||
self._on_selection_changed()
|
||||
|
||||
def _select_facts_only(self):
|
||||
"""只选事实表任务"""
|
||||
for code, checkbox in self._checkboxes.items():
|
||||
checkbox.blockSignals(True)
|
||||
is_dimension = checkbox.property("is_dimension")
|
||||
checkbox.setChecked(not is_dimension)
|
||||
checkbox.blockSignals(False)
|
||||
self._on_selection_changed()
|
||||
|
||||
def get_selected_codes(self) -> List[str]:
|
||||
"""获取选中的任务编码列表"""
|
||||
selected = []
|
||||
for code, checkbox in self._checkboxes.items():
|
||||
if checkbox.isChecked():
|
||||
selected.append(code)
|
||||
return selected
|
||||
|
||||
def set_selected_codes(self, codes: List[str]):
|
||||
"""设置选中的任务编码"""
|
||||
codes_set = set(codes)
|
||||
for code, checkbox in self._checkboxes.items():
|
||||
checkbox.blockSignals(True)
|
||||
checkbox.setChecked(code in codes_set)
|
||||
checkbox.blockSignals(False)
|
||||
self._on_selection_changed()
|
||||
|
||||
def get_all_codes(self) -> List[str]:
|
||||
"""获取所有任务编码"""
|
||||
return list(self._checkboxes.keys())
|
||||
|
||||
def is_any_selected(self) -> bool:
|
||||
"""是否有任何任务被选中"""
|
||||
return len(self.get_selected_codes()) > 0
|
||||
|
||||
|
||||
class CompactTaskSelector(QWidget):
|
||||
"""紧凑型任务选择器:单行显示业务域,点击展开选择"""
|
||||
|
||||
selection_changed = Signal(list)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
parent: Optional[QWidget] = None,
|
||||
show_dimensions: bool = True,
|
||||
show_facts: bool = True,
|
||||
default_select_facts: bool = True,
|
||||
default_select_dimensions: bool = False,
|
||||
):
|
||||
super().__init__(parent)
|
||||
self.show_dimensions = show_dimensions
|
||||
self.show_facts = show_facts
|
||||
self.default_select_facts = default_select_facts
|
||||
self.default_select_dimensions = default_select_dimensions
|
||||
|
||||
# 业务域复选框
|
||||
self._domain_checkboxes: Dict[BusinessDomain, QCheckBox] = {}
|
||||
# 业务域下的任务编码
|
||||
self._domain_tasks: Dict[BusinessDomain, List[str]] = {}
|
||||
|
||||
self._init_ui()
|
||||
self._apply_default_selection()
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(0, 0, 0, 0)
|
||||
layout.setSpacing(4)
|
||||
|
||||
# 工具栏
|
||||
toolbar = QHBoxLayout()
|
||||
toolbar.setSpacing(8)
|
||||
|
||||
self.select_all_btn = QPushButton("全选")
|
||||
self.select_all_btn.setProperty("secondary", True)
|
||||
self.select_all_btn.setFixedWidth(50)
|
||||
self.select_all_btn.clicked.connect(self._select_all)
|
||||
toolbar.addWidget(self.select_all_btn)
|
||||
|
||||
self.deselect_all_btn = QPushButton("清空")
|
||||
self.deselect_all_btn.setProperty("secondary", True)
|
||||
self.deselect_all_btn.setFixedWidth(50)
|
||||
self.deselect_all_btn.clicked.connect(self._deselect_all)
|
||||
toolbar.addWidget(self.deselect_all_btn)
|
||||
|
||||
toolbar.addStretch()
|
||||
|
||||
self.count_label = QLabel("已选: 0")
|
||||
self.count_label.setProperty("subheading", True)
|
||||
toolbar.addWidget(self.count_label)
|
||||
|
||||
layout.addLayout(toolbar)
|
||||
|
||||
# 业务域复选框(横向排列)
|
||||
domains_layout = QHBoxLayout()
|
||||
domains_layout.setSpacing(12)
|
||||
|
||||
grouped_tasks = task_registry.get_ods_tasks_grouped()
|
||||
domain_order = [
|
||||
BusinessDomain.MEMBER,
|
||||
BusinessDomain.SETTLEMENT,
|
||||
BusinessDomain.ASSISTANT,
|
||||
BusinessDomain.GOODS,
|
||||
BusinessDomain.TABLE,
|
||||
BusinessDomain.PROMOTION,
|
||||
BusinessDomain.INVENTORY,
|
||||
]
|
||||
|
||||
for domain in domain_order:
|
||||
if domain not in grouped_tasks:
|
||||
continue
|
||||
|
||||
tasks = grouped_tasks[domain]
|
||||
# 过滤任务
|
||||
task_codes = []
|
||||
for task in tasks:
|
||||
if task.is_dimension and not self.show_dimensions:
|
||||
continue
|
||||
if not task.is_dimension and not self.show_facts:
|
||||
continue
|
||||
task_codes.append(task.code)
|
||||
|
||||
if not task_codes:
|
||||
continue
|
||||
|
||||
self._domain_tasks[domain] = task_codes
|
||||
|
||||
checkbox = QCheckBox(DOMAIN_LABELS.get(domain, str(domain.value)))
|
||||
checkbox.setToolTip(f"包含: {', '.join(task_codes)}")
|
||||
checkbox.stateChanged.connect(self._on_selection_changed)
|
||||
self._domain_checkboxes[domain] = checkbox
|
||||
domains_layout.addWidget(checkbox)
|
||||
|
||||
domains_layout.addStretch()
|
||||
layout.addLayout(domains_layout)
|
||||
|
||||
def _apply_default_selection(self):
|
||||
"""应用默认选择"""
|
||||
# 默认选中所有业务域
|
||||
for domain, checkbox in self._domain_checkboxes.items():
|
||||
checkbox.setChecked(True)
|
||||
self._update_count_label()
|
||||
|
||||
def _on_selection_changed(self):
|
||||
"""选择变化时"""
|
||||
self._update_count_label()
|
||||
self.selection_changed.emit(self.get_selected_codes())
|
||||
|
||||
def _update_count_label(self):
|
||||
"""更新计数标签"""
|
||||
count = len(self.get_selected_codes())
|
||||
self.count_label.setText(f"已选: {count} 个任务")
|
||||
|
||||
def _select_all(self):
|
||||
"""全选所有业务域"""
|
||||
for checkbox in self._domain_checkboxes.values():
|
||||
checkbox.blockSignals(True)
|
||||
checkbox.setChecked(True)
|
||||
checkbox.blockSignals(False)
|
||||
self._on_selection_changed()
|
||||
|
||||
def _deselect_all(self):
|
||||
"""取消全选"""
|
||||
for checkbox in self._domain_checkboxes.values():
|
||||
checkbox.blockSignals(True)
|
||||
checkbox.setChecked(False)
|
||||
checkbox.blockSignals(False)
|
||||
self._on_selection_changed()
|
||||
|
||||
def get_selected_codes(self) -> List[str]:
|
||||
"""获取选中的任务编码"""
|
||||
selected = []
|
||||
for domain, checkbox in self._domain_checkboxes.items():
|
||||
if checkbox.isChecked():
|
||||
selected.extend(self._domain_tasks.get(domain, []))
|
||||
return selected
|
||||
|
||||
def set_selected_domains(self, domains: List[BusinessDomain]):
|
||||
"""设置选中的业务域"""
|
||||
domains_set = set(domains)
|
||||
for domain, checkbox in self._domain_checkboxes.items():
|
||||
checkbox.blockSignals(True)
|
||||
checkbox.setChecked(domain in domains_set)
|
||||
checkbox.blockSignals(False)
|
||||
self._on_selection_changed()
|
||||
|
||||
def is_any_selected(self) -> bool:
|
||||
"""是否有任何任务被选中"""
|
||||
return len(self.get_selected_codes()) > 0
|
||||
|
||||
|
||||
class DwdTableSelectorWidget(QWidget):
|
||||
"""DWD 表选择组件:按业务域分组显示,类似 ODS 任务选择器。
|
||||
|
||||
每个复选框对应一组 DWD 表(主表 + _ex 扩展表),
|
||||
默认全选,不使用内部滚动。
|
||||
"""
|
||||
|
||||
# 选择变化信号:发射选中的 DWD 表编码列表
|
||||
selection_changed = Signal(list)
|
||||
|
||||
def __init__(self, parent: Optional[QWidget] = None):
|
||||
super().__init__(parent)
|
||||
# code -> QCheckBox
|
||||
self._checkboxes: Dict[str, QCheckBox] = {}
|
||||
# domain -> QGroupBox
|
||||
self._domain_groups: Dict[BusinessDomain, QGroupBox] = {}
|
||||
self._init_ui()
|
||||
# 默认全选
|
||||
self._select_all()
|
||||
|
||||
# ------------------------------------------------------------------ UI
|
||||
def _init_ui(self):
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(0, 0, 0, 0)
|
||||
layout.setSpacing(4)
|
||||
|
||||
# 工具栏
|
||||
toolbar = QHBoxLayout()
|
||||
toolbar.setSpacing(8)
|
||||
|
||||
self.select_all_btn = QPushButton("全选")
|
||||
self.select_all_btn.setProperty("secondary", True)
|
||||
self.select_all_btn.setFixedWidth(60)
|
||||
self.select_all_btn.clicked.connect(self._select_all)
|
||||
toolbar.addWidget(self.select_all_btn)
|
||||
|
||||
self.deselect_all_btn = QPushButton("全不选")
|
||||
self.deselect_all_btn.setProperty("secondary", True)
|
||||
self.deselect_all_btn.setFixedWidth(60)
|
||||
self.deselect_all_btn.clicked.connect(self._deselect_all)
|
||||
toolbar.addWidget(self.deselect_all_btn)
|
||||
|
||||
self.select_facts_btn = QPushButton("选事实表")
|
||||
self.select_facts_btn.setProperty("secondary", True)
|
||||
self.select_facts_btn.setFixedWidth(70)
|
||||
self.select_facts_btn.setToolTip("仅选中事实表,取消维度表")
|
||||
self.select_facts_btn.clicked.connect(self._select_facts_only)
|
||||
toolbar.addWidget(self.select_facts_btn)
|
||||
|
||||
toolbar.addStretch()
|
||||
|
||||
self.selected_count_label = QLabel("已选: 0")
|
||||
self.selected_count_label.setProperty("subheading", True)
|
||||
toolbar.addWidget(self.selected_count_label)
|
||||
|
||||
layout.addLayout(toolbar)
|
||||
|
||||
# 按业务域分组
|
||||
grouped = get_dwd_tables_grouped()
|
||||
for domain in DWD_TABLE_DOMAIN_ORDER:
|
||||
tables = grouped.get(domain)
|
||||
if not tables:
|
||||
continue
|
||||
group_box = self._create_domain_group(domain, tables)
|
||||
self._domain_groups[domain] = group_box
|
||||
layout.addWidget(group_box)
|
||||
|
||||
def _create_domain_group(
|
||||
self, domain: BusinessDomain, tables: List[DwdTableDefinition]
|
||||
) -> QGroupBox:
|
||||
group_box = QGroupBox(DOMAIN_LABELS.get(domain, str(domain.value)))
|
||||
group_layout = QVBoxLayout(group_box)
|
||||
group_layout.setContentsMargins(8, 4, 8, 4)
|
||||
group_layout.setSpacing(2)
|
||||
|
||||
for tbl in tables:
|
||||
tag = "[维]" if tbl.is_dimension else "[事]"
|
||||
checkbox = QCheckBox(f"{tag} {tbl.name}")
|
||||
checkbox.setToolTip(
|
||||
f"{tbl.code}: {tbl.description}\n表: {', '.join(tbl.tables)}"
|
||||
)
|
||||
checkbox.setProperty("table_code", tbl.code)
|
||||
checkbox.setProperty("is_dimension", tbl.is_dimension)
|
||||
checkbox.stateChanged.connect(self._on_selection_changed)
|
||||
self._checkboxes[tbl.code] = checkbox
|
||||
group_layout.addWidget(checkbox)
|
||||
|
||||
return group_box
|
||||
|
||||
# -------------------------------------------------------------- 交互
|
||||
def _on_selection_changed(self):
|
||||
self._update_count_label()
|
||||
self.selection_changed.emit(self.get_selected_codes())
|
||||
|
||||
def _update_count_label(self):
|
||||
count = len(self.get_selected_codes())
|
||||
total = len(self._checkboxes)
|
||||
self.selected_count_label.setText(f"已选: {count}/{total}")
|
||||
|
||||
def _select_all(self):
|
||||
for cb in self._checkboxes.values():
|
||||
cb.blockSignals(True)
|
||||
cb.setChecked(True)
|
||||
cb.blockSignals(False)
|
||||
self._on_selection_changed()
|
||||
|
||||
def _deselect_all(self):
|
||||
for cb in self._checkboxes.values():
|
||||
cb.blockSignals(True)
|
||||
cb.setChecked(False)
|
||||
cb.blockSignals(False)
|
||||
self._on_selection_changed()
|
||||
|
||||
def _select_facts_only(self):
|
||||
for cb in self._checkboxes.values():
|
||||
cb.blockSignals(True)
|
||||
cb.setChecked(not cb.property("is_dimension"))
|
||||
cb.blockSignals(False)
|
||||
self._on_selection_changed()
|
||||
|
||||
# -------------------------------------------------------------- API
|
||||
def get_selected_codes(self) -> List[str]:
|
||||
"""返回选中的 DWD 表编码列表(如 ['dim_member', 'dwd_payment', ...])"""
|
||||
return [code for code, cb in self._checkboxes.items() if cb.isChecked()]
|
||||
|
||||
def set_selected_codes(self, codes: List[str]):
|
||||
"""设置选中的 DWD 表编码"""
|
||||
codes_set = set(codes)
|
||||
for code, cb in self._checkboxes.items():
|
||||
cb.blockSignals(True)
|
||||
cb.setChecked(code in codes_set)
|
||||
cb.blockSignals(False)
|
||||
self._on_selection_changed()
|
||||
|
||||
def get_all_codes(self) -> List[str]:
|
||||
"""获取所有 DWD 表编码"""
|
||||
return list(self._checkboxes.keys())
|
||||
|
||||
def is_all_selected(self) -> bool:
|
||||
"""是否全部选中"""
|
||||
return len(self.get_selected_codes()) == len(self._checkboxes)
|
||||
|
||||
def is_any_selected(self) -> bool:
|
||||
"""是否有选中"""
|
||||
return len(self.get_selected_codes()) > 0
|
||||
7
gui/workers/__init__.py
Normal file
7
gui/workers/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""后台工作线程模块"""
|
||||
|
||||
from .task_worker import TaskWorker
|
||||
from .db_worker import DBWorker
|
||||
|
||||
__all__ = ["TaskWorker", "DBWorker"]
|
||||
192
gui/workers/db_worker.py
Normal file
192
gui/workers/db_worker.py
Normal file
@@ -0,0 +1,192 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""数据库查询工作线程"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
|
||||
from PySide6.QtCore import QThread, Signal
|
||||
|
||||
# 添加项目路径
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
|
||||
class DBWorker(QThread):
|
||||
"""数据库查询工作线程"""
|
||||
|
||||
# 信号
|
||||
query_finished = Signal(list, list) # 查询完成 (columns, rows)
|
||||
query_error = Signal(str) # 查询错误
|
||||
connection_status = Signal(bool, str) # 连接状态 (connected, message)
|
||||
tables_loaded = Signal(dict) # 表列表加载完成 {schema: [(table, rows, updated_at), ...]}
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.conn = None
|
||||
self._task = None
|
||||
self._task_args = None
|
||||
|
||||
def connect_db(self, dsn: str):
|
||||
"""连接数据库"""
|
||||
self._task = "connect"
|
||||
self._task_args = (dsn,)
|
||||
self.start()
|
||||
|
||||
def disconnect_db(self):
|
||||
"""断开数据库连接"""
|
||||
self._task = "disconnect"
|
||||
self._task_args = None
|
||||
self.start()
|
||||
|
||||
def execute_query(self, sql: str, params: Optional[tuple] = None):
|
||||
"""执行查询"""
|
||||
self._task = "query"
|
||||
self._task_args = (sql, params)
|
||||
self.start()
|
||||
|
||||
def load_tables(self, schemas: Optional[List[str]] = None):
|
||||
"""加载表列表"""
|
||||
self._task = "load_tables"
|
||||
self._task_args = (schemas,)
|
||||
self.start()
|
||||
|
||||
def run(self):
|
||||
"""执行任务"""
|
||||
if self._task == "connect":
|
||||
self._do_connect(*self._task_args)
|
||||
elif self._task == "disconnect":
|
||||
self._do_disconnect()
|
||||
elif self._task == "query":
|
||||
self._do_query(*self._task_args)
|
||||
elif self._task == "load_tables":
|
||||
self._do_load_tables(*self._task_args)
|
||||
|
||||
def _do_connect(self, dsn: str):
|
||||
"""执行连接"""
|
||||
try:
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
self.conn = psycopg2.connect(dsn, connect_timeout=10)
|
||||
self.conn.set_session(autocommit=True)
|
||||
|
||||
# 测试连接
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("SELECT version()")
|
||||
version = cur.fetchone()[0]
|
||||
|
||||
self.connection_status.emit(True, f"已连接: {version[:50]}...")
|
||||
except ImportError:
|
||||
self.connection_status.emit(False, "缺少 psycopg2 模块,请安装: pip install psycopg2-binary")
|
||||
except Exception as e:
|
||||
self.conn = None
|
||||
self.connection_status.emit(False, f"连接失败: {e}")
|
||||
|
||||
def _do_disconnect(self):
|
||||
"""执行断开连接"""
|
||||
if self.conn:
|
||||
try:
|
||||
self.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.conn = None
|
||||
self.connection_status.emit(False, "已断开连接")
|
||||
|
||||
def _do_query(self, sql: str, params: Optional[tuple]):
|
||||
"""执行查询"""
|
||||
if not self.conn:
|
||||
self.query_error.emit("未连接到数据库")
|
||||
return
|
||||
|
||||
try:
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
with self.conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute(sql, params)
|
||||
|
||||
# 检查是否有结果
|
||||
if cur.description:
|
||||
columns = [desc[0] for desc in cur.description]
|
||||
rows = [dict(row) for row in cur.fetchall()]
|
||||
self.query_finished.emit(columns, rows)
|
||||
else:
|
||||
self.query_finished.emit([], [])
|
||||
except Exception as e:
|
||||
self.query_error.emit(f"查询失败: {e}")
|
||||
|
||||
def _do_load_tables(self, schemas: Optional[List[str]]):
|
||||
"""加载表列表"""
|
||||
if not self.conn:
|
||||
self.query_error.emit("未连接到数据库")
|
||||
return
|
||||
|
||||
try:
|
||||
if schemas is None:
|
||||
schemas = ["billiards_ods", "billiards_dwd", "billiards_dws", "etl_admin"]
|
||||
|
||||
result = {}
|
||||
|
||||
for schema in schemas:
|
||||
tables = []
|
||||
|
||||
# 获取表列表
|
||||
sql = """
|
||||
SELECT
|
||||
t.table_name,
|
||||
COALESCE(s.n_live_tup, 0) as row_count
|
||||
FROM information_schema.tables t
|
||||
LEFT JOIN pg_stat_user_tables s
|
||||
ON t.table_name = s.relname
|
||||
AND t.table_schema = s.schemaname
|
||||
WHERE t.table_schema = %s
|
||||
AND t.table_type = 'BASE TABLE'
|
||||
ORDER BY t.table_name
|
||||
"""
|
||||
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute(sql, (schema,))
|
||||
for row in cur.fetchall():
|
||||
table_name = row[0]
|
||||
row_count = row[1] or 0
|
||||
|
||||
# 尝试获取最新更新时间
|
||||
updated_at = None
|
||||
try:
|
||||
# 尝试 fetched_at 字段
|
||||
cur.execute(f'SELECT MAX(fetched_at) FROM "{schema}"."{table_name}"')
|
||||
result_row = cur.fetchone()
|
||||
if result_row and result_row[0]:
|
||||
updated_at = str(result_row[0])[:19]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not updated_at:
|
||||
try:
|
||||
# 尝试 updated_at 字段
|
||||
cur.execute(f'SELECT MAX(updated_at) FROM "{schema}"."{table_name}"')
|
||||
result_row = cur.fetchone()
|
||||
if result_row and result_row[0]:
|
||||
updated_at = str(result_row[0])[:19]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
tables.append((table_name, row_count, updated_at or "-"))
|
||||
|
||||
result[schema] = tables
|
||||
|
||||
self.tables_loaded.emit(result)
|
||||
except Exception as e:
|
||||
self.query_error.emit(f"加载表列表失败: {e}")
|
||||
|
||||
def is_connected(self) -> bool:
|
||||
"""检查是否已连接"""
|
||||
if not self.conn:
|
||||
return False
|
||||
try:
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("SELECT 1")
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
378
gui/workers/task_worker.py
Normal file
378
gui/workers/task_worker.py
Normal file
@@ -0,0 +1,378 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""任务执行工作线程"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Dict
|
||||
|
||||
from PySide6.QtCore import QThread, Signal
|
||||
|
||||
from ..utils.app_settings import app_settings
|
||||
|
||||
|
||||
class TaskWorker(QThread):
|
||||
"""任务执行工作线程"""
|
||||
|
||||
# 信号
|
||||
output_received = Signal(str) # 收到输出行
|
||||
task_finished = Signal(int, str) # 任务完成 (exit_code, summary)
|
||||
error_occurred = Signal(str) # 发生错误
|
||||
progress_updated = Signal(int, int) # 进度更新 (current, total)
|
||||
|
||||
def __init__(self, command: List[str], working_dir: Optional[str] = None,
|
||||
extra_env: Optional[Dict[str, str]] = None, parent=None):
|
||||
super().__init__(parent)
|
||||
self.command = command
|
||||
self.extra_env = extra_env or {}
|
||||
|
||||
# 工作目录优先级: 参数 > 应用设置 > 自动检测
|
||||
if working_dir is not None:
|
||||
self.working_dir = working_dir
|
||||
elif app_settings.etl_project_path:
|
||||
self.working_dir = app_settings.etl_project_path
|
||||
else:
|
||||
# 回退到源码目录
|
||||
self.working_dir = str(Path(__file__).resolve().parents[2])
|
||||
|
||||
self.process: Optional[subprocess.Popen] = None
|
||||
self._stop_requested = False
|
||||
self._exit_code: Optional[int] = None
|
||||
self._output_lines: List[str] = []
|
||||
|
||||
def run(self):
|
||||
"""执行任务"""
|
||||
try:
|
||||
self._stop_requested = False
|
||||
self._output_lines = []
|
||||
|
||||
# 设置环境变量
|
||||
env = os.environ.copy()
|
||||
env["PYTHONIOENCODING"] = "utf-8"
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
|
||||
# 添加项目根目录到 PYTHONPATH
|
||||
project_root = self.working_dir
|
||||
existing_path = env.get("PYTHONPATH", "")
|
||||
if existing_path:
|
||||
env["PYTHONPATH"] = f"{project_root}{os.pathsep}{existing_path}"
|
||||
else:
|
||||
env["PYTHONPATH"] = project_root
|
||||
|
||||
# 添加额外的环境变量
|
||||
if self.extra_env:
|
||||
for key, value in self.extra_env.items():
|
||||
env[key] = str(value)
|
||||
self.output_received.emit(f"[环境变量] {key}={value}")
|
||||
|
||||
self.output_received.emit(f"[工作目录] {self.working_dir}")
|
||||
self.output_received.emit(f"[执行命令] {' '.join(self.command)}")
|
||||
|
||||
# 启动进程
|
||||
self.process = subprocess.Popen(
|
||||
self.command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
cwd=self.working_dir,
|
||||
env=env,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == "win32" else 0,
|
||||
)
|
||||
|
||||
# 读取输出
|
||||
if self.process.stdout:
|
||||
for line in iter(self.process.stdout.readline, ""):
|
||||
if self._stop_requested:
|
||||
break
|
||||
|
||||
line = line.rstrip("\n\r")
|
||||
if line:
|
||||
self._output_lines.append(line)
|
||||
self.output_received.emit(line)
|
||||
|
||||
# 解析进度信息(如果有)
|
||||
self._parse_progress(line)
|
||||
|
||||
# 等待进程结束
|
||||
if self.process:
|
||||
self.process.wait()
|
||||
self._exit_code = self.process.returncode
|
||||
|
||||
# 生成摘要
|
||||
summary = self._generate_summary()
|
||||
self.task_finished.emit(self._exit_code or 0, summary)
|
||||
|
||||
except FileNotFoundError as e:
|
||||
self.error_occurred.emit(f"找不到 Python 解释器: {e}")
|
||||
self.task_finished.emit(-1, f"执行失败: {e}")
|
||||
except Exception as e:
|
||||
self.error_occurred.emit(f"执行出错: {e}")
|
||||
self.task_finished.emit(-1, f"执行失败: {e}")
|
||||
finally:
|
||||
self.process = None
|
||||
|
||||
def stop(self):
|
||||
"""停止任务"""
|
||||
self._stop_requested = True
|
||||
if self.process:
|
||||
try:
|
||||
self.process.terminate()
|
||||
# 给进程一些时间来终止
|
||||
try:
|
||||
self.process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.process.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _parse_progress(self, line: str):
|
||||
"""解析进度信息"""
|
||||
# 尝试从日志中解析进度
|
||||
# 示例: "[INFO] 处理进度: 50/100"
|
||||
import re
|
||||
match = re.search(r'进度[:\s]*(\d+)/(\d+)', line)
|
||||
if match:
|
||||
current = int(match.group(1))
|
||||
total = int(match.group(2))
|
||||
self.progress_updated.emit(current, total)
|
||||
|
||||
def _generate_summary(self) -> str:
|
||||
"""生成执行摘要"""
|
||||
if not self._output_lines:
|
||||
return "无输出"
|
||||
|
||||
return self._parse_detailed_summary()
|
||||
|
||||
def _parse_detailed_summary(self) -> str:
|
||||
"""解析详细的执行摘要"""
|
||||
import re
|
||||
import json
|
||||
|
||||
summary_parts = []
|
||||
|
||||
# 统计各类信息
|
||||
ods_stats = [] # ODS 抓取统计
|
||||
dwd_stats = [] # DWD 装载统计
|
||||
import_stats = [] # 导入任务统计
|
||||
integrity_stats = {} # 数据校验统计
|
||||
errors = [] # 错误信息
|
||||
task_results = [] # 任务结果
|
||||
|
||||
for line in self._output_lines:
|
||||
# 1. 解析 ODS 抓取完成信息
|
||||
# 格式: "xxx: 抓取完成,文件=xxx,记录数=123"
|
||||
match = re.search(r'(\w+): 抓取完成.*记录数[=:]\s*(\d+)', line)
|
||||
if match:
|
||||
task_name = match.group(1)
|
||||
record_count = int(match.group(2))
|
||||
if record_count > 0:
|
||||
ods_stats.append(f"{task_name}: {record_count}条")
|
||||
continue
|
||||
|
||||
# 2. 解析 DWD 装载完成信息
|
||||
# 格式: "DWD 装载完成:xxx,用时 1.02s"
|
||||
match = re.search(r'DWD 装载完成[::]\s*(\S+).*用时\s*([\d.]+)s', line)
|
||||
if match:
|
||||
table_name = match.group(1).replace('billiards_dwd.', '')
|
||||
continue
|
||||
|
||||
# 3. 解析任务完成统计 (JSON格式)
|
||||
# 格式: "xxx: 完成,统计={'tables': [...]}"
|
||||
if "完成,统计=" in line or "完成,统计=" in line:
|
||||
try:
|
||||
match = re.search(r"统计=(\{.+\})", line)
|
||||
if match:
|
||||
stats_str = match.group(1).replace("'", '"')
|
||||
stats = json.loads(stats_str)
|
||||
|
||||
# 解析 DWD 装载统计
|
||||
if 'tables' in stats:
|
||||
total_dim_inserted = 0
|
||||
total_dim_updated = 0
|
||||
total_fact_inserted = 0
|
||||
total_fact_updated = 0
|
||||
|
||||
dim_tables = [] # 维表明细
|
||||
fact_tables = [] # 事实表明细
|
||||
|
||||
for tbl in stats['tables']:
|
||||
table_name = tbl.get('table', '').replace('billiards_dwd.', '')
|
||||
mode = tbl.get('mode', '')
|
||||
processed = int(tbl.get('processed', 0) or 0)
|
||||
inserted = int(tbl.get('inserted', 0) or 0)
|
||||
updated = int(tbl.get('updated', 0) or 0)
|
||||
has_new_counts = ('inserted' in tbl) or ('updated' in tbl)
|
||||
|
||||
# 忽略 _ex 扩展表
|
||||
if table_name.endswith('_ex'):
|
||||
continue
|
||||
|
||||
is_dim = table_name.startswith('dim_') or mode == 'SCD2'
|
||||
if is_dim:
|
||||
if has_new_counts:
|
||||
total_dim_inserted += inserted
|
||||
total_dim_updated += updated
|
||||
if inserted or updated:
|
||||
dim_tables.append(f"{table_name}: +{inserted}, ~{updated}")
|
||||
elif processed > 0:
|
||||
total_dim_updated += processed
|
||||
dim_tables.append(f"{table_name}: {processed}")
|
||||
else:
|
||||
if has_new_counts:
|
||||
total_fact_inserted += inserted
|
||||
total_fact_updated += updated
|
||||
if inserted or updated:
|
||||
fact_tables.append(f"{table_name}: +{inserted}, ~{updated}")
|
||||
elif processed > 0 or inserted > 0:
|
||||
total_fact_inserted += inserted
|
||||
if inserted > 0:
|
||||
fact_tables.append(f"{table_name}: +{inserted}")
|
||||
|
||||
if (total_dim_inserted or total_dim_updated or total_fact_inserted or total_fact_updated):
|
||||
dwd_stats.append(
|
||||
f"维表新增: {total_dim_inserted}条, 维表更新: {total_dim_updated}条, "
|
||||
f"事实表新增: {total_fact_inserted}条, 事实表更新: {total_fact_updated}条"
|
||||
)
|
||||
|
||||
# 维表明细
|
||||
if dim_tables:
|
||||
dwd_stats.append(" 维表: " + ", ".join(dim_tables))
|
||||
|
||||
# 事实表明细
|
||||
if fact_tables:
|
||||
dwd_stats.append(" 事实表: " + ", ".join(fact_tables))
|
||||
|
||||
# 解析 ML 台账导入/关系指数等轻量统计
|
||||
if any(k in stats for k in ("source_rows", "alloc_rows", "scopes", "records_inserted")):
|
||||
source_rows = int(stats.get("source_rows", 0) or 0)
|
||||
alloc_rows = int(stats.get("alloc_rows", 0) or 0)
|
||||
scopes = int(stats.get("scopes", 0) or 0)
|
||||
records_inserted = int(stats.get("records_inserted", 0) or 0)
|
||||
|
||||
if source_rows or alloc_rows or scopes:
|
||||
import_stats.append(
|
||||
f"ML台账导入: source={source_rows}, alloc={alloc_rows}, scopes={scopes}"
|
||||
)
|
||||
if records_inserted:
|
||||
import_stats.append(f"关系指数写入: {records_inserted}条")
|
||||
|
||||
|
||||
|
||||
# 解析错误信息
|
||||
if 'errors' in stats and stats['errors']:
|
||||
for err in stats['errors']:
|
||||
err_table = err.get('table', '').replace('billiards_dwd.', '')
|
||||
err_msg = err.get('error', '')
|
||||
errors.append(f"{err_table}: {err_msg}")
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
|
||||
# 4. 解析数据校验结果
|
||||
# 格式: "CHECK_DONE task=xxx missing=1 records=136 errors=0"
|
||||
match = re.search(r'CHECK_DONE task=(\w+) missing=(\d+) records=(\d+)', line)
|
||||
if match:
|
||||
task_name = match.group(1)
|
||||
missing = int(match.group(2))
|
||||
records = int(match.group(3))
|
||||
if missing > 0:
|
||||
if 'missing_tasks' not in integrity_stats:
|
||||
integrity_stats['missing_tasks'] = []
|
||||
integrity_stats['missing_tasks'].append(f"{task_name}: 缺失{missing}/{records}")
|
||||
integrity_stats['total_records'] = integrity_stats.get('total_records', 0) + records
|
||||
integrity_stats['total_missing'] = integrity_stats.get('total_missing', 0) + missing
|
||||
continue
|
||||
|
||||
# 5. 解析数据校验最终结果
|
||||
# 格式: "结果统计: {'missing': 463, 'errors': 0, 'backfilled': 0}"
|
||||
if "结果统计:" in line or "结果统计:" in line:
|
||||
try:
|
||||
match = re.search(r"\{.+\}", line)
|
||||
if match:
|
||||
stats_str = match.group(0).replace("'", '"')
|
||||
stats = json.loads(stats_str)
|
||||
integrity_stats['final_missing'] = stats.get('missing', 0)
|
||||
integrity_stats['final_errors'] = stats.get('errors', 0)
|
||||
integrity_stats['backfilled'] = stats.get('backfilled', 0)
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
|
||||
# 6. 解析错误信息
|
||||
if "[ERROR]" in line or "错误" in line.lower() or "error" in line.lower():
|
||||
if "Traceback" not in line and "File " not in line:
|
||||
errors.append(line.strip()[:100])
|
||||
|
||||
# 7. 解析任务完成信息
|
||||
if "任务执行成功" in line or "ETL运行完成" in line:
|
||||
task_results.append("✓ " + line.split("]")[-1].strip() if "]" in line else line.strip())
|
||||
elif "任务执行失败" in line:
|
||||
task_results.append("✗ " + line.split("]")[-1].strip() if "]" in line else line.strip())
|
||||
|
||||
# 构建摘要
|
||||
if ods_stats:
|
||||
summary_parts.append("【ODS 抓取】" + ", ".join(ods_stats[:5]))
|
||||
if len(ods_stats) > 5:
|
||||
summary_parts[-1] += f" 等{len(ods_stats)}项"
|
||||
|
||||
if dwd_stats:
|
||||
summary_parts.append("【DWD 装载】" + dwd_stats[0]) # 第一行是汇总
|
||||
for detail in dwd_stats[1:]: # 后面是详情
|
||||
summary_parts.append(detail)
|
||||
|
||||
if import_stats:
|
||||
summary_parts.append("【导入/指数】" + ";".join(import_stats[:3]))
|
||||
|
||||
if integrity_stats:
|
||||
total_missing = integrity_stats.get('final_missing', integrity_stats.get('total_missing', 0))
|
||||
total_records = integrity_stats.get('total_records', 0)
|
||||
backfilled = integrity_stats.get('backfilled', 0)
|
||||
|
||||
int_summary = f"【数据校验】检查 {total_records} 条记录"
|
||||
if total_missing > 0:
|
||||
int_summary += f", 发现 {total_missing} 条缺失"
|
||||
if backfilled > 0:
|
||||
int_summary += f", 已补全 {backfilled} 条"
|
||||
else:
|
||||
int_summary += ", 数据完整"
|
||||
summary_parts.append(int_summary)
|
||||
|
||||
# 显示缺失详情
|
||||
if integrity_stats.get('missing_tasks'):
|
||||
missing_detail = integrity_stats['missing_tasks'][:3]
|
||||
summary_parts.append(" 缺失: " + "; ".join(missing_detail))
|
||||
if len(integrity_stats['missing_tasks']) > 3:
|
||||
summary_parts[-1] += f" 等{len(integrity_stats['missing_tasks'])}项"
|
||||
|
||||
if errors:
|
||||
summary_parts.append("【错误】" + "; ".join(errors[:3]))
|
||||
|
||||
if task_results:
|
||||
summary_parts.append("【结果】" + " | ".join(task_results))
|
||||
|
||||
if summary_parts:
|
||||
return "\n".join(summary_parts)
|
||||
|
||||
# 如果没有解析到任何信息,返回最后几行关键信息
|
||||
key_lines = []
|
||||
for line in self._output_lines[-10:]:
|
||||
if "完成" in line or "成功" in line or "失败" in line:
|
||||
key_lines.append(line.strip()[:80])
|
||||
|
||||
if key_lines:
|
||||
return "\n".join(key_lines[-3:])
|
||||
|
||||
return self._output_lines[-1] if self._output_lines else "执行完成"
|
||||
|
||||
@property
|
||||
def exit_code(self) -> Optional[int]:
|
||||
"""获取退出码"""
|
||||
return self._exit_code
|
||||
|
||||
@property
|
||||
def output(self) -> str:
|
||||
"""获取完整输出"""
|
||||
return "\n".join(self._output_lines)
|
||||
Reference in New Issue
Block a user