合并
This commit is contained in:
141
etl_billiards/gui/README.md
Normal file
141
etl_billiards/gui/README.md
Normal file
@@ -0,0 +1,141 @@
|
||||
# 飞球 ETL GUI 管理系统
|
||||
|
||||
一个基于 PySide6 的图形化 ETL 管理工具。
|
||||
|
||||
## 功能特性
|
||||
|
||||
- **任务配置**: 选择和配置 ETL 任务,支持参数设置和 CLI 命令预览
|
||||
- **任务管理**: 任务队列管理、执行历史记录、自动执行
|
||||
- **环境配置**: 图形化编辑 `.env` 配置文件
|
||||
- **数据库查看**: 浏览表结构、执行 SQL 查询
|
||||
- **ETL 状态**: 实时查看 ODS/DWD 数据状态和执行记录
|
||||
- **日志查看**: 实时日志输出、过滤、导出
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 1. 安装依赖
|
||||
|
||||
```bash
|
||||
cd etl_billiards
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
### 2. 运行 GUI
|
||||
|
||||
**方法一:使用启动脚本**
|
||||
|
||||
```bash
|
||||
# Windows 命令行
|
||||
run_gui.bat
|
||||
|
||||
# 或 PowerShell
|
||||
.\run_gui.ps1
|
||||
```
|
||||
|
||||
**方法二:直接运行 Python**
|
||||
|
||||
```bash
|
||||
cd etl_billiards
|
||||
python -m gui.main
|
||||
```
|
||||
|
||||
## 打包为 EXE
|
||||
|
||||
### 安装打包工具
|
||||
|
||||
```bash
|
||||
pip install pyinstaller
|
||||
```
|
||||
|
||||
### 执行打包
|
||||
|
||||
```bash
|
||||
# 目录模式(推荐,启动更快)
|
||||
python build_exe.py
|
||||
|
||||
# 单文件模式
|
||||
python build_exe.py --onefile
|
||||
|
||||
# 显示控制台(调试用)
|
||||
python build_exe.py --console
|
||||
|
||||
# 清理并重新打包
|
||||
python build_exe.py --clean
|
||||
```
|
||||
|
||||
打包完成后,EXE 文件位于 `dist/ETL管理系统/` 目录。
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
gui/
|
||||
├── main.py # 应用入口
|
||||
├── main_window.py # 主窗口
|
||||
├── widgets/ # UI 组件
|
||||
│ ├── task_panel.py # 任务配置面板
|
||||
│ ├── task_manager.py # 任务管理器
|
||||
│ ├── env_editor.py # 环境变量编辑器
|
||||
│ ├── log_viewer.py # 日志查看器
|
||||
│ ├── db_viewer.py # 数据库查看器
|
||||
│ └── status_panel.py # ETL 状态面板
|
||||
├── workers/ # 后台工作线程
|
||||
│ ├── task_worker.py # 任务执行线程
|
||||
│ └── db_worker.py # 数据库查询线程
|
||||
├── models/ # 数据模型
|
||||
│ └── task_model.py # 任务数据模型
|
||||
├── utils/ # 工具模块
|
||||
│ ├── cli_builder.py # CLI 命令构建器
|
||||
│ └── config_helper.py # 配置辅助
|
||||
└── resources/ # 资源文件
|
||||
└── styles.qss # 样式表
|
||||
```
|
||||
|
||||
## 使用说明
|
||||
|
||||
### 任务配置
|
||||
|
||||
1. 在左侧选择任务分类
|
||||
2. 勾选要执行的任务
|
||||
3. 配置运行参数(Pipeline 模式、时间窗口等)
|
||||
4. 查看底部的 CLI 命令预览
|
||||
5. 点击「立即执行」或「添加到队列」
|
||||
|
||||
### 环境配置
|
||||
|
||||
1. 打开「环境配置」面板
|
||||
2. 编辑各项配置(数据库、API、路径等)
|
||||
3. 点击「保存」
|
||||
|
||||
### 数据库查看
|
||||
|
||||
1. 打开「数据库」面板
|
||||
2. 输入或使用 .env 中的 DSN
|
||||
3. 点击「连接」
|
||||
4. 浏览表结构或执行 SQL 查询
|
||||
|
||||
## 常见问题
|
||||
|
||||
### Q: 启动时提示缺少 PySide6
|
||||
|
||||
```bash
|
||||
pip install PySide6
|
||||
```
|
||||
|
||||
### Q: 连接数据库失败
|
||||
|
||||
检查 `.env` 中的 `PG_DSN` 配置是否正确。
|
||||
|
||||
### Q: 打包后运行闪退
|
||||
|
||||
使用 `--console` 参数重新打包,查看错误信息:
|
||||
|
||||
```bash
|
||||
python build_exe.py --console
|
||||
```
|
||||
|
||||
## 技术栈
|
||||
|
||||
- Python 3.10+
|
||||
- PySide6 (Qt for Python)
|
||||
- psycopg2 (PostgreSQL)
|
||||
- PyInstaller (打包)
|
||||
5
etl_billiards/gui/__init__.py
Normal file
5
etl_billiards/gui/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""ETL GUI 客户端模块"""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
__author__ = "ETL Team"
|
||||
46
etl_billiards/gui/main.py
Normal file
46
etl_billiards/gui/main.py
Normal file
@@ -0,0 +1,46 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""ETL GUI 应用入口"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# 确保项目根目录在 Python 路径中
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from PySide6.QtWidgets import QApplication
|
||||
from PySide6.QtCore import Qt
|
||||
from PySide6.QtGui import QFont
|
||||
|
||||
from gui.main_window import MainWindow
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
# 设置高 DPI 支持
|
||||
QApplication.setHighDpiScaleFactorRoundingPolicy(
|
||||
Qt.HighDpiScaleFactorRoundingPolicy.PassThrough
|
||||
)
|
||||
|
||||
# 创建应用
|
||||
app = QApplication(sys.argv)
|
||||
app.setApplicationName("飞球 ETL 管理系统")
|
||||
app.setApplicationVersion("1.0.0")
|
||||
app.setOrganizationName("Billiards")
|
||||
|
||||
# 设置默认字体
|
||||
font = QFont("Microsoft YaHei", 10)
|
||||
app.setFont(font)
|
||||
|
||||
# 创建主窗口
|
||||
window = MainWindow()
|
||||
window.show()
|
||||
|
||||
# 运行应用
|
||||
sys.exit(app.exec())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
397
etl_billiards/gui/main_window.py
Normal file
397
etl_billiards/gui/main_window.py
Normal file
@@ -0,0 +1,397 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""主窗口"""
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QMainWindow, QWidget, QVBoxLayout, QHBoxLayout,
|
||||
QStackedWidget, QListWidget, QListWidgetItem,
|
||||
QStatusBar, QLabel, QMessageBox, QSplitter
|
||||
)
|
||||
from PySide6.QtCore import Qt, QSize, Signal
|
||||
from PySide6.QtGui import QIcon, QAction
|
||||
|
||||
from .widgets.task_panel import TaskPanel
|
||||
from .widgets.task_manager import TaskManager
|
||||
from .widgets.env_editor import EnvEditor
|
||||
from .widgets.log_viewer import LogViewer
|
||||
from .widgets.db_viewer import DBViewer
|
||||
from .widgets.status_panel import StatusPanel
|
||||
from .resources import load_stylesheet
|
||||
|
||||
|
||||
class MainWindow(QMainWindow):
|
||||
"""ETL GUI 主窗口"""
|
||||
|
||||
# 信号
|
||||
status_message = Signal(str, int) # message, timeout_ms
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.setWindowTitle("飞球 ETL 管理系统")
|
||||
self.setMinimumSize(1200, 800)
|
||||
self.resize(1400, 900)
|
||||
|
||||
# 应用样式
|
||||
self.setStyleSheet(load_stylesheet())
|
||||
|
||||
# 初始化 UI
|
||||
self._init_ui()
|
||||
self._init_menu()
|
||||
self._init_status_bar()
|
||||
self._connect_signals()
|
||||
|
||||
# 默认选中第一项
|
||||
self.nav_list.setCurrentRow(0)
|
||||
|
||||
# 首次显示标记
|
||||
self._first_show = True
|
||||
|
||||
def showEvent(self, event):
|
||||
"""窗口显示事件"""
|
||||
super().showEvent(event)
|
||||
if self._first_show:
|
||||
self._first_show = False
|
||||
# 延迟检查配置,让窗口先显示
|
||||
from PySide6.QtCore import QTimer
|
||||
QTimer.singleShot(100, self._check_config_on_startup)
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
# 中央部件
|
||||
central_widget = QWidget()
|
||||
self.setCentralWidget(central_widget)
|
||||
|
||||
# 主布局
|
||||
main_layout = QHBoxLayout(central_widget)
|
||||
main_layout.setContentsMargins(0, 0, 0, 0)
|
||||
main_layout.setSpacing(0)
|
||||
|
||||
# 创建分割器
|
||||
splitter = QSplitter(Qt.Horizontal)
|
||||
main_layout.addWidget(splitter)
|
||||
|
||||
# 左侧导航栏
|
||||
nav_widget = self._create_nav_widget()
|
||||
splitter.addWidget(nav_widget)
|
||||
|
||||
# 右侧内容区
|
||||
self.content_stack = QStackedWidget()
|
||||
splitter.addWidget(self.content_stack)
|
||||
|
||||
# 设置分割比例
|
||||
splitter.setSizes([200, 1200])
|
||||
splitter.setStretchFactor(0, 0)
|
||||
splitter.setStretchFactor(1, 1)
|
||||
|
||||
# 创建各个面板
|
||||
self._create_panels()
|
||||
|
||||
def _create_nav_widget(self) -> QWidget:
|
||||
"""创建导航侧边栏"""
|
||||
nav_widget = QWidget()
|
||||
nav_widget.setMaximumWidth(220)
|
||||
nav_widget.setMinimumWidth(180)
|
||||
|
||||
layout = QVBoxLayout(nav_widget)
|
||||
layout.setContentsMargins(0, 0, 0, 0)
|
||||
layout.setSpacing(0)
|
||||
|
||||
# 标题
|
||||
title_label = QLabel(" ETL 控制台")
|
||||
title_label.setProperty("heading", True)
|
||||
title_label.setFixedHeight(60)
|
||||
title_label.setAlignment(Qt.AlignVCenter)
|
||||
layout.addWidget(title_label)
|
||||
|
||||
# 导航列表
|
||||
self.nav_list = QListWidget()
|
||||
self.nav_list.setObjectName("navList")
|
||||
layout.addWidget(self.nav_list)
|
||||
|
||||
# 添加导航项
|
||||
nav_items = [
|
||||
("任务配置", "配置并执行 ETL 任务"),
|
||||
("任务管理", "管理任务队列和历史记录"),
|
||||
("环境配置", "编辑 .env 配置文件"),
|
||||
("数据库", "查看数据库和执行查询"),
|
||||
("ETL 状态", "查看 ETL 运行状态"),
|
||||
("日志", "查看执行日志"),
|
||||
]
|
||||
|
||||
for name, tooltip in nav_items:
|
||||
item = QListWidgetItem(name)
|
||||
item.setToolTip(tooltip)
|
||||
item.setSizeHint(QSize(0, 44))
|
||||
self.nav_list.addItem(item)
|
||||
|
||||
return nav_widget
|
||||
|
||||
def _create_panels(self):
|
||||
"""创建各个功能面板"""
|
||||
# 任务配置面板
|
||||
self.task_panel = TaskPanel()
|
||||
self.content_stack.addWidget(self.task_panel)
|
||||
|
||||
# 任务管理面板
|
||||
self.task_manager = TaskManager()
|
||||
self.content_stack.addWidget(self.task_manager)
|
||||
|
||||
# 环境配置面板
|
||||
self.env_editor = EnvEditor()
|
||||
self.content_stack.addWidget(self.env_editor)
|
||||
|
||||
# 数据库查看器
|
||||
self.db_viewer = DBViewer()
|
||||
self.content_stack.addWidget(self.db_viewer)
|
||||
|
||||
# ETL 状态面板
|
||||
self.status_panel = StatusPanel()
|
||||
self.content_stack.addWidget(self.status_panel)
|
||||
|
||||
# 日志面板
|
||||
self.log_viewer = LogViewer()
|
||||
self.content_stack.addWidget(self.log_viewer)
|
||||
|
||||
def _init_menu(self):
|
||||
"""初始化菜单栏"""
|
||||
menubar = self.menuBar()
|
||||
|
||||
# 文件菜单
|
||||
file_menu = menubar.addMenu("文件(&F)")
|
||||
|
||||
refresh_action = QAction("刷新配置(&R)", self)
|
||||
refresh_action.setShortcut("Ctrl+R")
|
||||
refresh_action.triggered.connect(self._refresh_config)
|
||||
file_menu.addAction(refresh_action)
|
||||
|
||||
settings_action = QAction("设置(&S)...", self)
|
||||
settings_action.setShortcut("Ctrl+,")
|
||||
settings_action.triggered.connect(self._show_settings)
|
||||
file_menu.addAction(settings_action)
|
||||
|
||||
file_menu.addSeparator()
|
||||
|
||||
exit_action = QAction("退出(&X)", self)
|
||||
exit_action.setShortcut("Ctrl+Q")
|
||||
exit_action.triggered.connect(self.close)
|
||||
file_menu.addAction(exit_action)
|
||||
|
||||
# 视图菜单
|
||||
view_menu = menubar.addMenu("视图(&V)")
|
||||
|
||||
task_config_action = QAction("任务配置(&T)", self)
|
||||
task_config_action.setShortcut("Ctrl+1")
|
||||
task_config_action.triggered.connect(lambda: self._switch_panel(0))
|
||||
view_menu.addAction(task_config_action)
|
||||
|
||||
task_manager_action = QAction("任务管理(&M)", self)
|
||||
task_manager_action.setShortcut("Ctrl+2")
|
||||
task_manager_action.triggered.connect(lambda: self._switch_panel(1))
|
||||
view_menu.addAction(task_manager_action)
|
||||
|
||||
env_action = QAction("环境配置(&E)", self)
|
||||
env_action.setShortcut("Ctrl+3")
|
||||
env_action.triggered.connect(lambda: self._switch_panel(2))
|
||||
view_menu.addAction(env_action)
|
||||
|
||||
db_action = QAction("数据库(&D)", self)
|
||||
db_action.setShortcut("Ctrl+4")
|
||||
db_action.triggered.connect(lambda: self._switch_panel(3))
|
||||
view_menu.addAction(db_action)
|
||||
|
||||
status_action = QAction("ETL 状态(&S)", self)
|
||||
status_action.setShortcut("Ctrl+5")
|
||||
status_action.triggered.connect(lambda: self._switch_panel(4))
|
||||
view_menu.addAction(status_action)
|
||||
|
||||
log_action = QAction("日志(&L)", self)
|
||||
log_action.setShortcut("Ctrl+6")
|
||||
log_action.triggered.connect(lambda: self._switch_panel(5))
|
||||
view_menu.addAction(log_action)
|
||||
|
||||
# 帮助菜单
|
||||
help_menu = menubar.addMenu("帮助(&H)")
|
||||
|
||||
about_action = QAction("关于(&A)", self)
|
||||
about_action.triggered.connect(self._show_about)
|
||||
help_menu.addAction(about_action)
|
||||
|
||||
def _init_status_bar(self):
|
||||
"""初始化状态栏"""
|
||||
self.status_bar = QStatusBar()
|
||||
self.setStatusBar(self.status_bar)
|
||||
|
||||
# 连接状态
|
||||
self.conn_status_label = QLabel("数据库: 未连接")
|
||||
self.conn_status_label.setProperty("status", "warning")
|
||||
self.status_bar.addPermanentWidget(self.conn_status_label)
|
||||
|
||||
# 任务状态
|
||||
self.task_status_label = QLabel("任务: 空闲")
|
||||
self.status_bar.addPermanentWidget(self.task_status_label)
|
||||
|
||||
# 默认消息
|
||||
self.status_bar.showMessage("就绪", 3000)
|
||||
|
||||
def _connect_signals(self):
|
||||
"""连接信号"""
|
||||
# 导航切换
|
||||
self.nav_list.currentRowChanged.connect(self._on_nav_changed)
|
||||
|
||||
# 任务面板信号
|
||||
self.task_panel.task_started.connect(self._on_task_started)
|
||||
self.task_panel.task_finished.connect(self._on_task_finished)
|
||||
self.task_panel.log_message.connect(self.log_viewer.append_log)
|
||||
self.task_panel.add_to_queue.connect(self._on_add_to_queue)
|
||||
self.task_panel.create_schedule.connect(self._on_create_schedule)
|
||||
|
||||
# 任务管理器信号
|
||||
self.task_manager.task_started.connect(self._on_task_started)
|
||||
self.task_manager.task_finished.connect(self._on_task_finished)
|
||||
self.task_manager.log_message.connect(self.log_viewer.append_log)
|
||||
|
||||
# 数据库连接状态
|
||||
self.db_viewer.connection_changed.connect(self._on_db_connection_changed)
|
||||
|
||||
# 状态消息
|
||||
self.status_message.connect(self._show_status_message)
|
||||
|
||||
def _on_nav_changed(self, index: int):
|
||||
"""导航项切换"""
|
||||
self.content_stack.setCurrentIndex(index)
|
||||
|
||||
def _switch_panel(self, index: int):
|
||||
"""切换到指定面板"""
|
||||
self.nav_list.setCurrentRow(index)
|
||||
|
||||
def _refresh_config(self):
|
||||
"""刷新配置"""
|
||||
self.env_editor.load_config()
|
||||
self.task_panel.refresh_tasks()
|
||||
self.status_bar.showMessage("配置已刷新", 3000)
|
||||
|
||||
def _on_task_started(self, task_info: str):
|
||||
"""任务开始时"""
|
||||
self.task_status_label.setText(f"任务: 执行中 - {task_info}")
|
||||
self.task_status_label.setProperty("status", "info")
|
||||
self.task_status_label.style().unpolish(self.task_status_label)
|
||||
self.task_status_label.style().polish(self.task_status_label)
|
||||
|
||||
def _on_task_finished(self, success: bool, message: str):
|
||||
"""任务完成时"""
|
||||
if success:
|
||||
self.task_status_label.setText("任务: 完成")
|
||||
self.task_status_label.setProperty("status", "success")
|
||||
else:
|
||||
self.task_status_label.setText("任务: 失败")
|
||||
self.task_status_label.setProperty("status", "error")
|
||||
self.task_status_label.style().unpolish(self.task_status_label)
|
||||
self.task_status_label.style().polish(self.task_status_label)
|
||||
self.status_bar.showMessage(message, 5000)
|
||||
|
||||
def _on_db_connection_changed(self, connected: bool, message: str):
|
||||
"""数据库连接状态变化"""
|
||||
if connected:
|
||||
self.conn_status_label.setText("数据库: 已连接")
|
||||
self.conn_status_label.setProperty("status", "success")
|
||||
else:
|
||||
self.conn_status_label.setText("数据库: 未连接")
|
||||
self.conn_status_label.setProperty("status", "warning")
|
||||
self.conn_status_label.style().unpolish(self.conn_status_label)
|
||||
self.conn_status_label.style().polish(self.conn_status_label)
|
||||
if message:
|
||||
self.status_bar.showMessage(message, 3000)
|
||||
|
||||
def _show_status_message(self, message: str, timeout: int):
|
||||
"""显示状态栏消息"""
|
||||
self.status_bar.showMessage(message, timeout)
|
||||
|
||||
def _on_add_to_queue(self, config):
|
||||
"""添加任务到队列"""
|
||||
task_id = self.task_manager.add_task(config)
|
||||
self.status_bar.showMessage(f"任务已添加到队列 (ID: {task_id})", 3000)
|
||||
|
||||
def _on_create_schedule(self, name: str, task_codes: list, task_config: dict):
|
||||
"""创建调度任务"""
|
||||
# 打开调度编辑对话框
|
||||
from .widgets.task_manager import ScheduleEditDialog
|
||||
from .models.schedule_model import ScheduledTask, ScheduleConfig
|
||||
import uuid
|
||||
|
||||
# 创建一个预填充的调度任务
|
||||
task = ScheduledTask(
|
||||
id=str(uuid.uuid4())[:8],
|
||||
name=name,
|
||||
task_codes=task_codes,
|
||||
schedule=ScheduleConfig(),
|
||||
task_config=task_config,
|
||||
)
|
||||
|
||||
# 打开编辑对话框
|
||||
dialog = ScheduleEditDialog(task=task, parent=self)
|
||||
if dialog.exec():
|
||||
updated_task = dialog.get_task()
|
||||
if updated_task:
|
||||
self.task_manager.schedule_store.add_task(updated_task)
|
||||
self.task_manager._refresh_schedule_table()
|
||||
self.status_bar.showMessage(f"调度任务已创建: {updated_task.name}", 3000)
|
||||
# 切换到任务管理面板的调度选项卡
|
||||
self._switch_panel(1)
|
||||
|
||||
def _show_settings(self):
|
||||
"""显示设置对话框"""
|
||||
from .widgets.settings_dialog import SettingsDialog
|
||||
dialog = SettingsDialog(self)
|
||||
if dialog.exec():
|
||||
# 重新加载配置
|
||||
self._refresh_config()
|
||||
self.status_bar.showMessage("设置已保存", 3000)
|
||||
|
||||
def _check_config_on_startup(self):
|
||||
"""启动时检查配置"""
|
||||
from .utils.app_settings import app_settings
|
||||
if not app_settings.is_configured():
|
||||
QMessageBox.information(
|
||||
self,
|
||||
"首次配置",
|
||||
"欢迎使用 ETL 管理系统!\n\n"
|
||||
"请先配置 ETL 项目路径,否则无法执行任务。\n\n"
|
||||
"点击 文件 → 设置 进行配置。"
|
||||
)
|
||||
|
||||
def _show_about(self):
|
||||
"""显示关于对话框"""
|
||||
QMessageBox.about(
|
||||
self,
|
||||
"关于 飞球 ETL 管理系统",
|
||||
"<h3>飞球 ETL 管理系统</h3>"
|
||||
"<p>版本: 1.0.0</p>"
|
||||
"<p>一个用于管理台球场门店数据 ETL 的图形化工具。</p>"
|
||||
"<p>功能包括:</p>"
|
||||
"<ul>"
|
||||
"<li>任务配置与执行</li>"
|
||||
"<li>环境变量管理</li>"
|
||||
"<li>数据库查询</li>"
|
||||
"<li>ETL 状态监控</li>"
|
||||
"</ul>"
|
||||
)
|
||||
|
||||
def closeEvent(self, event):
|
||||
"""关闭事件"""
|
||||
# 检查是否有正在运行的任务
|
||||
if hasattr(self, 'task_panel') and self.task_panel.is_running():
|
||||
reply = QMessageBox.question(
|
||||
self,
|
||||
"确认退出",
|
||||
"当前有任务正在执行,确定要退出吗?",
|
||||
QMessageBox.Yes | QMessageBox.No,
|
||||
QMessageBox.No
|
||||
)
|
||||
if reply == QMessageBox.No:
|
||||
event.ignore()
|
||||
return
|
||||
|
||||
# 关闭数据库连接
|
||||
if hasattr(self, 'db_viewer'):
|
||||
self.db_viewer.close_connection()
|
||||
|
||||
event.accept()
|
||||
20
etl_billiards/gui/models/__init__.py
Normal file
20
etl_billiards/gui/models/__init__.py
Normal file
@@ -0,0 +1,20 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""数据模型模块"""
|
||||
|
||||
from .task_model import TaskItem, TaskStatus, TaskHistory, TaskConfig, QueuedTask
|
||||
from .schedule_model import (
|
||||
ScheduledTask, ScheduleConfig, ScheduleType, IntervalUnit, ScheduleStore
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"TaskItem",
|
||||
"TaskStatus",
|
||||
"TaskHistory",
|
||||
"TaskConfig",
|
||||
"QueuedTask",
|
||||
"ScheduledTask",
|
||||
"ScheduleConfig",
|
||||
"ScheduleType",
|
||||
"IntervalUnit",
|
||||
"ScheduleStore",
|
||||
]
|
||||
391
etl_billiards/gui/models/schedule_model.py
Normal file
391
etl_billiards/gui/models/schedule_model.py
Normal file
@@ -0,0 +1,391 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""调度任务数据模型"""
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime, timedelta
|
||||
from enum import Enum
|
||||
from typing import Optional, List, Dict, Any
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ScheduleType(Enum):
|
||||
"""调度类型"""
|
||||
ONCE = "once" # 一次性
|
||||
INTERVAL = "interval" # 固定间隔
|
||||
DAILY = "daily" # 每天
|
||||
WEEKLY = "weekly" # 每周
|
||||
CRON = "cron" # Cron 表达式
|
||||
|
||||
|
||||
class IntervalUnit(Enum):
|
||||
"""间隔单位"""
|
||||
MINUTES = "minutes"
|
||||
HOURS = "hours"
|
||||
DAYS = "days"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScheduleConfig:
|
||||
"""调度配置"""
|
||||
schedule_type: ScheduleType = ScheduleType.ONCE
|
||||
|
||||
# 间隔调度
|
||||
interval_value: int = 1
|
||||
interval_unit: IntervalUnit = IntervalUnit.HOURS
|
||||
|
||||
# 每日调度
|
||||
daily_time: str = "04:00" # HH:MM
|
||||
|
||||
# 每周调度
|
||||
weekly_days: List[int] = field(default_factory=lambda: [1]) # 1-7, 1=周一
|
||||
weekly_time: str = "04:00"
|
||||
|
||||
# Cron 表达式
|
||||
cron_expression: str = "0 4 * * *"
|
||||
|
||||
# 通用设置
|
||||
enabled: bool = True
|
||||
start_date: Optional[str] = None # YYYY-MM-DD
|
||||
end_date: Optional[str] = None # YYYY-MM-DD
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""转换为字典"""
|
||||
return {
|
||||
"schedule_type": self.schedule_type.value,
|
||||
"interval_value": self.interval_value,
|
||||
"interval_unit": self.interval_unit.value,
|
||||
"daily_time": self.daily_time,
|
||||
"weekly_days": self.weekly_days,
|
||||
"weekly_time": self.weekly_time,
|
||||
"cron_expression": self.cron_expression,
|
||||
"enabled": self.enabled,
|
||||
"start_date": self.start_date,
|
||||
"end_date": self.end_date,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "ScheduleConfig":
|
||||
"""从字典创建"""
|
||||
return cls(
|
||||
schedule_type=ScheduleType(data.get("schedule_type", "once")),
|
||||
interval_value=data.get("interval_value", 1),
|
||||
interval_unit=IntervalUnit(data.get("interval_unit", "hours")),
|
||||
daily_time=data.get("daily_time", "04:00"),
|
||||
weekly_days=data.get("weekly_days", [1]),
|
||||
weekly_time=data.get("weekly_time", "04:00"),
|
||||
cron_expression=data.get("cron_expression", "0 4 * * *"),
|
||||
enabled=data.get("enabled", True),
|
||||
start_date=data.get("start_date"),
|
||||
end_date=data.get("end_date"),
|
||||
)
|
||||
|
||||
def get_description(self) -> str:
|
||||
"""获取调度描述"""
|
||||
if self.schedule_type == ScheduleType.ONCE:
|
||||
return "一次性执行"
|
||||
elif self.schedule_type == ScheduleType.INTERVAL:
|
||||
unit_names = {"minutes": "分钟", "hours": "小时", "days": "天"}
|
||||
return f"每 {self.interval_value} {unit_names[self.interval_unit.value]}"
|
||||
elif self.schedule_type == ScheduleType.DAILY:
|
||||
return f"每天 {self.daily_time}"
|
||||
elif self.schedule_type == ScheduleType.WEEKLY:
|
||||
day_names = {1: "一", 2: "二", 3: "三", 4: "四", 5: "五", 6: "六", 7: "日"}
|
||||
days = "、".join(f"周{day_names[d]}" for d in sorted(self.weekly_days))
|
||||
return f"每周 {days} {self.weekly_time}"
|
||||
elif self.schedule_type == ScheduleType.CRON:
|
||||
return f"Cron: {self.cron_expression}"
|
||||
return "未知"
|
||||
|
||||
# 首次执行延迟秒数
|
||||
FIRST_RUN_DELAY_SECONDS = 60
|
||||
|
||||
def get_next_run_time(self, last_run: Optional[datetime] = None) -> Optional[datetime]:
|
||||
"""计算下次运行时间
|
||||
|
||||
注意:首次执行(last_run 为 None)时会延迟 60 秒,避免创建后立即执行
|
||||
"""
|
||||
now = datetime.now()
|
||||
|
||||
# 检查日期范围
|
||||
if self.start_date:
|
||||
start = datetime.strptime(self.start_date, "%Y-%m-%d")
|
||||
if now < start:
|
||||
now = start
|
||||
|
||||
if self.end_date:
|
||||
end = datetime.strptime(self.end_date, "%Y-%m-%d") + timedelta(days=1)
|
||||
if now >= end:
|
||||
return None
|
||||
|
||||
# 首次执行延迟 60 秒
|
||||
first_run_time = now + timedelta(seconds=self.FIRST_RUN_DELAY_SECONDS)
|
||||
|
||||
if self.schedule_type == ScheduleType.ONCE:
|
||||
return None if last_run else first_run_time
|
||||
|
||||
elif self.schedule_type == ScheduleType.INTERVAL:
|
||||
if not last_run:
|
||||
return first_run_time
|
||||
if self.interval_unit == IntervalUnit.MINUTES:
|
||||
delta = timedelta(minutes=self.interval_value)
|
||||
elif self.interval_unit == IntervalUnit.HOURS:
|
||||
delta = timedelta(hours=self.interval_value)
|
||||
else:
|
||||
delta = timedelta(days=self.interval_value)
|
||||
return last_run + delta
|
||||
|
||||
elif self.schedule_type == ScheduleType.DAILY:
|
||||
hour, minute = map(int, self.daily_time.split(":"))
|
||||
next_run = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||||
if next_run <= now:
|
||||
next_run += timedelta(days=1)
|
||||
return next_run
|
||||
|
||||
elif self.schedule_type == ScheduleType.WEEKLY:
|
||||
hour, minute = map(int, self.weekly_time.split(":"))
|
||||
# 找到下一个匹配的日期
|
||||
for i in range(8):
|
||||
check_date = now + timedelta(days=i)
|
||||
weekday = check_date.isoweekday() # 1-7
|
||||
if weekday in self.weekly_days:
|
||||
next_run = check_date.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||||
if next_run > now:
|
||||
return next_run
|
||||
return None
|
||||
|
||||
elif self.schedule_type == ScheduleType.CRON:
|
||||
# 简化版 Cron 解析(只支持基本格式)
|
||||
try:
|
||||
return self._parse_simple_cron(now)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def _parse_simple_cron(self, now: datetime) -> Optional[datetime]:
|
||||
"""简化版 Cron 解析"""
|
||||
parts = self.cron_expression.split()
|
||||
if len(parts) != 5:
|
||||
return None
|
||||
|
||||
minute, hour, day, month, weekday = parts
|
||||
|
||||
# 只处理简单情况
|
||||
if minute.isdigit() and hour.isdigit():
|
||||
next_run = now.replace(
|
||||
hour=int(hour),
|
||||
minute=int(minute),
|
||||
second=0,
|
||||
microsecond=0
|
||||
)
|
||||
if next_run <= now:
|
||||
next_run += timedelta(days=1)
|
||||
return next_run
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScheduleExecutionRecord:
|
||||
"""调度执行记录"""
|
||||
task_id: str # 关联的 QueuedTask ID
|
||||
executed_at: datetime # 执行时间
|
||||
status: str = "" # 状态:success, failed, pending
|
||||
exit_code: Optional[int] = None # 退出码
|
||||
duration_seconds: float = 0.0 # 耗时(秒)
|
||||
summary: str = "" # 执行摘要
|
||||
output: str = "" # 完整执行日志
|
||||
error: str = "" # 错误信息
|
||||
|
||||
# 日志最大长度限制(字符数)
|
||||
MAX_OUTPUT_LENGTH: int = 100000 # 100KB
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"task_id": self.task_id,
|
||||
"executed_at": self.executed_at.isoformat(),
|
||||
"status": self.status,
|
||||
"exit_code": self.exit_code,
|
||||
"duration_seconds": self.duration_seconds,
|
||||
"summary": self.summary,
|
||||
"output": self.output[:self.MAX_OUTPUT_LENGTH] if self.output else "",
|
||||
"error": self.error[:5000] if self.error else "",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "ScheduleExecutionRecord":
|
||||
return cls(
|
||||
task_id=data.get("task_id", ""),
|
||||
executed_at=datetime.fromisoformat(data["executed_at"]) if data.get("executed_at") else datetime.now(),
|
||||
status=data.get("status", ""),
|
||||
exit_code=data.get("exit_code"),
|
||||
duration_seconds=data.get("duration_seconds", 0.0),
|
||||
summary=data.get("summary", ""),
|
||||
output=data.get("output", ""),
|
||||
error=data.get("error", ""),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScheduledTask:
|
||||
"""调度任务"""
|
||||
id: str
|
||||
name: str
|
||||
task_codes: List[str]
|
||||
schedule: ScheduleConfig
|
||||
task_config: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
# 运行状态
|
||||
enabled: bool = True
|
||||
last_run: Optional[datetime] = None
|
||||
next_run: Optional[datetime] = None
|
||||
run_count: int = 0
|
||||
last_status: str = ""
|
||||
|
||||
# 执行历史(最近 N 次执行记录)
|
||||
execution_history: List[ScheduleExecutionRecord] = field(default_factory=list)
|
||||
MAX_HISTORY_SIZE: int = field(default=50, repr=False) # 保留最近50次执行记录
|
||||
|
||||
created_at: datetime = field(default_factory=datetime.now)
|
||||
updated_at: datetime = field(default_factory=datetime.now)
|
||||
|
||||
def add_execution_record(self, record: ScheduleExecutionRecord):
|
||||
"""添加执行记录"""
|
||||
self.execution_history.insert(0, record)
|
||||
# 限制历史记录数量
|
||||
if len(self.execution_history) > self.MAX_HISTORY_SIZE:
|
||||
self.execution_history = self.execution_history[:self.MAX_HISTORY_SIZE]
|
||||
|
||||
def update_execution_record(self, task_id: str, status: str, exit_code: int, duration: float,
|
||||
summary: str, output: str = "", error: str = ""):
|
||||
"""更新执行记录状态"""
|
||||
for record in self.execution_history:
|
||||
if record.task_id == task_id:
|
||||
record.status = status
|
||||
record.exit_code = exit_code
|
||||
record.duration_seconds = duration
|
||||
record.summary = summary
|
||||
record.output = output
|
||||
record.error = error
|
||||
break
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""转换为字典"""
|
||||
return {
|
||||
"id": self.id,
|
||||
"name": self.name,
|
||||
"task_codes": self.task_codes,
|
||||
"schedule": self.schedule.to_dict(),
|
||||
"task_config": self.task_config,
|
||||
"enabled": self.enabled,
|
||||
"last_run": self.last_run.isoformat() if self.last_run else None,
|
||||
"next_run": self.next_run.isoformat() if self.next_run else None,
|
||||
"run_count": self.run_count,
|
||||
"last_status": self.last_status,
|
||||
"execution_history": [r.to_dict() for r in self.execution_history],
|
||||
"created_at": self.created_at.isoformat(),
|
||||
"updated_at": self.updated_at.isoformat(),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "ScheduledTask":
|
||||
"""从字典创建"""
|
||||
history_data = data.get("execution_history", [])
|
||||
execution_history = [ScheduleExecutionRecord.from_dict(r) for r in history_data]
|
||||
|
||||
return cls(
|
||||
id=data["id"],
|
||||
name=data["name"],
|
||||
task_codes=data["task_codes"],
|
||||
schedule=ScheduleConfig.from_dict(data.get("schedule", {})),
|
||||
task_config=data.get("task_config", {}),
|
||||
enabled=data.get("enabled", True),
|
||||
last_run=datetime.fromisoformat(data["last_run"]) if data.get("last_run") else None,
|
||||
next_run=datetime.fromisoformat(data["next_run"]) if data.get("next_run") else None,
|
||||
run_count=data.get("run_count", 0),
|
||||
last_status=data.get("last_status", ""),
|
||||
execution_history=execution_history,
|
||||
created_at=datetime.fromisoformat(data["created_at"]) if data.get("created_at") else datetime.now(),
|
||||
updated_at=datetime.fromisoformat(data["updated_at"]) if data.get("updated_at") else datetime.now(),
|
||||
)
|
||||
|
||||
def update_next_run(self):
|
||||
"""更新下次运行时间"""
|
||||
if self.enabled and self.schedule.enabled:
|
||||
self.next_run = self.schedule.get_next_run_time(self.last_run)
|
||||
else:
|
||||
self.next_run = None
|
||||
self.updated_at = datetime.now()
|
||||
|
||||
|
||||
class ScheduleStore:
|
||||
"""调度任务存储"""
|
||||
|
||||
def __init__(self, storage_path: Optional[Path] = None):
|
||||
if storage_path is None:
|
||||
storage_path = Path(__file__).resolve().parents[2] / "scheduled_tasks.json"
|
||||
self.storage_path = storage_path
|
||||
self.tasks: Dict[str, ScheduledTask] = {}
|
||||
self.load()
|
||||
|
||||
def load(self):
|
||||
"""加载任务"""
|
||||
if self.storage_path.exists():
|
||||
try:
|
||||
data = json.loads(self.storage_path.read_text(encoding="utf-8"))
|
||||
self.tasks = {
|
||||
task_id: ScheduledTask.from_dict(task_data)
|
||||
for task_id, task_data in data.get("tasks", {}).items()
|
||||
}
|
||||
except Exception:
|
||||
self.tasks = {}
|
||||
|
||||
def save(self):
|
||||
"""保存任务"""
|
||||
data = {
|
||||
"tasks": {
|
||||
task_id: task.to_dict()
|
||||
for task_id, task in self.tasks.items()
|
||||
}
|
||||
}
|
||||
self.storage_path.write_text(
|
||||
json.dumps(data, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
def add_task(self, task: ScheduledTask):
|
||||
"""添加任务"""
|
||||
task.update_next_run()
|
||||
self.tasks[task.id] = task
|
||||
self.save()
|
||||
|
||||
def remove_task(self, task_id: str):
|
||||
"""移除任务"""
|
||||
if task_id in self.tasks:
|
||||
del self.tasks[task_id]
|
||||
self.save()
|
||||
|
||||
def update_task(self, task: ScheduledTask):
|
||||
"""更新任务"""
|
||||
task.update_next_run()
|
||||
task.updated_at = datetime.now()
|
||||
self.tasks[task.id] = task
|
||||
self.save()
|
||||
|
||||
def get_task(self, task_id: str) -> Optional[ScheduledTask]:
|
||||
"""获取任务"""
|
||||
return self.tasks.get(task_id)
|
||||
|
||||
def get_all_tasks(self) -> List[ScheduledTask]:
|
||||
"""获取所有任务"""
|
||||
return list(self.tasks.values())
|
||||
|
||||
def get_due_tasks(self) -> List[ScheduledTask]:
|
||||
"""获取到期需要执行的任务"""
|
||||
now = datetime.now()
|
||||
due_tasks = []
|
||||
for task in self.tasks.values():
|
||||
if task.enabled and task.next_run and task.next_run <= now:
|
||||
due_tasks.append(task)
|
||||
return due_tasks
|
||||
179
etl_billiards/gui/models/task_model.py
Normal file
179
etl_billiards/gui/models/task_model.py
Normal file
@@ -0,0 +1,179 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""任务数据模型"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
|
||||
class TaskStatus(Enum):
|
||||
"""任务状态枚举"""
|
||||
PENDING = "pending" # 待执行
|
||||
RUNNING = "running" # 执行中
|
||||
SUCCESS = "success" # 成功
|
||||
FAILED = "failed" # 失败
|
||||
CANCELLED = "cancelled" # 已取消
|
||||
|
||||
|
||||
class TaskCategory(Enum):
|
||||
"""任务分类"""
|
||||
ODS = "ODS" # ODS 数据抓取任务
|
||||
DWD = "DWD" # DWD 装载任务
|
||||
DWS = "DWS" # DWS 汇总任务
|
||||
SCHEMA = "Schema" # Schema 初始化任务
|
||||
QUALITY = "Quality" # 质量检查任务
|
||||
OTHER = "Other" # 其他任务
|
||||
|
||||
|
||||
# 任务分类映射
|
||||
TASK_CATEGORIES: Dict[str, TaskCategory] = {
|
||||
# ODS 任务
|
||||
"ODS_PAYMENT": TaskCategory.ODS,
|
||||
"ODS_MEMBER": TaskCategory.ODS,
|
||||
"ODS_MEMBER_CARD": TaskCategory.ODS,
|
||||
"ODS_MEMBER_BALANCE": TaskCategory.ODS,
|
||||
"ODS_SETTLEMENT_RECORDS": TaskCategory.ODS,
|
||||
"ODS_TABLE_USE": TaskCategory.ODS,
|
||||
"ODS_ASSISTANT_ACCOUNT": TaskCategory.ODS,
|
||||
"ODS_ASSISTANT_LEDGER": TaskCategory.ODS,
|
||||
"ODS_ASSISTANT_ABOLISH": TaskCategory.ODS,
|
||||
"ODS_REFUND": TaskCategory.ODS,
|
||||
"ODS_PLATFORM_COUPON": TaskCategory.ODS,
|
||||
"ODS_RECHARGE_SETTLE": TaskCategory.ODS,
|
||||
"ODS_GROUP_PACKAGE": TaskCategory.ODS,
|
||||
"ODS_GROUP_BUY_REDEMPTION": TaskCategory.ODS,
|
||||
"ODS_INVENTORY_STOCK": TaskCategory.ODS,
|
||||
"ODS_INVENTORY_CHANGE": TaskCategory.ODS,
|
||||
"ODS_TABLES": TaskCategory.ODS,
|
||||
"ODS_GOODS_CATEGORY": TaskCategory.ODS,
|
||||
"ODS_STORE_GOODS": TaskCategory.ODS,
|
||||
"ODS_STORE_GOODS_SALES": TaskCategory.ODS,
|
||||
"ODS_TABLE_FEE_DISCOUNT": TaskCategory.ODS,
|
||||
"ODS_TENANT_GOODS": TaskCategory.ODS,
|
||||
"ODS_SETTLEMENT_TICKET": TaskCategory.ODS,
|
||||
# DWD 任务
|
||||
"DWD_LOAD_FROM_ODS": TaskCategory.DWD,
|
||||
"DWD_QUALITY_CHECK": TaskCategory.QUALITY,
|
||||
"PAYMENTS_DWD": TaskCategory.DWD,
|
||||
"MEMBERS_DWD": TaskCategory.DWD,
|
||||
"TICKET_DWD": TaskCategory.DWD,
|
||||
# DWS 任务
|
||||
"INIT_DWS_SCHEMA": TaskCategory.SCHEMA,
|
||||
"DWS_BUILD_ORDER_SUMMARY": TaskCategory.DWS,
|
||||
# Schema 任务
|
||||
"INIT_ODS_SCHEMA": TaskCategory.SCHEMA,
|
||||
"INIT_DWD_SCHEMA": TaskCategory.SCHEMA,
|
||||
# 其他任务
|
||||
"MANUAL_INGEST": TaskCategory.OTHER,
|
||||
"CHECK_CUTOFF": TaskCategory.OTHER,
|
||||
"DATA_INTEGRITY_CHECK": TaskCategory.QUALITY,
|
||||
"ODS_JSON_ARCHIVE": TaskCategory.OTHER,
|
||||
# 旧版任务(兼容)
|
||||
"PRODUCTS": TaskCategory.ODS,
|
||||
"TABLES": TaskCategory.ODS,
|
||||
"MEMBERS": TaskCategory.ODS,
|
||||
"ASSISTANTS": TaskCategory.ODS,
|
||||
"PACKAGES_DEF": TaskCategory.ODS,
|
||||
"ORDERS": TaskCategory.ODS,
|
||||
"PAYMENTS": TaskCategory.ODS,
|
||||
"REFUNDS": TaskCategory.ODS,
|
||||
"COUPON_USAGE": TaskCategory.ODS,
|
||||
"INVENTORY_CHANGE": TaskCategory.ODS,
|
||||
"TOPUPS": TaskCategory.ODS,
|
||||
"TABLE_DISCOUNT": TaskCategory.ODS,
|
||||
"ASSISTANT_ABOLISH": TaskCategory.ODS,
|
||||
"LEDGER": TaskCategory.ODS,
|
||||
}
|
||||
|
||||
|
||||
def get_task_category(task_code: str) -> TaskCategory:
|
||||
"""获取任务分类"""
|
||||
return TASK_CATEGORIES.get(task_code.upper(), TaskCategory.OTHER)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskItem:
|
||||
"""任务项"""
|
||||
task_code: str
|
||||
name: str = ""
|
||||
description: str = ""
|
||||
category: TaskCategory = TaskCategory.OTHER
|
||||
enabled: bool = True
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.name:
|
||||
self.name = self.task_code
|
||||
if not self.category or self.category == TaskCategory.OTHER:
|
||||
self.category = get_task_category(self.task_code)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskConfig:
|
||||
"""任务执行配置"""
|
||||
tasks: List[str] = field(default_factory=list)
|
||||
pipeline_flow: str = "FULL" # FULL, FETCH_ONLY, INGEST_ONLY
|
||||
dry_run: bool = False
|
||||
window_start: Optional[str] = None
|
||||
window_end: Optional[str] = None
|
||||
window_split: Optional[str] = None # none, month
|
||||
window_compensation: int = 0 # 补偿小时数
|
||||
ingest_source: Optional[str] = None
|
||||
store_id: Optional[int] = None
|
||||
pg_dsn: Optional[str] = None
|
||||
api_token: Optional[str] = None
|
||||
extra_args: Dict[str, Any] = field(default_factory=dict)
|
||||
env_vars: Dict[str, str] = field(default_factory=dict) # 额外环境变量
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskHistory:
|
||||
"""任务执行历史"""
|
||||
id: str
|
||||
task_codes: List[str]
|
||||
status: TaskStatus
|
||||
start_time: datetime
|
||||
end_time: Optional[datetime] = None
|
||||
exit_code: Optional[int] = None
|
||||
command: str = ""
|
||||
output_log: str = ""
|
||||
error_message: str = ""
|
||||
summary: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def duration_seconds(self) -> Optional[float]:
|
||||
"""执行时长(秒)"""
|
||||
if self.end_time and self.start_time:
|
||||
return (self.end_time - self.start_time).total_seconds()
|
||||
return None
|
||||
|
||||
@property
|
||||
def duration_str(self) -> str:
|
||||
"""格式化的执行时长"""
|
||||
secs = self.duration_seconds
|
||||
if secs is None:
|
||||
return "-"
|
||||
if secs < 60:
|
||||
return f"{secs:.1f}秒"
|
||||
elif secs < 3600:
|
||||
mins = int(secs // 60)
|
||||
secs = secs % 60
|
||||
return f"{mins}分{secs:.0f}秒"
|
||||
else:
|
||||
hours = int(secs // 3600)
|
||||
mins = int((secs % 3600) // 60)
|
||||
return f"{hours}时{mins}分"
|
||||
|
||||
|
||||
@dataclass
|
||||
class QueuedTask:
|
||||
"""队列中的任务"""
|
||||
id: str
|
||||
config: TaskConfig
|
||||
status: TaskStatus = TaskStatus.PENDING
|
||||
created_at: datetime = field(default_factory=datetime.now)
|
||||
started_at: Optional[datetime] = None
|
||||
finished_at: Optional[datetime] = None
|
||||
output: str = ""
|
||||
error: str = ""
|
||||
exit_code: Optional[int] = None
|
||||
14
etl_billiards/gui/resources/__init__.py
Normal file
14
etl_billiards/gui/resources/__init__.py
Normal file
@@ -0,0 +1,14 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""GUI 资源模块"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
RESOURCES_DIR = Path(__file__).parent
|
||||
STYLES_PATH = RESOURCES_DIR / "styles.qss"
|
||||
|
||||
|
||||
def load_stylesheet() -> str:
|
||||
"""加载样式表"""
|
||||
if STYLES_PATH.exists():
|
||||
return STYLES_PATH.read_text(encoding="utf-8")
|
||||
return ""
|
||||
458
etl_billiards/gui/resources/styles.qss
Normal file
458
etl_billiards/gui/resources/styles.qss
Normal file
@@ -0,0 +1,458 @@
|
||||
/* ETL GUI 现代浅色主题样式表 */
|
||||
|
||||
/* ========== 全局样式 ========== */
|
||||
QWidget {
|
||||
font-family: "Microsoft YaHei", "Segoe UI", sans-serif;
|
||||
font-size: 13px;
|
||||
color: #333333;
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
|
||||
QMainWindow {
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
|
||||
/* ========== 菜单栏 ========== */
|
||||
QMenuBar {
|
||||
background-color: #ffffff;
|
||||
border-bottom: 1px solid #e0e0e0;
|
||||
padding: 4px;
|
||||
}
|
||||
|
||||
QMenuBar::item {
|
||||
padding: 6px 12px;
|
||||
background-color: transparent;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
QMenuBar::item:selected {
|
||||
background-color: #e8f0fe;
|
||||
}
|
||||
|
||||
QMenu {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #e0e0e0;
|
||||
border-radius: 8px;
|
||||
padding: 4px;
|
||||
}
|
||||
|
||||
QMenu::item {
|
||||
padding: 8px 24px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
QMenu::item:selected {
|
||||
background-color: #e8f0fe;
|
||||
}
|
||||
|
||||
/* ========== 工具栏 ========== */
|
||||
QToolBar {
|
||||
background-color: #ffffff;
|
||||
border-bottom: 1px solid #e0e0e0;
|
||||
padding: 4px;
|
||||
spacing: 4px;
|
||||
}
|
||||
|
||||
QToolButton {
|
||||
background-color: transparent;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
QToolButton:hover {
|
||||
background-color: #e8f0fe;
|
||||
}
|
||||
|
||||
QToolButton:pressed {
|
||||
background-color: #d2e3fc;
|
||||
}
|
||||
|
||||
/* ========== 按钮 ========== */
|
||||
QPushButton {
|
||||
background-color: #1a73e8;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
padding: 8px 16px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
QPushButton:hover {
|
||||
background-color: #1557b0;
|
||||
}
|
||||
|
||||
QPushButton:pressed {
|
||||
background-color: #104080;
|
||||
}
|
||||
|
||||
QPushButton:disabled {
|
||||
background-color: #dadce0;
|
||||
color: #9aa0a6;
|
||||
}
|
||||
|
||||
QPushButton[secondary="true"] {
|
||||
background-color: #ffffff;
|
||||
color: #1a73e8;
|
||||
border: 1px solid #dadce0;
|
||||
}
|
||||
|
||||
QPushButton[secondary="true"]:hover {
|
||||
background-color: #f8f9fa;
|
||||
border-color: #1a73e8;
|
||||
}
|
||||
|
||||
QPushButton[danger="true"] {
|
||||
background-color: #ea4335;
|
||||
}
|
||||
|
||||
QPushButton[danger="true"]:hover {
|
||||
background-color: #c5221f;
|
||||
}
|
||||
|
||||
/* ========== 输入框 ========== */
|
||||
QLineEdit, QTextEdit, QPlainTextEdit {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 6px;
|
||||
padding: 8px 12px;
|
||||
selection-background-color: #d2e3fc;
|
||||
}
|
||||
|
||||
QLineEdit:focus, QTextEdit:focus, QPlainTextEdit:focus {
|
||||
border-color: #1a73e8;
|
||||
border-width: 2px;
|
||||
padding: 7px 11px;
|
||||
}
|
||||
|
||||
QLineEdit:disabled, QTextEdit:disabled, QPlainTextEdit:disabled {
|
||||
background-color: #f1f3f4;
|
||||
color: #9aa0a6;
|
||||
}
|
||||
|
||||
/* ========== 下拉框 ========== */
|
||||
QComboBox {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 6px;
|
||||
padding: 8px 12px;
|
||||
padding-right: 30px;
|
||||
}
|
||||
|
||||
QComboBox:hover {
|
||||
border-color: #1a73e8;
|
||||
}
|
||||
|
||||
QComboBox:focus {
|
||||
border-color: #1a73e8;
|
||||
border-width: 2px;
|
||||
}
|
||||
|
||||
QComboBox::drop-down {
|
||||
border: none;
|
||||
width: 24px;
|
||||
}
|
||||
|
||||
QComboBox::down-arrow {
|
||||
image: none;
|
||||
border-left: 5px solid transparent;
|
||||
border-right: 5px solid transparent;
|
||||
border-top: 6px solid #5f6368;
|
||||
margin-right: 8px;
|
||||
}
|
||||
|
||||
QComboBox QAbstractItemView {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 8px;
|
||||
selection-background-color: #e8f0fe;
|
||||
}
|
||||
|
||||
/* ========== 复选框 ========== */
|
||||
QCheckBox {
|
||||
spacing: 8px;
|
||||
}
|
||||
|
||||
QCheckBox::indicator {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
border-radius: 4px;
|
||||
border: 2px solid #5f6368;
|
||||
}
|
||||
|
||||
QCheckBox::indicator:checked {
|
||||
background-color: #1a73e8;
|
||||
border-color: #1a73e8;
|
||||
}
|
||||
|
||||
QCheckBox::indicator:hover {
|
||||
border-color: #1a73e8;
|
||||
}
|
||||
|
||||
/* ========== 列表和树 ========== */
|
||||
QListWidget, QTreeWidget, QTableWidget {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 8px;
|
||||
outline: none;
|
||||
}
|
||||
|
||||
QListWidget::item, QTreeWidget::item {
|
||||
padding: 8px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
QListWidget::item:selected, QTreeWidget::item:selected {
|
||||
background-color: #e8f0fe;
|
||||
color: #1a73e8;
|
||||
}
|
||||
|
||||
QListWidget::item:hover, QTreeWidget::item:hover {
|
||||
background-color: #f8f9fa;
|
||||
}
|
||||
|
||||
QHeaderView::section {
|
||||
background-color: #f8f9fa;
|
||||
border: none;
|
||||
border-bottom: 1px solid #dadce0;
|
||||
padding: 10px 16px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
QTableWidget {
|
||||
gridline-color: #e8eaed;
|
||||
}
|
||||
|
||||
QTableWidget::item {
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
QTableWidget::item:selected {
|
||||
background-color: #e8f0fe;
|
||||
color: #1a73e8;
|
||||
}
|
||||
|
||||
/* ========== 滚动条 ========== */
|
||||
QScrollBar:vertical {
|
||||
background-color: transparent;
|
||||
width: 12px;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
QScrollBar::handle:vertical {
|
||||
background-color: #dadce0;
|
||||
border-radius: 6px;
|
||||
min-height: 30px;
|
||||
margin: 2px;
|
||||
}
|
||||
|
||||
QScrollBar::handle:vertical:hover {
|
||||
background-color: #bdc1c6;
|
||||
}
|
||||
|
||||
QScrollBar::add-line:vertical, QScrollBar::sub-line:vertical {
|
||||
height: 0;
|
||||
}
|
||||
|
||||
QScrollBar:horizontal {
|
||||
background-color: transparent;
|
||||
height: 12px;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
QScrollBar::handle:horizontal {
|
||||
background-color: #dadce0;
|
||||
border-radius: 6px;
|
||||
min-width: 30px;
|
||||
margin: 2px;
|
||||
}
|
||||
|
||||
QScrollBar::handle:horizontal:hover {
|
||||
background-color: #bdc1c6;
|
||||
}
|
||||
|
||||
QScrollBar::add-line:horizontal, QScrollBar::sub-line:horizontal {
|
||||
width: 0;
|
||||
}
|
||||
|
||||
/* ========== 选项卡 ========== */
|
||||
QTabWidget::pane {
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 8px;
|
||||
background-color: #ffffff;
|
||||
margin-top: -1px;
|
||||
}
|
||||
|
||||
QTabBar::tab {
|
||||
background-color: transparent;
|
||||
border: none;
|
||||
padding: 10px 20px;
|
||||
margin-right: 4px;
|
||||
color: #5f6368;
|
||||
}
|
||||
|
||||
QTabBar::tab:selected {
|
||||
color: #1a73e8;
|
||||
border-bottom: 2px solid #1a73e8;
|
||||
}
|
||||
|
||||
QTabBar::tab:hover:!selected {
|
||||
background-color: #f8f9fa;
|
||||
border-radius: 6px 6px 0 0;
|
||||
}
|
||||
|
||||
/* ========== 分组框 ========== */
|
||||
QGroupBox {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 8px;
|
||||
margin-top: 16px;
|
||||
padding: 16px;
|
||||
padding-top: 24px;
|
||||
}
|
||||
|
||||
QGroupBox::title {
|
||||
subcontrol-origin: margin;
|
||||
subcontrol-position: top left;
|
||||
left: 16px;
|
||||
padding: 0 8px;
|
||||
background-color: #ffffff;
|
||||
color: #5f6368;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* ========== 进度条 ========== */
|
||||
QProgressBar {
|
||||
background-color: #e8eaed;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
height: 8px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
QProgressBar::chunk {
|
||||
background-color: #1a73e8;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
/* ========== 分割器 ========== */
|
||||
QSplitter::handle {
|
||||
background-color: #e0e0e0;
|
||||
}
|
||||
|
||||
QSplitter::handle:horizontal {
|
||||
width: 2px;
|
||||
}
|
||||
|
||||
QSplitter::handle:vertical {
|
||||
height: 2px;
|
||||
}
|
||||
|
||||
QSplitter::handle:hover {
|
||||
background-color: #1a73e8;
|
||||
}
|
||||
|
||||
/* ========== 状态栏 ========== */
|
||||
QStatusBar {
|
||||
background-color: #ffffff;
|
||||
border-top: 1px solid #e0e0e0;
|
||||
padding: 4px;
|
||||
}
|
||||
|
||||
QStatusBar::item {
|
||||
border: none;
|
||||
}
|
||||
|
||||
/* ========== 提示框 ========== */
|
||||
QToolTip {
|
||||
background-color: #3c4043;
|
||||
color: #ffffff;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
padding: 8px 12px;
|
||||
}
|
||||
|
||||
/* ========== 消息框 ========== */
|
||||
QMessageBox {
|
||||
background-color: #ffffff;
|
||||
}
|
||||
|
||||
/* ========== 导航侧边栏 ========== */
|
||||
QListWidget#navList {
|
||||
background-color: #ffffff;
|
||||
border: none;
|
||||
border-right: 1px solid #e0e0e0;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
QListWidget#navList::item {
|
||||
padding: 12px 16px;
|
||||
border-radius: 8px;
|
||||
margin: 2px 0;
|
||||
}
|
||||
|
||||
QListWidget#navList::item:selected {
|
||||
background-color: #e8f0fe;
|
||||
color: #1a73e8;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* ========== 日志查看器 ========== */
|
||||
QPlainTextEdit#logViewer {
|
||||
font-family: "Consolas", "Courier New", monospace;
|
||||
font-size: 12px;
|
||||
background-color: #fafafa;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
/* ========== SQL 编辑器 ========== */
|
||||
QPlainTextEdit#sqlEditor {
|
||||
font-family: "Consolas", "Courier New", monospace;
|
||||
font-size: 13px;
|
||||
background-color: #ffffff;
|
||||
}
|
||||
|
||||
/* ========== 卡片样式 ========== */
|
||||
QFrame[card="true"] {
|
||||
background-color: #ffffff;
|
||||
border: 1px solid #dadce0;
|
||||
border-radius: 12px;
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
QFrame[card="true"]:hover {
|
||||
border-color: #1a73e8;
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
/* ========== 标签 ========== */
|
||||
QLabel[heading="true"] {
|
||||
font-size: 18px;
|
||||
font-weight: 600;
|
||||
color: #202124;
|
||||
}
|
||||
|
||||
QLabel[subheading="true"] {
|
||||
font-size: 14px;
|
||||
color: #5f6368;
|
||||
}
|
||||
|
||||
QLabel[status="success"] {
|
||||
color: #1e8e3e;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
QLabel[status="error"] {
|
||||
color: #d93025;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
QLabel[status="warning"] {
|
||||
color: #f9ab00;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
QLabel[status="info"] {
|
||||
color: #1a73e8;
|
||||
font-weight: 500;
|
||||
}
|
||||
8
etl_billiards/gui/utils/__init__.py
Normal file
8
etl_billiards/gui/utils/__init__.py
Normal file
@@ -0,0 +1,8 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""工具模块"""
|
||||
|
||||
from .cli_builder import CLIBuilder
|
||||
from .config_helper import ConfigHelper
|
||||
from .app_settings import app_settings, AppSettings
|
||||
|
||||
__all__ = ["CLIBuilder", "ConfigHelper", "app_settings", "AppSettings"]
|
||||
434
etl_billiards/gui/utils/app_settings.py
Normal file
434
etl_billiards/gui/utils/app_settings.py
Normal file
@@ -0,0 +1,434 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""应用程序设置管理"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
class AppSettings:
|
||||
"""应用程序设置单例"""
|
||||
|
||||
_instance: Optional["AppSettings"] = None
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
cls._instance._initialized = False
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
if self._initialized:
|
||||
return
|
||||
self._initialized = True
|
||||
|
||||
# 配置文件路径
|
||||
self._settings_file = self._get_settings_path()
|
||||
|
||||
# 默认设置
|
||||
self._settings = {
|
||||
"etl_project_path": "", # ETL 项目路径
|
||||
"env_file_path": "", # .env 文件路径
|
||||
# 自动更新配置
|
||||
"auto_update": {
|
||||
"hours": 24,
|
||||
"overlap_seconds": 3600,
|
||||
"include_dwd": True,
|
||||
"auto_verify": False,
|
||||
"selected_tasks": [],
|
||||
},
|
||||
# 数据校验配置
|
||||
"integrity_check": {
|
||||
"mode": "history",
|
||||
"history_start": "",
|
||||
"history_end": "",
|
||||
"lookback_hours": 24,
|
||||
"include_dimensions": False,
|
||||
"auto_backfill": False,
|
||||
"ods_tasks": "",
|
||||
},
|
||||
# 高级配置
|
||||
"advanced": {
|
||||
"pipeline_flow": "FULL",
|
||||
"dry_run": False,
|
||||
"window_start": "",
|
||||
"window_end": "",
|
||||
"window_split": "none",
|
||||
"window_compensation": 0,
|
||||
"ingest_source": "",
|
||||
"store_id": "",
|
||||
"pg_dsn": "",
|
||||
"api_token": "",
|
||||
},
|
||||
}
|
||||
|
||||
# 加载设置
|
||||
self._load()
|
||||
|
||||
# 如果没有配置,尝试自动检测
|
||||
if not self._settings["etl_project_path"]:
|
||||
self._auto_detect_paths()
|
||||
|
||||
def _get_settings_path(self) -> Path:
|
||||
"""获取设置文件路径"""
|
||||
# 优先使用用户目录
|
||||
if sys.platform == "win32":
|
||||
app_data = os.environ.get("APPDATA", "")
|
||||
if app_data:
|
||||
settings_dir = Path(app_data) / "ETL管理系统"
|
||||
else:
|
||||
settings_dir = Path.home() / ".etl_gui"
|
||||
else:
|
||||
settings_dir = Path.home() / ".etl_gui"
|
||||
|
||||
settings_dir.mkdir(parents=True, exist_ok=True)
|
||||
return settings_dir / "settings.json"
|
||||
|
||||
def _auto_detect_paths(self):
|
||||
"""自动检测 ETL 项目路径"""
|
||||
# 方法1: 检查是否从源码目录运行
|
||||
try:
|
||||
source_dir = Path(__file__).resolve().parents[2]
|
||||
cli_main = source_dir / "cli" / "main.py"
|
||||
if cli_main.exists():
|
||||
rel_source = Path(os.path.relpath(source_dir, Path.cwd()))
|
||||
self._settings["etl_project_path"] = str(rel_source)
|
||||
env_file = rel_source / ".env"
|
||||
if env_file.exists():
|
||||
self._settings["env_file_path"] = str(env_file)
|
||||
self._save()
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 方法2: 检查常见位置
|
||||
common_paths = [
|
||||
Path("etl_billiards"),
|
||||
Path("."),
|
||||
]
|
||||
|
||||
for path in common_paths:
|
||||
if path.exists() and (path / "cli" / "main.py").exists():
|
||||
self._settings["etl_project_path"] = str(path)
|
||||
env_file = path / ".env"
|
||||
if env_file.exists():
|
||||
self._settings["env_file_path"] = str(env_file)
|
||||
self._save()
|
||||
return
|
||||
|
||||
def _load(self):
|
||||
"""加载设置"""
|
||||
if self._settings_file.exists():
|
||||
try:
|
||||
data = json.loads(self._settings_file.read_text(encoding="utf-8"))
|
||||
self._settings.update(data)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _save(self):
|
||||
"""保存设置"""
|
||||
try:
|
||||
self._settings_file.write_text(
|
||||
json.dumps(self._settings, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@property
|
||||
def etl_project_path(self) -> str:
|
||||
"""获取 ETL 项目路径"""
|
||||
return self._settings.get("etl_project_path", "")
|
||||
|
||||
@etl_project_path.setter
|
||||
def etl_project_path(self, value: str):
|
||||
"""设置 ETL 项目路径"""
|
||||
self._settings["etl_project_path"] = value
|
||||
# 同时更新 .env 路径
|
||||
if value:
|
||||
env_path = Path(value) / ".env"
|
||||
if env_path.exists():
|
||||
self._settings["env_file_path"] = str(env_path)
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def env_file_path(self) -> str:
|
||||
"""获取 .env 文件路径"""
|
||||
path = self._settings.get("env_file_path", "")
|
||||
if not path and self.etl_project_path:
|
||||
path = str(Path(self.etl_project_path) / ".env")
|
||||
return path
|
||||
|
||||
@env_file_path.setter
|
||||
def env_file_path(self, value: str):
|
||||
"""设置 .env 文件路径"""
|
||||
self._settings["env_file_path"] = value
|
||||
self._save()
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
"""检查是否已配置"""
|
||||
path = self.etl_project_path
|
||||
if not path:
|
||||
return False
|
||||
return Path(path).exists() and (Path(path) / "cli" / "main.py").exists()
|
||||
|
||||
def validate(self) -> tuple[bool, str]:
|
||||
"""验证配置"""
|
||||
path = self.etl_project_path
|
||||
if not path:
|
||||
return False, "未配置 ETL 项目路径"
|
||||
|
||||
project_path = Path(path)
|
||||
if not project_path.exists():
|
||||
return False, f"ETL 项目路径不存在: {path}"
|
||||
|
||||
cli_main = project_path / "cli" / "main.py"
|
||||
if not cli_main.exists():
|
||||
return False, f"找不到 CLI 入口: {cli_main}"
|
||||
|
||||
return True, "配置有效"
|
||||
|
||||
# ==================== 自动更新配置 ====================
|
||||
|
||||
@property
|
||||
def auto_update_hours(self) -> int:
|
||||
return self._settings.get("auto_update", {}).get("hours", 24)
|
||||
|
||||
@auto_update_hours.setter
|
||||
def auto_update_hours(self, value: int):
|
||||
self._settings.setdefault("auto_update", {})["hours"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def auto_update_overlap_seconds(self) -> int:
|
||||
return self._settings.get("auto_update", {}).get("overlap_seconds", 3600)
|
||||
|
||||
@auto_update_overlap_seconds.setter
|
||||
def auto_update_overlap_seconds(self, value: int):
|
||||
self._settings.setdefault("auto_update", {})["overlap_seconds"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def auto_update_include_dwd(self) -> bool:
|
||||
return self._settings.get("auto_update", {}).get("include_dwd", True)
|
||||
|
||||
@auto_update_include_dwd.setter
|
||||
def auto_update_include_dwd(self, value: bool):
|
||||
self._settings.setdefault("auto_update", {})["include_dwd"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def auto_update_auto_verify(self) -> bool:
|
||||
return self._settings.get("auto_update", {}).get("auto_verify", False)
|
||||
|
||||
@auto_update_auto_verify.setter
|
||||
def auto_update_auto_verify(self, value: bool):
|
||||
self._settings.setdefault("auto_update", {})["auto_verify"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def auto_update_selected_tasks(self) -> list:
|
||||
return self._settings.get("auto_update", {}).get("selected_tasks", [])
|
||||
|
||||
@auto_update_selected_tasks.setter
|
||||
def auto_update_selected_tasks(self, value: list):
|
||||
self._settings.setdefault("auto_update", {})["selected_tasks"] = value
|
||||
self._save()
|
||||
|
||||
# ==================== 数据校验配置 ====================
|
||||
|
||||
@property
|
||||
def integrity_mode(self) -> str:
|
||||
return self._settings.get("integrity_check", {}).get("mode", "history")
|
||||
|
||||
@integrity_mode.setter
|
||||
def integrity_mode(self, value: str):
|
||||
self._settings.setdefault("integrity_check", {})["mode"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_history_start(self) -> str:
|
||||
return self._settings.get("integrity_check", {}).get("history_start", "")
|
||||
|
||||
@integrity_history_start.setter
|
||||
def integrity_history_start(self, value: str):
|
||||
self._settings.setdefault("integrity_check", {})["history_start"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_history_end(self) -> str:
|
||||
return self._settings.get("integrity_check", {}).get("history_end", "")
|
||||
|
||||
@integrity_history_end.setter
|
||||
def integrity_history_end(self, value: str):
|
||||
self._settings.setdefault("integrity_check", {})["history_end"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_lookback_hours(self) -> int:
|
||||
return self._settings.get("integrity_check", {}).get("lookback_hours", 24)
|
||||
|
||||
@integrity_lookback_hours.setter
|
||||
def integrity_lookback_hours(self, value: int):
|
||||
self._settings.setdefault("integrity_check", {})["lookback_hours"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_include_dimensions(self) -> bool:
|
||||
return self._settings.get("integrity_check", {}).get("include_dimensions", False)
|
||||
|
||||
@integrity_include_dimensions.setter
|
||||
def integrity_include_dimensions(self, value: bool):
|
||||
self._settings.setdefault("integrity_check", {})["include_dimensions"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_auto_backfill(self) -> bool:
|
||||
return self._settings.get("integrity_check", {}).get("auto_backfill", False)
|
||||
|
||||
@integrity_auto_backfill.setter
|
||||
def integrity_auto_backfill(self, value: bool):
|
||||
self._settings.setdefault("integrity_check", {})["auto_backfill"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def integrity_ods_tasks(self) -> str:
|
||||
return self._settings.get("integrity_check", {}).get("ods_tasks", "")
|
||||
|
||||
@integrity_ods_tasks.setter
|
||||
def integrity_ods_tasks(self, value: str):
|
||||
self._settings.setdefault("integrity_check", {})["ods_tasks"] = value
|
||||
self._save()
|
||||
|
||||
# ==================== 高级配置 ====================
|
||||
|
||||
@property
|
||||
def advanced_pipeline_flow(self) -> str:
|
||||
return self._settings.get("advanced", {}).get("pipeline_flow", "FULL")
|
||||
|
||||
@advanced_pipeline_flow.setter
|
||||
def advanced_pipeline_flow(self, value: str):
|
||||
self._settings.setdefault("advanced", {})["pipeline_flow"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_dry_run(self) -> bool:
|
||||
return self._settings.get("advanced", {}).get("dry_run", False)
|
||||
|
||||
@advanced_dry_run.setter
|
||||
def advanced_dry_run(self, value: bool):
|
||||
self._settings.setdefault("advanced", {})["dry_run"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_window_start(self) -> str:
|
||||
return self._settings.get("advanced", {}).get("window_start", "")
|
||||
|
||||
@advanced_window_start.setter
|
||||
def advanced_window_start(self, value: str):
|
||||
self._settings.setdefault("advanced", {})["window_start"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_window_end(self) -> str:
|
||||
return self._settings.get("advanced", {}).get("window_end", "")
|
||||
|
||||
@advanced_window_end.setter
|
||||
def advanced_window_end(self, value: str):
|
||||
self._settings.setdefault("advanced", {})["window_end"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_ingest_source(self) -> str:
|
||||
return self._settings.get("advanced", {}).get("ingest_source", "")
|
||||
|
||||
@advanced_ingest_source.setter
|
||||
def advanced_ingest_source(self, value: str):
|
||||
self._settings.setdefault("advanced", {})["ingest_source"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_window_split(self) -> str:
|
||||
return self._settings.get("advanced", {}).get("window_split", "none")
|
||||
|
||||
@advanced_window_split.setter
|
||||
def advanced_window_split(self, value: str):
|
||||
self._settings.setdefault("advanced", {})["window_split"] = value
|
||||
self._save()
|
||||
|
||||
@property
|
||||
def advanced_window_compensation(self) -> int:
|
||||
return self._settings.get("advanced", {}).get("window_compensation", 0)
|
||||
|
||||
@advanced_window_compensation.setter
|
||||
def advanced_window_compensation(self, value: int):
|
||||
self._settings.setdefault("advanced", {})["window_compensation"] = value
|
||||
self._save()
|
||||
|
||||
def get_all_settings(self) -> Dict[str, Any]:
|
||||
"""获取所有设置(用于调试)"""
|
||||
return self._settings.copy()
|
||||
|
||||
def save_all(self):
|
||||
"""强制保存所有设置"""
|
||||
self._save()
|
||||
|
||||
# ==================== 任务历史存储 ====================
|
||||
|
||||
def _get_history_path(self) -> Path:
|
||||
"""获取任务历史文件路径"""
|
||||
return self._settings_file.parent / "task_history.json"
|
||||
|
||||
def save_task_history(self, history_list: list):
|
||||
"""保存任务历史到文件"""
|
||||
try:
|
||||
history_path = self._get_history_path()
|
||||
|
||||
# 序列化任务历史
|
||||
serialized = []
|
||||
for task in history_list[:100]: # 最多保存100条
|
||||
try:
|
||||
task_data = {
|
||||
"id": task.id,
|
||||
"tasks": task.config.tasks if hasattr(task, 'config') else [],
|
||||
"status": task.status.value if hasattr(task.status, 'value') else str(task.status),
|
||||
"created_at": task.created_at.isoformat() if task.created_at else None,
|
||||
"started_at": task.started_at.isoformat() if task.started_at else None,
|
||||
"finished_at": task.finished_at.isoformat() if task.finished_at else None,
|
||||
"exit_code": task.exit_code,
|
||||
"error": task.error[:500] if task.error else "", # 限制长度
|
||||
"output_preview": task.output[:1000] if task.output else "", # 输出预览
|
||||
# 保存配置信息
|
||||
"pipeline_flow": task.config.pipeline_flow if hasattr(task, 'config') else "FULL",
|
||||
"window_start": task.config.window_start if hasattr(task, 'config') else None,
|
||||
"window_end": task.config.window_end if hasattr(task, 'config') else None,
|
||||
}
|
||||
serialized.append(task_data)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
history_path.write_text(
|
||||
json.dumps(serialized, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8"
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"保存任务历史失败: {e}")
|
||||
|
||||
def load_task_history(self) -> list:
|
||||
"""从文件加载任务历史"""
|
||||
try:
|
||||
history_path = self._get_history_path()
|
||||
if not history_path.exists():
|
||||
return []
|
||||
|
||||
data = json.loads(history_path.read_text(encoding="utf-8"))
|
||||
return data
|
||||
except Exception as e:
|
||||
print(f"加载任务历史失败: {e}")
|
||||
return []
|
||||
|
||||
|
||||
# 全局单例
|
||||
app_settings = AppSettings()
|
||||
131
etl_billiards/gui/utils/cli_builder.py
Normal file
131
etl_billiards/gui/utils/cli_builder.py
Normal file
@@ -0,0 +1,131 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""CLI 命令构建器"""
|
||||
|
||||
from typing import List, Dict, Any, Optional
|
||||
from ..models.task_model import TaskConfig
|
||||
|
||||
|
||||
# CLI 支持的命令行参数(来自 cli/main.py)
|
||||
CLI_SUPPORTED_ARGS = {
|
||||
# 值类型参数
|
||||
"store_id", "tasks", "pg_dsn", "pg_host", "pg_port", "pg_name",
|
||||
"pg_user", "pg_password", "api_base", "api_token", "api_timeout",
|
||||
"api_page_size", "api_retry_max", "window_start", "window_end",
|
||||
"export_root", "log_root", "pipeline_flow", "fetch_root",
|
||||
"ingest_source", "idle_start", "idle_end",
|
||||
# 布尔类型参数
|
||||
"dry_run", "force_window_override", "write_pretty_json", "allow_empty_advance",
|
||||
}
|
||||
|
||||
|
||||
class CLIBuilder:
|
||||
"""构建 CLI 命令行参数"""
|
||||
|
||||
def __init__(self, python_executable: str = "python"):
|
||||
self.python_executable = python_executable
|
||||
|
||||
def build_command(self, config: TaskConfig) -> List[str]:
|
||||
"""
|
||||
根据任务配置构建命令行参数列表
|
||||
|
||||
Args:
|
||||
config: 任务配置对象
|
||||
|
||||
Returns:
|
||||
命令行参数列表
|
||||
"""
|
||||
cmd = [self.python_executable, "-m", "cli.main"]
|
||||
|
||||
# 任务列表
|
||||
if config.tasks:
|
||||
cmd.extend(["--tasks", ",".join(config.tasks)])
|
||||
|
||||
# Pipeline 流程
|
||||
if config.pipeline_flow:
|
||||
cmd.extend(["--pipeline-flow", config.pipeline_flow])
|
||||
|
||||
# Dry-run 模式
|
||||
if config.dry_run:
|
||||
cmd.append("--dry-run")
|
||||
|
||||
# 时间窗口
|
||||
if config.window_start:
|
||||
cmd.extend(["--window-start", config.window_start])
|
||||
if config.window_end:
|
||||
cmd.extend(["--window-end", config.window_end])
|
||||
|
||||
# 数据源目录
|
||||
if config.ingest_source:
|
||||
cmd.extend(["--ingest-source", config.ingest_source])
|
||||
|
||||
# 门店 ID
|
||||
if config.store_id is not None:
|
||||
cmd.extend(["--store-id", str(config.store_id)])
|
||||
|
||||
# 数据库 DSN
|
||||
if config.pg_dsn:
|
||||
cmd.extend(["--pg-dsn", config.pg_dsn])
|
||||
|
||||
# API Token
|
||||
if config.api_token:
|
||||
cmd.extend(["--api-token", config.api_token])
|
||||
|
||||
# 额外参数(只传递 CLI 支持的参数)
|
||||
for key, value in config.extra_args.items():
|
||||
if value is not None and key in CLI_SUPPORTED_ARGS:
|
||||
arg_name = f"--{key.replace('_', '-')}"
|
||||
if isinstance(value, bool):
|
||||
if value:
|
||||
cmd.append(arg_name)
|
||||
else:
|
||||
cmd.extend([arg_name, str(value)])
|
||||
|
||||
return cmd
|
||||
|
||||
def build_command_string(self, config: TaskConfig) -> str:
|
||||
"""
|
||||
构建命令行字符串(用于显示)
|
||||
|
||||
Args:
|
||||
config: 任务配置对象
|
||||
|
||||
Returns:
|
||||
命令行字符串
|
||||
"""
|
||||
cmd = self.build_command(config)
|
||||
# 对包含空格的参数添加引号
|
||||
quoted_cmd = []
|
||||
for arg in cmd:
|
||||
if ' ' in arg or '"' in arg:
|
||||
quoted_cmd.append(f'"{arg}"')
|
||||
else:
|
||||
quoted_cmd.append(arg)
|
||||
return " ".join(quoted_cmd)
|
||||
|
||||
def build_from_dict(self, params: Dict[str, Any]) -> List[str]:
|
||||
"""
|
||||
从字典构建命令行参数
|
||||
|
||||
Args:
|
||||
params: 参数字典
|
||||
|
||||
Returns:
|
||||
命令行参数列表
|
||||
"""
|
||||
config = TaskConfig(
|
||||
tasks=params.get("tasks", []),
|
||||
pipeline_flow=params.get("pipeline_flow", "FULL"),
|
||||
dry_run=params.get("dry_run", False),
|
||||
window_start=params.get("window_start"),
|
||||
window_end=params.get("window_end"),
|
||||
ingest_source=params.get("ingest_source"),
|
||||
store_id=params.get("store_id"),
|
||||
pg_dsn=params.get("pg_dsn"),
|
||||
api_token=params.get("api_token"),
|
||||
extra_args=params.get("extra_args", {}),
|
||||
)
|
||||
return self.build_command(config)
|
||||
|
||||
|
||||
# 全局实例
|
||||
cli_builder = CLIBuilder()
|
||||
309
etl_billiards/gui/utils/config_helper.py
Normal file
309
etl_billiards/gui/utils/config_helper.py
Normal file
@@ -0,0 +1,309 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""配置辅助工具"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple, Optional, Any
|
||||
|
||||
|
||||
# 环境变量分组
|
||||
ENV_GROUPS = {
|
||||
"database": {
|
||||
"title": "数据库配置",
|
||||
"keys": ["PG_DSN", "PG_HOST", "PG_PORT", "PG_NAME", "PG_USER", "PG_PASSWORD", "PG_CONNECT_TIMEOUT"],
|
||||
"sensitive": ["PG_PASSWORD"],
|
||||
},
|
||||
"api": {
|
||||
"title": "API 配置",
|
||||
"keys": ["API_BASE", "API_TOKEN", "FICOO_TOKEN", "API_TIMEOUT", "API_PAGE_SIZE", "API_RETRY_MAX"],
|
||||
"sensitive": ["API_TOKEN", "FICOO_TOKEN"],
|
||||
},
|
||||
"store": {
|
||||
"title": "门店配置",
|
||||
"keys": ["STORE_ID", "TIMEZONE", "SCHEMA_OLTP", "SCHEMA_ETL"],
|
||||
"sensitive": [],
|
||||
},
|
||||
"paths": {
|
||||
"title": "路径配置",
|
||||
"keys": ["EXPORT_ROOT", "LOG_ROOT", "FETCH_ROOT", "INGEST_SOURCE_DIR", "JSON_FETCH_ROOT", "JSON_SOURCE_DIR"],
|
||||
"sensitive": [],
|
||||
},
|
||||
"pipeline": {
|
||||
"title": "流水线配置",
|
||||
"keys": ["PIPELINE_FLOW", "RUN_TASKS", "OVERLAP_SECONDS"],
|
||||
"sensitive": [],
|
||||
},
|
||||
"window": {
|
||||
"title": "时间窗口配置",
|
||||
"keys": ["WINDOW_START", "WINDOW_END", "WINDOW_BUSY_MIN", "WINDOW_IDLE_MIN", "IDLE_START", "IDLE_END"],
|
||||
"sensitive": [],
|
||||
},
|
||||
"integrity": {
|
||||
"title": "数据完整性配置",
|
||||
"keys": ["INTEGRITY_MODE", "INTEGRITY_HISTORY_START", "INTEGRITY_HISTORY_END",
|
||||
"INTEGRITY_INCLUDE_DIMENSIONS", "INTEGRITY_AUTO_CHECK", "INTEGRITY_ODS_TASK_CODES"],
|
||||
"sensitive": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ConfigHelper:
|
||||
"""配置文件辅助类"""
|
||||
|
||||
def __init__(self, env_path: Optional[Path] = None):
|
||||
"""
|
||||
初始化配置辅助器
|
||||
|
||||
Args:
|
||||
env_path: .env 文件路径,默认使用 AppSettings 中的路径
|
||||
"""
|
||||
if env_path is not None:
|
||||
self.env_path = Path(env_path)
|
||||
else:
|
||||
# 从 AppSettings 获取路径
|
||||
from .app_settings import app_settings
|
||||
settings_path = app_settings.env_file_path
|
||||
if settings_path:
|
||||
self.env_path = Path(settings_path)
|
||||
else:
|
||||
# 回退到源码目录
|
||||
self.env_path = Path(__file__).resolve().parents[2] / ".env"
|
||||
|
||||
def load_env(self) -> Dict[str, str]:
|
||||
"""
|
||||
加载 .env 文件内容
|
||||
|
||||
Returns:
|
||||
环境变量字典
|
||||
"""
|
||||
env_vars = {}
|
||||
if not self.env_path.exists():
|
||||
return env_vars
|
||||
|
||||
try:
|
||||
content = self.env_path.read_text(encoding="utf-8", errors="ignore")
|
||||
for line in content.splitlines():
|
||||
parsed = self._parse_line(line)
|
||||
if parsed:
|
||||
key, value = parsed
|
||||
env_vars[key] = value
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return env_vars
|
||||
|
||||
def save_env(self, env_vars: Dict[str, str]) -> bool:
|
||||
"""
|
||||
保存环境变量到 .env 文件
|
||||
|
||||
Args:
|
||||
env_vars: 环境变量字典
|
||||
|
||||
Returns:
|
||||
是否保存成功
|
||||
"""
|
||||
try:
|
||||
lines = []
|
||||
# 按分组输出
|
||||
written_keys = set()
|
||||
|
||||
for group_id, group_info in ENV_GROUPS.items():
|
||||
group_lines = []
|
||||
for key in group_info["keys"]:
|
||||
if key in env_vars:
|
||||
value = env_vars[key]
|
||||
group_lines.append(self._format_line(key, value))
|
||||
written_keys.add(key)
|
||||
|
||||
if group_lines:
|
||||
lines.append(f"\n# {group_info['title']}")
|
||||
lines.extend(group_lines)
|
||||
|
||||
# 写入未分组的变量
|
||||
other_lines = []
|
||||
for key, value in env_vars.items():
|
||||
if key not in written_keys:
|
||||
other_lines.append(self._format_line(key, value))
|
||||
|
||||
if other_lines:
|
||||
lines.append("\n# 其他配置")
|
||||
lines.extend(other_lines)
|
||||
|
||||
content = "\n".join(lines).strip() + "\n"
|
||||
self.env_path.write_text(content, encoding="utf-8")
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def get_grouped_env(self) -> Dict[str, List[Tuple[str, str, bool]]]:
|
||||
"""
|
||||
获取分组的环境变量
|
||||
|
||||
Returns:
|
||||
分组字典 {group_id: [(key, value, is_sensitive), ...]}
|
||||
"""
|
||||
env_vars = self.load_env()
|
||||
result = {}
|
||||
used_keys = set()
|
||||
|
||||
for group_id, group_info in ENV_GROUPS.items():
|
||||
items = []
|
||||
for key in group_info["keys"]:
|
||||
value = env_vars.get(key, "")
|
||||
is_sensitive = key in group_info.get("sensitive", [])
|
||||
items.append((key, value, is_sensitive))
|
||||
if key in env_vars:
|
||||
used_keys.add(key)
|
||||
result[group_id] = items
|
||||
|
||||
# 添加未分组的变量到 "other" 组
|
||||
other_items = []
|
||||
for key, value in env_vars.items():
|
||||
if key not in used_keys:
|
||||
other_items.append((key, value, False))
|
||||
if other_items:
|
||||
result["other"] = other_items
|
||||
|
||||
return result
|
||||
|
||||
def validate_env(self, env_vars: Dict[str, str]) -> List[str]:
|
||||
"""
|
||||
验证环境变量
|
||||
|
||||
Args:
|
||||
env_vars: 环境变量字典
|
||||
|
||||
Returns:
|
||||
错误消息列表
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# 验证 PG_DSN 格式
|
||||
pg_dsn = env_vars.get("PG_DSN", "")
|
||||
if pg_dsn and not pg_dsn.startswith("postgresql://"):
|
||||
errors.append("PG_DSN 应以 'postgresql://' 开头")
|
||||
|
||||
# 验证端口号
|
||||
pg_port = env_vars.get("PG_PORT", "")
|
||||
if pg_port:
|
||||
try:
|
||||
port = int(pg_port)
|
||||
if port < 1 or port > 65535:
|
||||
errors.append("PG_PORT 应在 1-65535 范围内")
|
||||
except ValueError:
|
||||
errors.append("PG_PORT 应为数字")
|
||||
|
||||
# 验证 STORE_ID
|
||||
store_id = env_vars.get("STORE_ID", "")
|
||||
if store_id:
|
||||
try:
|
||||
int(store_id)
|
||||
except ValueError:
|
||||
errors.append("STORE_ID 应为数字")
|
||||
|
||||
# 验证路径存在性(可选)
|
||||
for key in ["EXPORT_ROOT", "LOG_ROOT", "FETCH_ROOT"]:
|
||||
path = env_vars.get(key, "")
|
||||
if path and not os.path.isabs(path):
|
||||
errors.append(f"{key} 建议使用绝对路径")
|
||||
|
||||
return errors
|
||||
|
||||
def mask_sensitive(self, value: str, visible_chars: int = 4) -> str:
|
||||
"""
|
||||
脱敏敏感值
|
||||
|
||||
Args:
|
||||
value: 原始值
|
||||
visible_chars: 可见字符数
|
||||
|
||||
Returns:
|
||||
脱敏后的值
|
||||
"""
|
||||
if not value or len(value) <= visible_chars:
|
||||
return "*" * len(value) if value else ""
|
||||
return value[:visible_chars] + "*" * (len(value) - visible_chars)
|
||||
|
||||
def _parse_line(self, line: str) -> Optional[Tuple[str, str]]:
|
||||
"""解析 .env 文件的一行"""
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
return None
|
||||
if stripped.startswith("export "):
|
||||
stripped = stripped[7:].strip()
|
||||
if "=" not in stripped:
|
||||
return None
|
||||
|
||||
key, value = stripped.split("=", 1)
|
||||
key = key.strip()
|
||||
value = self._unquote_value(value)
|
||||
return key, value
|
||||
|
||||
def _unquote_value(self, value: str) -> str:
|
||||
"""处理引号和注释"""
|
||||
# 去除内联注释
|
||||
value = self._strip_inline_comment(value)
|
||||
value = value.rstrip(",").strip()
|
||||
|
||||
if not value:
|
||||
return value
|
||||
|
||||
# 去除引号
|
||||
if len(value) >= 2 and value[0] in ("'", '"') and value[-1] == value[0]:
|
||||
return value[1:-1]
|
||||
if len(value) >= 3 and value[0] in ("r", "R") and value[1] in ("'", '"') and value[-1] == value[1]:
|
||||
return value[2:-1]
|
||||
|
||||
return value
|
||||
|
||||
def _strip_inline_comment(self, value: str) -> str:
|
||||
"""去除内联注释"""
|
||||
result = []
|
||||
in_quote = False
|
||||
quote_char = ""
|
||||
escape = False
|
||||
|
||||
for ch in value:
|
||||
if escape:
|
||||
result.append(ch)
|
||||
escape = False
|
||||
continue
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
result.append(ch)
|
||||
continue
|
||||
if ch in ("'", '"'):
|
||||
if not in_quote:
|
||||
in_quote = True
|
||||
quote_char = ch
|
||||
elif quote_char == ch:
|
||||
in_quote = False
|
||||
quote_char = ""
|
||||
result.append(ch)
|
||||
continue
|
||||
if ch == "#" and not in_quote:
|
||||
break
|
||||
result.append(ch)
|
||||
|
||||
return "".join(result).rstrip()
|
||||
|
||||
def _format_line(self, key: str, value: str) -> str:
|
||||
"""格式化为 .env 行"""
|
||||
# 如果值包含特殊字符,使用引号包裹
|
||||
if any(c in value for c in [' ', '"', "'", '#', '\n', '\r']):
|
||||
# 使用双引号,转义内部的双引号
|
||||
escaped = value.replace('\\', '\\\\').replace('"', '\\"')
|
||||
return f'{key}="{escaped}"'
|
||||
return f"{key}={value}"
|
||||
|
||||
@staticmethod
|
||||
def get_group_title(group_id: str) -> str:
|
||||
"""获取分组标题"""
|
||||
if group_id in ENV_GROUPS:
|
||||
return ENV_GROUPS[group_id]["title"]
|
||||
return "其他配置"
|
||||
|
||||
|
||||
# 全局实例
|
||||
config_helper = ConfigHelper()
|
||||
18
etl_billiards/gui/widgets/__init__.py
Normal file
18
etl_billiards/gui/widgets/__init__.py
Normal file
@@ -0,0 +1,18 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""GUI 组件模块"""
|
||||
|
||||
from .task_panel import TaskPanel
|
||||
from .env_editor import EnvEditor
|
||||
from .log_viewer import LogViewer
|
||||
from .db_viewer import DBViewer
|
||||
from .status_panel import StatusPanel
|
||||
from .task_manager import TaskManager
|
||||
|
||||
__all__ = [
|
||||
"TaskPanel",
|
||||
"EnvEditor",
|
||||
"LogViewer",
|
||||
"DBViewer",
|
||||
"StatusPanel",
|
||||
"TaskManager",
|
||||
]
|
||||
390
etl_billiards/gui/widgets/db_viewer.py
Normal file
390
etl_billiards/gui/widgets/db_viewer.py
Normal file
@@ -0,0 +1,390 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""数据库查看器"""
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QWidget, QVBoxLayout, QHBoxLayout, QSplitter,
|
||||
QGroupBox, QLabel, QPushButton, QLineEdit, QPlainTextEdit,
|
||||
QTableWidget, QTableWidgetItem, QTreeWidget, QTreeWidgetItem,
|
||||
QHeaderView, QComboBox, QTabWidget, QMessageBox, QFrame
|
||||
)
|
||||
from PySide6.QtCore import Qt, Signal
|
||||
from PySide6.QtGui import QFont
|
||||
|
||||
from ..workers.db_worker import DBWorker
|
||||
from ..utils.config_helper import ConfigHelper
|
||||
|
||||
|
||||
# 常用查询模板
|
||||
QUERY_TEMPLATES = {
|
||||
"ODS 行数统计": """
|
||||
SELECT
|
||||
table_name,
|
||||
(xpath('/row/cnt/text()',
|
||||
query_to_xml('SELECT COUNT(*) AS cnt FROM ' || table_schema || '.' || table_name, false, false, ''))
|
||||
)[1]::text::bigint AS row_count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'billiards_ods'
|
||||
ORDER BY table_name;
|
||||
""",
|
||||
"DWD 行数统计": """
|
||||
SELECT
|
||||
table_name,
|
||||
(xpath('/row/cnt/text()',
|
||||
query_to_xml('SELECT COUNT(*) AS cnt FROM ' || table_schema || '.' || table_name, false, false, ''))
|
||||
)[1]::text::bigint AS row_count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'billiards_dwd'
|
||||
ORDER BY table_name;
|
||||
""",
|
||||
"ETL 游标状态": """
|
||||
SELECT
|
||||
task_code,
|
||||
last_start,
|
||||
last_end,
|
||||
last_run_id,
|
||||
updated_at
|
||||
FROM etl_admin.etl_cursor
|
||||
ORDER BY task_code;
|
||||
""",
|
||||
"最近运行记录": """
|
||||
SELECT
|
||||
run_id,
|
||||
task_code,
|
||||
status,
|
||||
started_at,
|
||||
finished_at,
|
||||
EXTRACT(EPOCH FROM (finished_at - started_at))::int AS duration_sec,
|
||||
rows_affected
|
||||
FROM etl_admin.run_tracker
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 50;
|
||||
""",
|
||||
"ODS 最新入库时间": """
|
||||
SELECT
|
||||
'payment_transactions' AS table_name, MAX(fetched_at) AS max_fetched_at FROM billiards_ods.payment_transactions
|
||||
UNION ALL
|
||||
SELECT 'member_profiles', MAX(fetched_at) FROM billiards_ods.member_profiles
|
||||
UNION ALL
|
||||
SELECT 'settlement_records', MAX(fetched_at) FROM billiards_ods.settlement_records
|
||||
UNION ALL
|
||||
SELECT 'recharge_settlements', MAX(fetched_at) FROM billiards_ods.recharge_settlements
|
||||
ORDER BY table_name;
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
class DBViewer(QWidget):
|
||||
"""数据库查看器"""
|
||||
|
||||
# 信号
|
||||
connection_changed = Signal(bool, str) # 连接状态变化
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.config_helper = ConfigHelper()
|
||||
self.db_worker = DBWorker(self)
|
||||
self._connected = False
|
||||
|
||||
self._init_ui()
|
||||
self._connect_signals()
|
||||
self._load_dsn_from_env()
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(16, 16, 16, 16)
|
||||
layout.setSpacing(16)
|
||||
|
||||
# 标题
|
||||
title = QLabel("数据库查看器")
|
||||
title.setProperty("heading", True)
|
||||
layout.addWidget(title)
|
||||
|
||||
# 连接配置
|
||||
conn_group = QGroupBox("数据库连接")
|
||||
conn_layout = QHBoxLayout(conn_group)
|
||||
|
||||
conn_layout.addWidget(QLabel("DSN:"))
|
||||
self.dsn_edit = QLineEdit()
|
||||
self.dsn_edit.setPlaceholderText("postgresql://user:password@host:5432/dbname")
|
||||
self.dsn_edit.setEchoMode(QLineEdit.Password)
|
||||
conn_layout.addWidget(self.dsn_edit, 1)
|
||||
|
||||
self.show_dsn_btn = QPushButton("显示")
|
||||
self.show_dsn_btn.setProperty("secondary", True)
|
||||
self.show_dsn_btn.setCheckable(True)
|
||||
self.show_dsn_btn.setFixedWidth(60)
|
||||
conn_layout.addWidget(self.show_dsn_btn)
|
||||
|
||||
self.connect_btn = QPushButton("连接")
|
||||
self.connect_btn.setFixedWidth(80)
|
||||
conn_layout.addWidget(self.connect_btn)
|
||||
|
||||
self.disconnect_btn = QPushButton("断开")
|
||||
self.disconnect_btn.setProperty("secondary", True)
|
||||
self.disconnect_btn.setFixedWidth(80)
|
||||
self.disconnect_btn.setEnabled(False)
|
||||
conn_layout.addWidget(self.disconnect_btn)
|
||||
|
||||
layout.addWidget(conn_group)
|
||||
|
||||
# 主分割器
|
||||
main_splitter = QSplitter(Qt.Horizontal)
|
||||
layout.addWidget(main_splitter, 1)
|
||||
|
||||
# 左侧:表浏览器
|
||||
left_widget = self._create_table_browser()
|
||||
main_splitter.addWidget(left_widget)
|
||||
|
||||
# 右侧:查询和结果
|
||||
right_widget = self._create_query_area()
|
||||
main_splitter.addWidget(right_widget)
|
||||
|
||||
# 设置分割比例
|
||||
main_splitter.setSizes([300, 700])
|
||||
|
||||
def _create_table_browser(self) -> QWidget:
|
||||
"""创建表浏览器"""
|
||||
widget = QWidget()
|
||||
layout = QVBoxLayout(widget)
|
||||
layout.setContentsMargins(0, 0, 8, 0)
|
||||
|
||||
# 标题和刷新按钮
|
||||
header_layout = QHBoxLayout()
|
||||
header_layout.addWidget(QLabel("表结构"))
|
||||
self.refresh_tables_btn = QPushButton("刷新")
|
||||
self.refresh_tables_btn.setProperty("secondary", True)
|
||||
self.refresh_tables_btn.setEnabled(False)
|
||||
header_layout.addWidget(self.refresh_tables_btn)
|
||||
layout.addLayout(header_layout)
|
||||
|
||||
# 表树形视图
|
||||
self.table_tree = QTreeWidget()
|
||||
self.table_tree.setHeaderLabels(["名称", "行数", "最后更新"])
|
||||
self.table_tree.header().setSectionResizeMode(0, QHeaderView.Stretch)
|
||||
self.table_tree.setColumnWidth(1, 80)
|
||||
self.table_tree.setColumnWidth(2, 130)
|
||||
layout.addWidget(self.table_tree, 1)
|
||||
|
||||
return widget
|
||||
|
||||
def _create_query_area(self) -> QWidget:
|
||||
"""创建查询区域"""
|
||||
widget = QWidget()
|
||||
layout = QVBoxLayout(widget)
|
||||
layout.setContentsMargins(8, 0, 0, 0)
|
||||
|
||||
# 查询输入区
|
||||
query_group = QGroupBox("SQL 查询")
|
||||
query_layout = QVBoxLayout(query_group)
|
||||
|
||||
# 模板选择
|
||||
template_layout = QHBoxLayout()
|
||||
template_layout.addWidget(QLabel("常用查询:"))
|
||||
self.template_combo = QComboBox()
|
||||
self.template_combo.addItem("-- 选择模板 --")
|
||||
for name in QUERY_TEMPLATES.keys():
|
||||
self.template_combo.addItem(name)
|
||||
template_layout.addWidget(self.template_combo, 1)
|
||||
query_layout.addLayout(template_layout)
|
||||
|
||||
# SQL 编辑器
|
||||
self.sql_editor = QPlainTextEdit()
|
||||
self.sql_editor.setObjectName("sqlEditor")
|
||||
self.sql_editor.setPlaceholderText("输入 SQL 查询语句...")
|
||||
self.sql_editor.setFont(QFont("Consolas", 11))
|
||||
self.sql_editor.setMaximumHeight(150)
|
||||
query_layout.addWidget(self.sql_editor)
|
||||
|
||||
# 执行按钮
|
||||
exec_layout = QHBoxLayout()
|
||||
exec_layout.addStretch()
|
||||
|
||||
self.exec_btn = QPushButton("执行查询 (Ctrl+Enter)")
|
||||
self.exec_btn.setEnabled(False)
|
||||
exec_layout.addWidget(self.exec_btn)
|
||||
|
||||
query_layout.addLayout(exec_layout)
|
||||
layout.addWidget(query_group)
|
||||
|
||||
# 结果区域
|
||||
result_group = QGroupBox("查询结果")
|
||||
result_layout = QVBoxLayout(result_group)
|
||||
|
||||
# 结果表格
|
||||
self.result_table = QTableWidget()
|
||||
self.result_table.setAlternatingRowColors(True)
|
||||
self.result_table.horizontalHeader().setStretchLastSection(True)
|
||||
result_layout.addWidget(self.result_table, 1)
|
||||
|
||||
# 结果统计
|
||||
self.result_label = QLabel("就绪")
|
||||
self.result_label.setProperty("subheading", True)
|
||||
result_layout.addWidget(self.result_label)
|
||||
|
||||
layout.addWidget(result_group, 1)
|
||||
|
||||
return widget
|
||||
|
||||
def _connect_signals(self):
|
||||
"""连接信号"""
|
||||
# 连接按钮
|
||||
self.show_dsn_btn.toggled.connect(self._toggle_dsn_visibility)
|
||||
self.connect_btn.clicked.connect(self._connect_db)
|
||||
self.disconnect_btn.clicked.connect(self._disconnect_db)
|
||||
self.refresh_tables_btn.clicked.connect(self._refresh_tables)
|
||||
|
||||
# 模板选择
|
||||
self.template_combo.currentIndexChanged.connect(self._on_template_selected)
|
||||
|
||||
# 执行查询
|
||||
self.exec_btn.clicked.connect(self._execute_query)
|
||||
|
||||
# 表双击
|
||||
self.table_tree.itemDoubleClicked.connect(self._on_table_double_clicked)
|
||||
|
||||
# 工作线程信号
|
||||
self.db_worker.connection_status.connect(self._on_connection_status)
|
||||
self.db_worker.tables_loaded.connect(self._on_tables_loaded)
|
||||
self.db_worker.query_finished.connect(self._on_query_finished)
|
||||
self.db_worker.query_error.connect(self._on_query_error)
|
||||
|
||||
def _load_dsn_from_env(self):
|
||||
"""从环境变量加载 DSN"""
|
||||
env_vars = self.config_helper.load_env()
|
||||
dsn = env_vars.get("PG_DSN", "")
|
||||
if dsn:
|
||||
self.dsn_edit.setText(dsn)
|
||||
|
||||
def _toggle_dsn_visibility(self, checked: bool):
|
||||
"""切换 DSN 可见性"""
|
||||
self.dsn_edit.setEchoMode(
|
||||
QLineEdit.Normal if checked else QLineEdit.Password
|
||||
)
|
||||
self.show_dsn_btn.setText("隐藏" if checked else "显示")
|
||||
|
||||
def _connect_db(self):
|
||||
"""连接数据库"""
|
||||
dsn = self.dsn_edit.text().strip()
|
||||
if not dsn:
|
||||
QMessageBox.warning(self, "提示", "请输入数据库连接字符串")
|
||||
return
|
||||
|
||||
self.connect_btn.setEnabled(False)
|
||||
self.connect_btn.setText("连接中...")
|
||||
self.db_worker.connect_db(dsn)
|
||||
|
||||
def _disconnect_db(self):
|
||||
"""断开数据库连接"""
|
||||
self.db_worker.disconnect_db()
|
||||
|
||||
def _refresh_tables(self):
|
||||
"""刷新表列表"""
|
||||
self.db_worker.load_tables()
|
||||
|
||||
def _on_connection_status(self, connected: bool, message: str):
|
||||
"""处理连接状态变化"""
|
||||
self._connected = connected
|
||||
self.connect_btn.setEnabled(not connected)
|
||||
self.connect_btn.setText("连接")
|
||||
self.disconnect_btn.setEnabled(connected)
|
||||
self.refresh_tables_btn.setEnabled(connected)
|
||||
self.exec_btn.setEnabled(connected)
|
||||
|
||||
self.connection_changed.emit(connected, message)
|
||||
|
||||
if connected:
|
||||
# 自动加载表列表
|
||||
self._refresh_tables()
|
||||
|
||||
def _on_tables_loaded(self, tables_dict: dict):
|
||||
"""处理表列表加载完成"""
|
||||
self.table_tree.clear()
|
||||
|
||||
for schema, tables in tables_dict.items():
|
||||
schema_item = QTreeWidgetItem([schema, "", ""])
|
||||
schema_item.setExpanded(True)
|
||||
|
||||
for table_name, row_count, updated_at in tables:
|
||||
table_item = QTreeWidgetItem([table_name, str(row_count), updated_at])
|
||||
table_item.setData(0, Qt.UserRole, f"{schema}.{table_name}")
|
||||
schema_item.addChild(table_item)
|
||||
|
||||
self.table_tree.addTopLevelItem(schema_item)
|
||||
|
||||
def _on_template_selected(self, index: int):
|
||||
"""模板选择变化"""
|
||||
if index <= 0:
|
||||
return
|
||||
|
||||
template_name = self.template_combo.currentText()
|
||||
if template_name in QUERY_TEMPLATES:
|
||||
self.sql_editor.setPlainText(QUERY_TEMPLATES[template_name].strip())
|
||||
|
||||
# 重置选择
|
||||
self.template_combo.setCurrentIndex(0)
|
||||
|
||||
def _on_table_double_clicked(self, item: QTreeWidgetItem, column: int):
|
||||
"""表双击事件"""
|
||||
full_name = item.data(0, Qt.UserRole)
|
||||
if full_name:
|
||||
# 生成预览查询
|
||||
sql = f"SELECT * FROM {full_name} LIMIT 100;"
|
||||
self.sql_editor.setPlainText(sql)
|
||||
self._execute_query()
|
||||
|
||||
def _execute_query(self):
|
||||
"""执行查询"""
|
||||
sql = self.sql_editor.toPlainText().strip()
|
||||
if not sql:
|
||||
QMessageBox.warning(self, "提示", "请输入 SQL 语句")
|
||||
return
|
||||
|
||||
self.exec_btn.setEnabled(False)
|
||||
self.exec_btn.setText("执行中...")
|
||||
self.result_label.setText("正在查询...")
|
||||
|
||||
self.db_worker.execute_query(sql)
|
||||
|
||||
def _on_query_finished(self, columns: list, rows: list):
|
||||
"""查询完成"""
|
||||
self.exec_btn.setEnabled(True)
|
||||
self.exec_btn.setText("执行查询 (Ctrl+Enter)")
|
||||
|
||||
# 更新结果表格
|
||||
self.result_table.clear()
|
||||
self.result_table.setColumnCount(len(columns))
|
||||
self.result_table.setRowCount(len(rows))
|
||||
self.result_table.setHorizontalHeaderLabels(columns)
|
||||
|
||||
for row_idx, row_data in enumerate(rows):
|
||||
for col_idx, col_name in enumerate(columns):
|
||||
value = row_data.get(col_name, "")
|
||||
item = QTableWidgetItem(str(value) if value is not None else "NULL")
|
||||
if value is None:
|
||||
item.setForeground(Qt.gray)
|
||||
self.result_table.setItem(row_idx, col_idx, item)
|
||||
|
||||
# 更新统计
|
||||
self.result_label.setText(f"返回 {len(rows)} 行, {len(columns)} 列")
|
||||
|
||||
def _on_query_error(self, error: str):
|
||||
"""查询错误"""
|
||||
self.exec_btn.setEnabled(True)
|
||||
self.exec_btn.setText("执行查询 (Ctrl+Enter)")
|
||||
self.result_label.setText(f"错误: {error}")
|
||||
QMessageBox.critical(self, "查询错误", error)
|
||||
|
||||
def close_connection(self):
|
||||
"""关闭连接"""
|
||||
if self._connected:
|
||||
self.db_worker.disconnect_db()
|
||||
|
||||
def keyPressEvent(self, event):
|
||||
"""键盘事件"""
|
||||
# Ctrl+Enter 执行查询
|
||||
if event.modifiers() == Qt.ControlModifier and event.key() == Qt.Key_Return:
|
||||
if self._connected:
|
||||
self._execute_query()
|
||||
else:
|
||||
super().keyPressEvent(event)
|
||||
318
etl_billiards/gui/widgets/env_editor.py
Normal file
318
etl_billiards/gui/widgets/env_editor.py
Normal file
@@ -0,0 +1,318 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""环境变量编辑器"""
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||||
QGroupBox, QLabel, QLineEdit, QPushButton, QScrollArea,
|
||||
QFrame, QMessageBox, QFileDialog, QCheckBox
|
||||
)
|
||||
from PySide6.QtCore import Qt, Signal
|
||||
|
||||
from ..utils.config_helper import ConfigHelper, ENV_GROUPS
|
||||
|
||||
|
||||
class EnvEditor(QWidget):
|
||||
"""环境变量编辑器"""
|
||||
|
||||
# 信号
|
||||
config_saved = Signal() # 配置保存成功
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.config_helper = ConfigHelper()
|
||||
self.field_widgets = {} # 存储字段控件
|
||||
self.show_sensitive = False
|
||||
|
||||
self._init_ui()
|
||||
self.load_config()
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(16, 16, 16, 16)
|
||||
layout.setSpacing(16)
|
||||
|
||||
# 标题和按钮
|
||||
header_layout = QHBoxLayout()
|
||||
|
||||
title = QLabel("环境配置")
|
||||
title.setProperty("heading", True)
|
||||
header_layout.addWidget(title)
|
||||
|
||||
header_layout.addStretch()
|
||||
|
||||
self.show_sensitive_check = QCheckBox("显示敏感信息")
|
||||
self.show_sensitive_check.stateChanged.connect(self._toggle_sensitive)
|
||||
header_layout.addWidget(self.show_sensitive_check)
|
||||
|
||||
self.import_btn = QPushButton("导入")
|
||||
self.import_btn.setProperty("secondary", True)
|
||||
self.import_btn.clicked.connect(self._import_config)
|
||||
header_layout.addWidget(self.import_btn)
|
||||
|
||||
self.export_btn = QPushButton("导出")
|
||||
self.export_btn.setProperty("secondary", True)
|
||||
self.export_btn.clicked.connect(self._export_config)
|
||||
header_layout.addWidget(self.export_btn)
|
||||
|
||||
self.reload_btn = QPushButton("重新加载")
|
||||
self.reload_btn.setProperty("secondary", True)
|
||||
self.reload_btn.clicked.connect(self.load_config)
|
||||
header_layout.addWidget(self.reload_btn)
|
||||
|
||||
self.save_btn = QPushButton("保存")
|
||||
self.save_btn.clicked.connect(self._save_config)
|
||||
header_layout.addWidget(self.save_btn)
|
||||
|
||||
layout.addLayout(header_layout)
|
||||
|
||||
# 配置文件路径
|
||||
path_layout = QHBoxLayout()
|
||||
path_layout.addWidget(QLabel("配置文件:"))
|
||||
self.path_label = QLabel(str(self.config_helper.env_path))
|
||||
self.path_label.setProperty("subheading", True)
|
||||
path_layout.addWidget(self.path_label, 1)
|
||||
layout.addLayout(path_layout)
|
||||
|
||||
# 滚动区域
|
||||
scroll_area = QScrollArea()
|
||||
scroll_area.setWidgetResizable(True)
|
||||
scroll_area.setFrameShape(QFrame.NoFrame)
|
||||
layout.addWidget(scroll_area, 1)
|
||||
|
||||
# 配置组容器
|
||||
config_widget = QWidget()
|
||||
self.config_layout = QVBoxLayout(config_widget)
|
||||
self.config_layout.setSpacing(16)
|
||||
|
||||
# 创建各配置组
|
||||
self._create_config_groups()
|
||||
|
||||
# 弹性空间
|
||||
self.config_layout.addStretch()
|
||||
|
||||
scroll_area.setWidget(config_widget)
|
||||
|
||||
# 验证结果
|
||||
self.validation_label = QLabel()
|
||||
self.validation_label.setWordWrap(True)
|
||||
layout.addWidget(self.validation_label)
|
||||
|
||||
def _create_config_groups(self):
|
||||
"""创建配置分组"""
|
||||
for group_id, group_info in ENV_GROUPS.items():
|
||||
group = QGroupBox(group_info["title"])
|
||||
grid_layout = QGridLayout(group)
|
||||
|
||||
for row, key in enumerate(group_info["keys"]):
|
||||
# 标签
|
||||
label = QLabel(f"{key}:")
|
||||
label.setMinimumWidth(180)
|
||||
grid_layout.addWidget(label, row, 0)
|
||||
|
||||
# 输入框
|
||||
edit = QLineEdit()
|
||||
edit.setPlaceholderText(self._get_placeholder(key))
|
||||
|
||||
# 敏感字段处理
|
||||
if key in group_info.get("sensitive", []):
|
||||
edit.setEchoMode(QLineEdit.Password)
|
||||
edit.setProperty("sensitive", True)
|
||||
|
||||
edit.textChanged.connect(self._on_value_changed)
|
||||
grid_layout.addWidget(edit, row, 1)
|
||||
|
||||
# 存储控件引用
|
||||
self.field_widgets[key] = edit
|
||||
|
||||
self.config_layout.addWidget(group)
|
||||
|
||||
# 其他配置组(动态添加)
|
||||
self.other_group = QGroupBox("其他配置")
|
||||
self.other_layout = QGridLayout(self.other_group)
|
||||
self.other_group.setVisible(False)
|
||||
self.config_layout.addWidget(self.other_group)
|
||||
|
||||
def load_config(self):
|
||||
"""加载配置"""
|
||||
env_vars = self.config_helper.load_env()
|
||||
|
||||
# 更新已知字段
|
||||
for key, edit in self.field_widgets.items():
|
||||
value = env_vars.get(key, "")
|
||||
edit.blockSignals(True)
|
||||
edit.setText(value)
|
||||
edit.blockSignals(False)
|
||||
|
||||
# 处理其他字段
|
||||
known_keys = set(self.field_widgets.keys())
|
||||
other_keys = [k for k in env_vars.keys() if k not in known_keys]
|
||||
|
||||
# 清除旧的其他字段
|
||||
while self.other_layout.count():
|
||||
item = self.other_layout.takeAt(0)
|
||||
if item.widget():
|
||||
item.widget().deleteLater()
|
||||
|
||||
# 添加其他字段
|
||||
if other_keys:
|
||||
self.other_group.setVisible(True)
|
||||
for row, key in enumerate(sorted(other_keys)):
|
||||
label = QLabel(f"{key}:")
|
||||
self.other_layout.addWidget(label, row, 0)
|
||||
|
||||
edit = QLineEdit(env_vars[key])
|
||||
edit.textChanged.connect(self._on_value_changed)
|
||||
self.other_layout.addWidget(edit, row, 1)
|
||||
|
||||
self.field_widgets[key] = edit
|
||||
else:
|
||||
self.other_group.setVisible(False)
|
||||
|
||||
self._validate()
|
||||
|
||||
def _save_config(self):
|
||||
"""保存配置"""
|
||||
# 收集所有值
|
||||
env_vars = {}
|
||||
for key, edit in self.field_widgets.items():
|
||||
value = edit.text().strip()
|
||||
if value:
|
||||
env_vars[key] = value
|
||||
|
||||
# 验证
|
||||
errors = self.config_helper.validate_env(env_vars)
|
||||
if errors:
|
||||
reply = QMessageBox.question(
|
||||
self,
|
||||
"验证警告",
|
||||
"配置存在以下问题:\n\n" + "\n".join(f"• {e}" for e in errors) + "\n\n是否仍要保存?",
|
||||
QMessageBox.Yes | QMessageBox.No,
|
||||
QMessageBox.No
|
||||
)
|
||||
if reply == QMessageBox.No:
|
||||
return
|
||||
|
||||
# 保存
|
||||
if self.config_helper.save_env(env_vars):
|
||||
QMessageBox.information(self, "成功", "配置已保存")
|
||||
self.config_saved.emit()
|
||||
else:
|
||||
QMessageBox.critical(self, "错误", "保存配置失败")
|
||||
|
||||
def _import_config(self):
|
||||
"""导入配置"""
|
||||
file_path, _ = QFileDialog.getOpenFileName(
|
||||
self,
|
||||
"导入配置文件",
|
||||
"",
|
||||
"环境文件 (*.env);;所有文件 (*.*)"
|
||||
)
|
||||
if not file_path:
|
||||
return
|
||||
|
||||
try:
|
||||
from pathlib import Path
|
||||
temp_helper = ConfigHelper(Path(file_path))
|
||||
env_vars = temp_helper.load_env()
|
||||
|
||||
# 更新字段
|
||||
for key, value in env_vars.items():
|
||||
if key in self.field_widgets:
|
||||
self.field_widgets[key].setText(value)
|
||||
|
||||
QMessageBox.information(self, "成功", f"已导入 {len(env_vars)} 个配置项")
|
||||
except Exception as e:
|
||||
QMessageBox.critical(self, "错误", f"导入失败: {e}")
|
||||
|
||||
def _export_config(self):
|
||||
"""导出配置"""
|
||||
file_path, _ = QFileDialog.getSaveFileName(
|
||||
self,
|
||||
"导出配置文件",
|
||||
".env.backup",
|
||||
"环境文件 (*.env);;所有文件 (*.*)"
|
||||
)
|
||||
if not file_path:
|
||||
return
|
||||
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
# 收集当前值
|
||||
env_vars = {}
|
||||
for key, edit in self.field_widgets.items():
|
||||
value = edit.text().strip()
|
||||
if value:
|
||||
env_vars[key] = value
|
||||
|
||||
# 保存到指定路径
|
||||
temp_helper = ConfigHelper(Path(file_path))
|
||||
if temp_helper.save_env(env_vars):
|
||||
QMessageBox.information(self, "成功", f"配置已导出到:\n{file_path}")
|
||||
else:
|
||||
QMessageBox.critical(self, "错误", "导出失败")
|
||||
except Exception as e:
|
||||
QMessageBox.critical(self, "错误", f"导出失败: {e}")
|
||||
|
||||
def _toggle_sensitive(self, state: int):
|
||||
"""切换敏感信息显示"""
|
||||
self.show_sensitive = state == Qt.Checked
|
||||
|
||||
for key, edit in self.field_widgets.items():
|
||||
if edit.property("sensitive"):
|
||||
edit.setEchoMode(
|
||||
QLineEdit.Normal if self.show_sensitive else QLineEdit.Password
|
||||
)
|
||||
|
||||
def _on_value_changed(self):
|
||||
"""值变化时验证"""
|
||||
self._validate()
|
||||
|
||||
def _validate(self):
|
||||
"""验证配置"""
|
||||
env_vars = {}
|
||||
for key, edit in self.field_widgets.items():
|
||||
value = edit.text().strip()
|
||||
if value:
|
||||
env_vars[key] = value
|
||||
|
||||
errors = self.config_helper.validate_env(env_vars)
|
||||
|
||||
if errors:
|
||||
self.validation_label.setText("⚠ " + "; ".join(errors))
|
||||
self.validation_label.setProperty("status", "warning")
|
||||
else:
|
||||
self.validation_label.setText("✓ 配置验证通过")
|
||||
self.validation_label.setProperty("status", "success")
|
||||
|
||||
self.validation_label.style().unpolish(self.validation_label)
|
||||
self.validation_label.style().polish(self.validation_label)
|
||||
|
||||
@staticmethod
|
||||
def _get_placeholder(key: str) -> str:
|
||||
"""获取占位符提示"""
|
||||
placeholders = {
|
||||
"PG_DSN": "postgresql://user:password@host:5432/dbname",
|
||||
"PG_HOST": "localhost",
|
||||
"PG_PORT": "5432",
|
||||
"PG_NAME": "billiards",
|
||||
"PG_USER": "postgres",
|
||||
"PG_PASSWORD": "密码",
|
||||
"API_BASE": "https://pc.ficoo.vip/apiprod/admin/v1",
|
||||
"API_TOKEN": "Bearer token",
|
||||
"API_TIMEOUT": "20",
|
||||
"API_PAGE_SIZE": "200",
|
||||
"STORE_ID": "门店ID (数字)",
|
||||
"TIMEZONE": "Asia/Taipei",
|
||||
"EXPORT_ROOT": "export/JSON",
|
||||
"LOG_ROOT": "export/LOG",
|
||||
"FETCH_ROOT": "JSON 抓取输出目录",
|
||||
"INGEST_SOURCE_DIR": "本地 JSON 输入目录",
|
||||
"PIPELINE_FLOW": "FULL / FETCH_ONLY / INGEST_ONLY",
|
||||
"RUN_TASKS": "任务列表,逗号分隔",
|
||||
"OVERLAP_SECONDS": "3600",
|
||||
"WINDOW_START": "2025-07-01 00:00:00",
|
||||
"WINDOW_END": "2025-08-01 00:00:00",
|
||||
}
|
||||
return placeholders.get(key, "")
|
||||
247
etl_billiards/gui/widgets/log_viewer.py
Normal file
247
etl_billiards/gui/widgets/log_viewer.py
Normal file
@@ -0,0 +1,247 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""日志查看器"""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QWidget, QVBoxLayout, QHBoxLayout,
|
||||
QPlainTextEdit, QPushButton, QLineEdit, QLabel,
|
||||
QComboBox, QCheckBox, QFileDialog, QMessageBox
|
||||
)
|
||||
from PySide6.QtCore import Qt, Signal, Slot
|
||||
from PySide6.QtGui import QTextCharFormat, QColor, QFont, QTextCursor
|
||||
|
||||
|
||||
class LogViewer(QWidget):
|
||||
"""日志查看器"""
|
||||
|
||||
# 信号
|
||||
log_cleared = Signal()
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.max_lines = 10000
|
||||
self.auto_scroll = True
|
||||
self.filter_text = ""
|
||||
self.filter_level = "ALL"
|
||||
self._all_logs = [] # 存储所有日志
|
||||
|
||||
self._init_ui()
|
||||
self._connect_signals()
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(16, 16, 16, 16)
|
||||
layout.setSpacing(8)
|
||||
|
||||
# 标题和工具栏
|
||||
header_layout = QHBoxLayout()
|
||||
|
||||
title = QLabel("执行日志")
|
||||
title.setProperty("heading", True)
|
||||
header_layout.addWidget(title)
|
||||
|
||||
header_layout.addStretch()
|
||||
|
||||
# 日志级别过滤
|
||||
header_layout.addWidget(QLabel("级别:"))
|
||||
self.level_combo = QComboBox()
|
||||
self.level_combo.addItems(["ALL", "INFO", "WARNING", "ERROR", "DEBUG"])
|
||||
self.level_combo.setFixedWidth(100)
|
||||
header_layout.addWidget(self.level_combo)
|
||||
|
||||
# 搜索框
|
||||
header_layout.addWidget(QLabel("搜索:"))
|
||||
self.search_edit = QLineEdit()
|
||||
self.search_edit.setPlaceholderText("输入关键字...")
|
||||
self.search_edit.setFixedWidth(200)
|
||||
header_layout.addWidget(self.search_edit)
|
||||
|
||||
# 自动滚动
|
||||
self.auto_scroll_check = QCheckBox("自动滚动")
|
||||
self.auto_scroll_check.setChecked(True)
|
||||
header_layout.addWidget(self.auto_scroll_check)
|
||||
|
||||
layout.addLayout(header_layout)
|
||||
|
||||
# 日志文本区域
|
||||
self.log_text = QPlainTextEdit()
|
||||
self.log_text.setObjectName("logViewer")
|
||||
self.log_text.setReadOnly(True)
|
||||
self.log_text.setFont(QFont("Consolas", 10))
|
||||
self.log_text.setLineWrapMode(QPlainTextEdit.NoWrap)
|
||||
layout.addWidget(self.log_text, 1)
|
||||
|
||||
# 底部工具栏
|
||||
footer_layout = QHBoxLayout()
|
||||
|
||||
self.line_count_label = QLabel("0 行")
|
||||
self.line_count_label.setProperty("subheading", True)
|
||||
footer_layout.addWidget(self.line_count_label)
|
||||
|
||||
footer_layout.addStretch()
|
||||
|
||||
self.copy_btn = QPushButton("复制全部")
|
||||
self.copy_btn.setProperty("secondary", True)
|
||||
footer_layout.addWidget(self.copy_btn)
|
||||
|
||||
self.export_btn = QPushButton("导出")
|
||||
self.export_btn.setProperty("secondary", True)
|
||||
footer_layout.addWidget(self.export_btn)
|
||||
|
||||
self.clear_btn = QPushButton("清空")
|
||||
self.clear_btn.setProperty("secondary", True)
|
||||
footer_layout.addWidget(self.clear_btn)
|
||||
|
||||
layout.addLayout(footer_layout)
|
||||
|
||||
def _connect_signals(self):
|
||||
"""连接信号"""
|
||||
self.level_combo.currentTextChanged.connect(self._apply_filter)
|
||||
self.search_edit.textChanged.connect(self._apply_filter)
|
||||
self.auto_scroll_check.stateChanged.connect(self._toggle_auto_scroll)
|
||||
self.copy_btn.clicked.connect(self._copy_all)
|
||||
self.export_btn.clicked.connect(self._export_log)
|
||||
self.clear_btn.clicked.connect(self._clear_log)
|
||||
|
||||
@Slot(str)
|
||||
def append_log(self, text: str):
|
||||
"""追加日志"""
|
||||
# 添加时间戳(如果没有)
|
||||
if not re.match(r'^\d{4}-\d{2}-\d{2}', text) and not text.startswith('['):
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
text = f"[{timestamp}] {text}"
|
||||
|
||||
# 存储到全部日志
|
||||
self._all_logs.append(text)
|
||||
|
||||
# 限制日志行数
|
||||
if len(self._all_logs) > self.max_lines:
|
||||
self._all_logs = self._all_logs[-self.max_lines:]
|
||||
|
||||
# 检查是否通过过滤器
|
||||
if self._matches_filter(text):
|
||||
self._append_formatted_line(text)
|
||||
|
||||
# 更新行数
|
||||
self._update_line_count()
|
||||
|
||||
def _append_formatted_line(self, text: str):
|
||||
"""追加格式化的行"""
|
||||
cursor = self.log_text.textCursor()
|
||||
cursor.movePosition(QTextCursor.End)
|
||||
|
||||
# 设置格式
|
||||
fmt = QTextCharFormat()
|
||||
|
||||
text_lower = text.lower()
|
||||
if "[error]" in text_lower or "错误" in text or "失败" in text:
|
||||
fmt.setForeground(QColor("#d93025"))
|
||||
fmt.setFontWeight(QFont.Bold)
|
||||
elif "[warning]" in text_lower or "警告" in text or "warn" in text_lower:
|
||||
fmt.setForeground(QColor("#f9ab00"))
|
||||
elif "[info]" in text_lower:
|
||||
fmt.setForeground(QColor("#1a73e8"))
|
||||
elif "[debug]" in text_lower:
|
||||
fmt.setForeground(QColor("#9aa0a6"))
|
||||
elif "[gui]" in text_lower:
|
||||
fmt.setForeground(QColor("#1e8e3e"))
|
||||
else:
|
||||
fmt.setForeground(QColor("#333333"))
|
||||
|
||||
cursor.insertText(text + "\n", fmt)
|
||||
|
||||
# 自动滚动
|
||||
if self.auto_scroll:
|
||||
self.log_text.verticalScrollBar().setValue(
|
||||
self.log_text.verticalScrollBar().maximum()
|
||||
)
|
||||
|
||||
def _matches_filter(self, text: str) -> bool:
|
||||
"""检查是否匹配过滤器"""
|
||||
# 级别过滤
|
||||
if self.filter_level != "ALL":
|
||||
level_marker = f"[{self.filter_level}]"
|
||||
if level_marker.lower() not in text.lower():
|
||||
return False
|
||||
|
||||
# 文本过滤
|
||||
if self.filter_text:
|
||||
if self.filter_text.lower() not in text.lower():
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _apply_filter(self):
|
||||
"""应用过滤器"""
|
||||
self.filter_level = self.level_combo.currentText()
|
||||
self.filter_text = self.search_edit.text().strip()
|
||||
|
||||
# 重新显示日志
|
||||
self.log_text.clear()
|
||||
for line in self._all_logs:
|
||||
if self._matches_filter(line):
|
||||
self._append_formatted_line(line)
|
||||
|
||||
self._update_line_count()
|
||||
|
||||
def _toggle_auto_scroll(self, state: int):
|
||||
"""切换自动滚动"""
|
||||
self.auto_scroll = state == Qt.Checked
|
||||
|
||||
def _copy_all(self):
|
||||
"""复制全部日志"""
|
||||
from PySide6.QtWidgets import QApplication
|
||||
text = self.log_text.toPlainText()
|
||||
QApplication.clipboard().setText(text)
|
||||
QMessageBox.information(self, "提示", "日志已复制到剪贴板")
|
||||
|
||||
def _export_log(self):
|
||||
"""导出日志"""
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
default_name = f"etl_log_{timestamp}.txt"
|
||||
|
||||
file_path, _ = QFileDialog.getSaveFileName(
|
||||
self,
|
||||
"导出日志",
|
||||
default_name,
|
||||
"文本文件 (*.txt);;日志文件 (*.log);;所有文件 (*.*)"
|
||||
)
|
||||
|
||||
if not file_path:
|
||||
return
|
||||
|
||||
try:
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
f.write(self.log_text.toPlainText())
|
||||
QMessageBox.information(self, "成功", f"日志已导出到:\n{file_path}")
|
||||
except Exception as e:
|
||||
QMessageBox.critical(self, "错误", f"导出失败: {e}")
|
||||
|
||||
def _clear_log(self):
|
||||
"""清空日志"""
|
||||
reply = QMessageBox.question(
|
||||
self,
|
||||
"确认",
|
||||
"确定要清空所有日志吗?",
|
||||
QMessageBox.Yes | QMessageBox.No,
|
||||
QMessageBox.No
|
||||
)
|
||||
|
||||
if reply == QMessageBox.Yes:
|
||||
self._all_logs.clear()
|
||||
self.log_text.clear()
|
||||
self._update_line_count()
|
||||
self.log_cleared.emit()
|
||||
|
||||
def _update_line_count(self):
|
||||
"""更新行数显示"""
|
||||
visible_count = self.log_text.document().blockCount() - 1
|
||||
total_count = len(self._all_logs)
|
||||
|
||||
if visible_count < total_count:
|
||||
self.line_count_label.setText(f"{visible_count} / {total_count} 行")
|
||||
else:
|
||||
self.line_count_label.setText(f"{total_count} 行")
|
||||
166
etl_billiards/gui/widgets/settings_dialog.py
Normal file
166
etl_billiards/gui/widgets/settings_dialog.py
Normal file
@@ -0,0 +1,166 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""应用程序设置对话框"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QDialog, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||||
QGroupBox, QLabel, QLineEdit, QPushButton,
|
||||
QFileDialog, QMessageBox, QDialogButtonBox
|
||||
)
|
||||
from PySide6.QtCore import Qt
|
||||
|
||||
from ..utils.app_settings import app_settings
|
||||
|
||||
|
||||
class SettingsDialog(QDialog):
|
||||
"""设置对话框"""
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.setWindowTitle("应用程序设置")
|
||||
self.setMinimumWidth(600)
|
||||
self._init_ui()
|
||||
self._load_settings()
|
||||
|
||||
def _init_ui(self):
|
||||
layout = QVBoxLayout(self)
|
||||
|
||||
# ETL 项目路径
|
||||
project_group = QGroupBox("ETL 项目配置")
|
||||
project_layout = QGridLayout(project_group)
|
||||
|
||||
project_layout.addWidget(QLabel("ETL 项目路径:"), 0, 0)
|
||||
self.project_path_edit = QLineEdit()
|
||||
self.project_path_edit.setPlaceholderText("例: etl_billiards")
|
||||
project_layout.addWidget(self.project_path_edit, 0, 1)
|
||||
|
||||
browse_project_btn = QPushButton("浏览...")
|
||||
browse_project_btn.clicked.connect(self._browse_project_path)
|
||||
project_layout.addWidget(browse_project_btn, 0, 2)
|
||||
|
||||
project_layout.addWidget(QLabel(".env 文件路径:"), 1, 0)
|
||||
self.env_path_edit = QLineEdit()
|
||||
self.env_path_edit.setPlaceholderText("例: etl_billiards/.env")
|
||||
project_layout.addWidget(self.env_path_edit, 1, 1)
|
||||
|
||||
browse_env_btn = QPushButton("浏览...")
|
||||
browse_env_btn.clicked.connect(self._browse_env_path)
|
||||
project_layout.addWidget(browse_env_btn, 1, 2)
|
||||
|
||||
# 验证按钮
|
||||
validate_btn = QPushButton("验证配置")
|
||||
validate_btn.clicked.connect(self._validate_config)
|
||||
project_layout.addWidget(validate_btn, 2, 1)
|
||||
|
||||
# 验证结果
|
||||
self.validation_label = QLabel()
|
||||
self.validation_label.setWordWrap(True)
|
||||
project_layout.addWidget(self.validation_label, 3, 0, 1, 3)
|
||||
|
||||
layout.addWidget(project_group)
|
||||
|
||||
# 说明
|
||||
note = QLabel(
|
||||
"说明:\n"
|
||||
"• ETL 项目路径:包含 cli/main.py 的目录\n"
|
||||
"• .env 文件路径:环境变量配置文件\n"
|
||||
"• 配置后才能正常执行 ETL 任务"
|
||||
)
|
||||
note.setProperty("subheading", True)
|
||||
note.setWordWrap(True)
|
||||
layout.addWidget(note)
|
||||
|
||||
layout.addStretch()
|
||||
|
||||
# 按钮
|
||||
btn_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
|
||||
btn_box.accepted.connect(self._save_and_accept)
|
||||
btn_box.rejected.connect(self.reject)
|
||||
layout.addWidget(btn_box)
|
||||
|
||||
def _load_settings(self):
|
||||
"""加载设置"""
|
||||
self.project_path_edit.setText(app_settings.etl_project_path)
|
||||
self.env_path_edit.setText(app_settings.env_file_path)
|
||||
self._validate_config()
|
||||
|
||||
def _browse_project_path(self):
|
||||
"""浏览项目路径"""
|
||||
path = QFileDialog.getExistingDirectory(
|
||||
self, "选择 ETL 项目目录",
|
||||
self.project_path_edit.text() or str(Path.home())
|
||||
)
|
||||
if path:
|
||||
self.project_path_edit.setText(path)
|
||||
# 自动填充 .env 路径
|
||||
env_path = Path(path) / ".env"
|
||||
if env_path.exists():
|
||||
self.env_path_edit.setText(str(env_path))
|
||||
self._validate_config()
|
||||
|
||||
def _browse_env_path(self):
|
||||
"""浏览 .env 文件"""
|
||||
path, _ = QFileDialog.getOpenFileName(
|
||||
self, "选择 .env 文件",
|
||||
self.env_path_edit.text() or str(Path.home()),
|
||||
"环境变量文件 (*.env);;所有文件 (*.*)"
|
||||
)
|
||||
if path:
|
||||
self.env_path_edit.setText(path)
|
||||
self._validate_config()
|
||||
|
||||
def _validate_config(self):
|
||||
"""验证配置"""
|
||||
project_path = self.project_path_edit.text().strip()
|
||||
env_path = self.env_path_edit.text().strip()
|
||||
|
||||
issues = []
|
||||
|
||||
if not project_path:
|
||||
issues.append("• 未设置 ETL 项目路径")
|
||||
else:
|
||||
p = Path(project_path)
|
||||
if not p.exists():
|
||||
issues.append(f"• ETL 项目路径不存在")
|
||||
elif not (p / "cli" / "main.py").exists():
|
||||
issues.append(f"• 找不到 cli/main.py")
|
||||
|
||||
if not env_path:
|
||||
issues.append("• 未设置 .env 文件路径")
|
||||
elif not Path(env_path).exists():
|
||||
issues.append("• .env 文件不存在")
|
||||
|
||||
if issues:
|
||||
self.validation_label.setText("❌ 配置问题:\n" + "\n".join(issues))
|
||||
self.validation_label.setStyleSheet("color: #d93025;")
|
||||
else:
|
||||
self.validation_label.setText("✅ 配置有效")
|
||||
self.validation_label.setStyleSheet("color: #1e8e3e;")
|
||||
|
||||
def _save_and_accept(self):
|
||||
"""保存并关闭"""
|
||||
project_path = self.project_path_edit.text().strip()
|
||||
env_path = self.env_path_edit.text().strip()
|
||||
|
||||
# 简单验证
|
||||
if project_path:
|
||||
p = Path(project_path)
|
||||
if not p.exists():
|
||||
QMessageBox.warning(self, "警告", "ETL 项目路径不存在")
|
||||
return
|
||||
if not (p / "cli" / "main.py").exists():
|
||||
reply = QMessageBox.question(
|
||||
self, "确认",
|
||||
"找不到 cli/main.py,确定要使用此路径吗?",
|
||||
QMessageBox.Yes | QMessageBox.No
|
||||
)
|
||||
if reply == QMessageBox.No:
|
||||
return
|
||||
|
||||
# 保存设置
|
||||
app_settings.etl_project_path = project_path
|
||||
if env_path:
|
||||
app_settings.env_file_path = env_path
|
||||
|
||||
self.accept()
|
||||
406
etl_billiards/gui/widgets/status_panel.py
Normal file
406
etl_billiards/gui/widgets/status_panel.py
Normal file
@@ -0,0 +1,406 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""ETL 状态面板"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
from PySide6.QtWidgets import (
|
||||
QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||||
QGroupBox, QLabel, QPushButton, QTableWidget, QTableWidgetItem,
|
||||
QHeaderView, QFrame, QScrollArea, QMessageBox
|
||||
)
|
||||
from PySide6.QtCore import Qt, Signal, QTimer
|
||||
from PySide6.QtGui import QColor
|
||||
|
||||
from ..workers.db_worker import DBWorker
|
||||
from ..utils.config_helper import ConfigHelper
|
||||
|
||||
|
||||
class StatusCard(QFrame):
|
||||
"""状态卡片"""
|
||||
|
||||
def __init__(self, title: str, parent=None):
|
||||
super().__init__(parent)
|
||||
self.setProperty("card", True)
|
||||
self.setFrameShape(QFrame.StyledPanel)
|
||||
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(16, 12, 16, 12)
|
||||
layout.setSpacing(8)
|
||||
|
||||
# 标题
|
||||
self.title_label = QLabel(title)
|
||||
self.title_label.setProperty("subheading", True)
|
||||
layout.addWidget(self.title_label)
|
||||
|
||||
# 值
|
||||
self.value_label = QLabel("-")
|
||||
self.value_label.setStyleSheet("font-size: 24px; font-weight: bold;")
|
||||
layout.addWidget(self.value_label)
|
||||
|
||||
# 描述
|
||||
self.desc_label = QLabel("")
|
||||
self.desc_label.setProperty("subheading", True)
|
||||
layout.addWidget(self.desc_label)
|
||||
|
||||
def set_value(self, value: str, description: str = "", status: str = ""):
|
||||
"""设置值"""
|
||||
self.value_label.setText(value)
|
||||
self.desc_label.setText(description)
|
||||
|
||||
if status:
|
||||
self.value_label.setProperty("status", status)
|
||||
self.value_label.style().unpolish(self.value_label)
|
||||
self.value_label.style().polish(self.value_label)
|
||||
|
||||
|
||||
class StatusPanel(QWidget):
|
||||
"""ETL 状态面板"""
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.config_helper = ConfigHelper()
|
||||
self.db_worker = DBWorker(self)
|
||||
self._connected = False
|
||||
|
||||
self._init_ui()
|
||||
self._connect_signals()
|
||||
|
||||
# 定时刷新
|
||||
self.refresh_timer = QTimer(self)
|
||||
self.refresh_timer.timeout.connect(self._auto_refresh)
|
||||
|
||||
def _init_ui(self):
|
||||
"""初始化界面"""
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(16, 16, 16, 16)
|
||||
layout.setSpacing(16)
|
||||
|
||||
# 标题和按钮
|
||||
header_layout = QHBoxLayout()
|
||||
|
||||
title = QLabel("ETL 状态")
|
||||
title.setProperty("heading", True)
|
||||
header_layout.addWidget(title)
|
||||
|
||||
header_layout.addStretch()
|
||||
|
||||
self.auto_refresh_btn = QPushButton("自动刷新: 关")
|
||||
self.auto_refresh_btn.setProperty("secondary", True)
|
||||
self.auto_refresh_btn.setCheckable(True)
|
||||
header_layout.addWidget(self.auto_refresh_btn)
|
||||
|
||||
self.refresh_btn = QPushButton("刷新")
|
||||
self.refresh_btn.clicked.connect(self._refresh_all)
|
||||
header_layout.addWidget(self.refresh_btn)
|
||||
|
||||
layout.addLayout(header_layout)
|
||||
|
||||
# 连接状态
|
||||
self.conn_status_label = QLabel("数据库: 未连接")
|
||||
self.conn_status_label.setProperty("status", "warning")
|
||||
layout.addWidget(self.conn_status_label)
|
||||
|
||||
# 滚动区域
|
||||
scroll_area = QScrollArea()
|
||||
scroll_area.setWidgetResizable(True)
|
||||
scroll_area.setFrameShape(QFrame.NoFrame)
|
||||
layout.addWidget(scroll_area, 1)
|
||||
|
||||
# 内容容器
|
||||
content_widget = QWidget()
|
||||
content_layout = QVBoxLayout(content_widget)
|
||||
content_layout.setSpacing(16)
|
||||
|
||||
# 概览卡片
|
||||
cards_layout = QHBoxLayout()
|
||||
|
||||
self.ods_card = StatusCard("ODS 表数量")
|
||||
cards_layout.addWidget(self.ods_card)
|
||||
|
||||
self.dwd_card = StatusCard("DWD 表数量")
|
||||
cards_layout.addWidget(self.dwd_card)
|
||||
|
||||
self.last_update_card = StatusCard("最后更新")
|
||||
cards_layout.addWidget(self.last_update_card)
|
||||
|
||||
self.task_count_card = StatusCard("今日任务")
|
||||
cards_layout.addWidget(self.task_count_card)
|
||||
|
||||
content_layout.addLayout(cards_layout)
|
||||
|
||||
# ODS Cutoff 状态
|
||||
cutoff_group = QGroupBox("ODS Cutoff 状态")
|
||||
cutoff_layout = QVBoxLayout(cutoff_group)
|
||||
|
||||
self.cutoff_table = QTableWidget()
|
||||
self.cutoff_table.setColumnCount(4)
|
||||
self.cutoff_table.setHorizontalHeaderLabels(["表名", "最新 fetched_at", "行数", "状态"])
|
||||
self.cutoff_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.Stretch)
|
||||
self.cutoff_table.setMaximumHeight(250)
|
||||
cutoff_layout.addWidget(self.cutoff_table)
|
||||
|
||||
content_layout.addWidget(cutoff_group)
|
||||
|
||||
# 最近运行记录
|
||||
history_group = QGroupBox("最近运行记录")
|
||||
history_layout = QVBoxLayout(history_group)
|
||||
|
||||
self.history_table = QTableWidget()
|
||||
self.history_table.setColumnCount(6)
|
||||
self.history_table.setHorizontalHeaderLabels(["运行ID", "任务", "状态", "开始时间", "耗时", "影响行数"])
|
||||
self.history_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.Stretch)
|
||||
self.history_table.setMaximumHeight(250)
|
||||
history_layout.addWidget(self.history_table)
|
||||
|
||||
content_layout.addWidget(history_group)
|
||||
|
||||
# 弹性空间
|
||||
content_layout.addStretch()
|
||||
|
||||
scroll_area.setWidget(content_widget)
|
||||
|
||||
def _connect_signals(self):
|
||||
"""连接信号"""
|
||||
self.auto_refresh_btn.toggled.connect(self._toggle_auto_refresh)
|
||||
self.db_worker.connection_status.connect(self._on_connection_status)
|
||||
self.db_worker.query_finished.connect(self._on_query_finished)
|
||||
self.db_worker.query_error.connect(self._on_query_error)
|
||||
|
||||
def _toggle_auto_refresh(self, checked: bool):
|
||||
"""切换自动刷新"""
|
||||
if checked:
|
||||
self.auto_refresh_btn.setText("自动刷新: 开")
|
||||
self.refresh_timer.start(30000) # 30秒刷新一次
|
||||
self._refresh_all()
|
||||
else:
|
||||
self.auto_refresh_btn.setText("自动刷新: 关")
|
||||
self.refresh_timer.stop()
|
||||
|
||||
def _auto_refresh(self):
|
||||
"""自动刷新"""
|
||||
if self._connected:
|
||||
self._refresh_all()
|
||||
|
||||
def _refresh_all(self):
|
||||
"""刷新所有数据"""
|
||||
# 尝试连接数据库
|
||||
if not self._connected:
|
||||
env_vars = self.config_helper.load_env()
|
||||
dsn = env_vars.get("PG_DSN", "")
|
||||
if dsn:
|
||||
self.db_worker.connect_db(dsn)
|
||||
else:
|
||||
self.conn_status_label.setText("数据库: 未配置 DSN")
|
||||
return
|
||||
else:
|
||||
self._load_status_data()
|
||||
|
||||
def _on_connection_status(self, connected: bool, message: str):
|
||||
"""处理连接状态"""
|
||||
self._connected = connected
|
||||
|
||||
if connected:
|
||||
self.conn_status_label.setText(f"数据库: 已连接")
|
||||
self.conn_status_label.setProperty("status", "success")
|
||||
self._load_status_data()
|
||||
else:
|
||||
self.conn_status_label.setText(f"数据库: {message}")
|
||||
self.conn_status_label.setProperty("status", "error")
|
||||
|
||||
self.conn_status_label.style().unpolish(self.conn_status_label)
|
||||
self.conn_status_label.style().polish(self.conn_status_label)
|
||||
|
||||
def _load_status_data(self):
|
||||
"""加载状态数据"""
|
||||
# 加载表统计
|
||||
self._current_query = "table_count"
|
||||
self.db_worker.execute_query("""
|
||||
SELECT
|
||||
table_schema,
|
||||
COUNT(*) as table_count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema IN ('billiards_ods', 'billiards_dwd', 'billiards_dws')
|
||||
GROUP BY table_schema
|
||||
""")
|
||||
|
||||
def _on_query_finished(self, columns: list, rows: list):
|
||||
"""处理查询结果"""
|
||||
query_type = getattr(self, '_current_query', '')
|
||||
|
||||
if query_type == "table_count":
|
||||
self._process_table_count(rows)
|
||||
# 继续加载 cutoff 数据
|
||||
self._current_query = "cutoff"
|
||||
self.db_worker.execute_query("""
|
||||
SELECT
|
||||
'payment_transactions' AS table_name,
|
||||
MAX(fetched_at) AS max_fetched_at,
|
||||
COUNT(*) AS row_count
|
||||
FROM billiards_ods.payment_transactions
|
||||
UNION ALL
|
||||
SELECT 'member_profiles', MAX(fetched_at), COUNT(*)
|
||||
FROM billiards_ods.member_profiles
|
||||
UNION ALL
|
||||
SELECT 'settlement_records', MAX(fetched_at), COUNT(*)
|
||||
FROM billiards_ods.settlement_records
|
||||
UNION ALL
|
||||
SELECT 'recharge_settlements', MAX(fetched_at), COUNT(*)
|
||||
FROM billiards_ods.recharge_settlements
|
||||
UNION ALL
|
||||
SELECT 'assistant_service_records', MAX(fetched_at), COUNT(*)
|
||||
FROM billiards_ods.assistant_service_records
|
||||
ORDER BY table_name
|
||||
""")
|
||||
elif query_type == "cutoff":
|
||||
self._process_cutoff_data(rows)
|
||||
# 继续加载运行历史
|
||||
self._current_query = "history"
|
||||
self.db_worker.execute_query("""
|
||||
SELECT
|
||||
run_id,
|
||||
task_code,
|
||||
status,
|
||||
started_at,
|
||||
finished_at,
|
||||
rows_affected
|
||||
FROM etl_admin.run_tracker
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 20
|
||||
""")
|
||||
elif query_type == "history":
|
||||
self._process_history_data(rows)
|
||||
self._current_query = ""
|
||||
|
||||
def _process_table_count(self, rows: list):
|
||||
"""处理表数量数据"""
|
||||
ods_count = 0
|
||||
dwd_count = 0
|
||||
|
||||
for row in rows:
|
||||
schema = row.get("table_schema", "")
|
||||
count = row.get("table_count", 0)
|
||||
|
||||
if schema == "billiards_ods":
|
||||
ods_count = count
|
||||
elif schema == "billiards_dwd":
|
||||
dwd_count = count
|
||||
|
||||
self.ods_card.set_value(str(ods_count), "个表")
|
||||
self.dwd_card.set_value(str(dwd_count), "个表")
|
||||
|
||||
def _process_cutoff_data(self, rows: list):
|
||||
"""处理 Cutoff 数据"""
|
||||
self.cutoff_table.setRowCount(len(rows))
|
||||
|
||||
latest_time = None
|
||||
now = datetime.now()
|
||||
|
||||
for row_idx, row in enumerate(rows):
|
||||
table_name = row.get("table_name", "")
|
||||
max_fetched = row.get("max_fetched_at")
|
||||
row_count = row.get("row_count", 0)
|
||||
|
||||
self.cutoff_table.setItem(row_idx, 0, QTableWidgetItem(table_name))
|
||||
|
||||
if max_fetched:
|
||||
time_str = str(max_fetched)[:19]
|
||||
self.cutoff_table.setItem(row_idx, 1, QTableWidgetItem(time_str))
|
||||
|
||||
# 更新最新时间
|
||||
if latest_time is None or max_fetched > latest_time:
|
||||
latest_time = max_fetched
|
||||
|
||||
# 计算状态
|
||||
if isinstance(max_fetched, datetime):
|
||||
hours_ago = (now - max_fetched).total_seconds() / 3600
|
||||
if hours_ago < 1:
|
||||
status = "正常"
|
||||
status_color = QColor("#1e8e3e")
|
||||
elif hours_ago < 24:
|
||||
status = "较新"
|
||||
status_color = QColor("#1a73e8")
|
||||
else:
|
||||
status = f"落后 {int(hours_ago)}h"
|
||||
status_color = QColor("#f9ab00")
|
||||
else:
|
||||
status = "-"
|
||||
status_color = QColor("#9aa0a6")
|
||||
else:
|
||||
self.cutoff_table.setItem(row_idx, 1, QTableWidgetItem("-"))
|
||||
status = "无数据"
|
||||
status_color = QColor("#d93025")
|
||||
|
||||
self.cutoff_table.setItem(row_idx, 2, QTableWidgetItem(str(row_count)))
|
||||
|
||||
status_item = QTableWidgetItem(status)
|
||||
status_item.setForeground(status_color)
|
||||
self.cutoff_table.setItem(row_idx, 3, status_item)
|
||||
|
||||
# 更新最后更新时间卡片
|
||||
if latest_time:
|
||||
time_str = str(latest_time)[:16]
|
||||
self.last_update_card.set_value(time_str, "")
|
||||
else:
|
||||
self.last_update_card.set_value("-", "无数据")
|
||||
|
||||
def _process_history_data(self, rows: list):
|
||||
"""处理运行历史数据"""
|
||||
self.history_table.setRowCount(len(rows))
|
||||
|
||||
today_count = 0
|
||||
today = datetime.now().date()
|
||||
|
||||
for row_idx, row in enumerate(rows):
|
||||
run_id = row.get("run_id", "")
|
||||
task_code = row.get("task_code", "")
|
||||
status = row.get("status", "")
|
||||
started_at = row.get("started_at")
|
||||
finished_at = row.get("finished_at")
|
||||
rows_affected = row.get("rows_affected", 0)
|
||||
|
||||
# 统计今日任务
|
||||
if started_at and isinstance(started_at, datetime):
|
||||
if started_at.date() == today:
|
||||
today_count += 1
|
||||
|
||||
self.history_table.setItem(row_idx, 0, QTableWidgetItem(str(run_id)[:8] if run_id else "-"))
|
||||
self.history_table.setItem(row_idx, 1, QTableWidgetItem(task_code))
|
||||
|
||||
# 状态
|
||||
status_item = QTableWidgetItem(status)
|
||||
if status and "success" in status.lower():
|
||||
status_item.setForeground(QColor("#1e8e3e"))
|
||||
elif status and ("fail" in status.lower() or "error" in status.lower()):
|
||||
status_item.setForeground(QColor("#d93025"))
|
||||
self.history_table.setItem(row_idx, 2, status_item)
|
||||
|
||||
# 开始时间
|
||||
time_str = str(started_at)[:19] if started_at else "-"
|
||||
self.history_table.setItem(row_idx, 3, QTableWidgetItem(time_str))
|
||||
|
||||
# 耗时
|
||||
if started_at and finished_at:
|
||||
try:
|
||||
duration = (finished_at - started_at).total_seconds()
|
||||
if duration < 60:
|
||||
duration_str = f"{duration:.1f}秒"
|
||||
else:
|
||||
duration_str = f"{int(duration // 60)}分{int(duration % 60)}秒"
|
||||
except:
|
||||
duration_str = "-"
|
||||
else:
|
||||
duration_str = "-"
|
||||
self.history_table.setItem(row_idx, 4, QTableWidgetItem(duration_str))
|
||||
|
||||
# 影响行数
|
||||
self.history_table.setItem(row_idx, 5, QTableWidgetItem(str(rows_affected or 0)))
|
||||
|
||||
# 更新今日任务卡片
|
||||
self.task_count_card.set_value(str(today_count), "次执行")
|
||||
|
||||
def _on_query_error(self, error: str):
|
||||
"""处理查询错误"""
|
||||
self._current_query = ""
|
||||
# 可能是表不存在,忽略错误继续
|
||||
pass
|
||||
1948
etl_billiards/gui/widgets/task_manager.py
Normal file
1948
etl_billiards/gui/widgets/task_manager.py
Normal file
File diff suppressed because it is too large
Load Diff
1061
etl_billiards/gui/widgets/task_panel.py
Normal file
1061
etl_billiards/gui/widgets/task_panel.py
Normal file
File diff suppressed because it is too large
Load Diff
7
etl_billiards/gui/workers/__init__.py
Normal file
7
etl_billiards/gui/workers/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""后台工作线程模块"""
|
||||
|
||||
from .task_worker import TaskWorker
|
||||
from .db_worker import DBWorker
|
||||
|
||||
__all__ = ["TaskWorker", "DBWorker"]
|
||||
192
etl_billiards/gui/workers/db_worker.py
Normal file
192
etl_billiards/gui/workers/db_worker.py
Normal file
@@ -0,0 +1,192 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""数据库查询工作线程"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
|
||||
from PySide6.QtCore import QThread, Signal
|
||||
|
||||
# 添加项目路径
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
|
||||
class DBWorker(QThread):
|
||||
"""数据库查询工作线程"""
|
||||
|
||||
# 信号
|
||||
query_finished = Signal(list, list) # 查询完成 (columns, rows)
|
||||
query_error = Signal(str) # 查询错误
|
||||
connection_status = Signal(bool, str) # 连接状态 (connected, message)
|
||||
tables_loaded = Signal(dict) # 表列表加载完成 {schema: [(table, rows, updated_at), ...]}
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.conn = None
|
||||
self._task = None
|
||||
self._task_args = None
|
||||
|
||||
def connect_db(self, dsn: str):
|
||||
"""连接数据库"""
|
||||
self._task = "connect"
|
||||
self._task_args = (dsn,)
|
||||
self.start()
|
||||
|
||||
def disconnect_db(self):
|
||||
"""断开数据库连接"""
|
||||
self._task = "disconnect"
|
||||
self._task_args = None
|
||||
self.start()
|
||||
|
||||
def execute_query(self, sql: str, params: Optional[tuple] = None):
|
||||
"""执行查询"""
|
||||
self._task = "query"
|
||||
self._task_args = (sql, params)
|
||||
self.start()
|
||||
|
||||
def load_tables(self, schemas: Optional[List[str]] = None):
|
||||
"""加载表列表"""
|
||||
self._task = "load_tables"
|
||||
self._task_args = (schemas,)
|
||||
self.start()
|
||||
|
||||
def run(self):
|
||||
"""执行任务"""
|
||||
if self._task == "connect":
|
||||
self._do_connect(*self._task_args)
|
||||
elif self._task == "disconnect":
|
||||
self._do_disconnect()
|
||||
elif self._task == "query":
|
||||
self._do_query(*self._task_args)
|
||||
elif self._task == "load_tables":
|
||||
self._do_load_tables(*self._task_args)
|
||||
|
||||
def _do_connect(self, dsn: str):
|
||||
"""执行连接"""
|
||||
try:
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
self.conn = psycopg2.connect(dsn, connect_timeout=10)
|
||||
self.conn.set_session(autocommit=True)
|
||||
|
||||
# 测试连接
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("SELECT version()")
|
||||
version = cur.fetchone()[0]
|
||||
|
||||
self.connection_status.emit(True, f"已连接: {version[:50]}...")
|
||||
except ImportError:
|
||||
self.connection_status.emit(False, "缺少 psycopg2 模块,请安装: pip install psycopg2-binary")
|
||||
except Exception as e:
|
||||
self.conn = None
|
||||
self.connection_status.emit(False, f"连接失败: {e}")
|
||||
|
||||
def _do_disconnect(self):
|
||||
"""执行断开连接"""
|
||||
if self.conn:
|
||||
try:
|
||||
self.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.conn = None
|
||||
self.connection_status.emit(False, "已断开连接")
|
||||
|
||||
def _do_query(self, sql: str, params: Optional[tuple]):
|
||||
"""执行查询"""
|
||||
if not self.conn:
|
||||
self.query_error.emit("未连接到数据库")
|
||||
return
|
||||
|
||||
try:
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
with self.conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute(sql, params)
|
||||
|
||||
# 检查是否有结果
|
||||
if cur.description:
|
||||
columns = [desc[0] for desc in cur.description]
|
||||
rows = [dict(row) for row in cur.fetchall()]
|
||||
self.query_finished.emit(columns, rows)
|
||||
else:
|
||||
self.query_finished.emit([], [])
|
||||
except Exception as e:
|
||||
self.query_error.emit(f"查询失败: {e}")
|
||||
|
||||
def _do_load_tables(self, schemas: Optional[List[str]]):
|
||||
"""加载表列表"""
|
||||
if not self.conn:
|
||||
self.query_error.emit("未连接到数据库")
|
||||
return
|
||||
|
||||
try:
|
||||
if schemas is None:
|
||||
schemas = ["billiards_ods", "billiards_dwd", "billiards_dws", "etl_admin"]
|
||||
|
||||
result = {}
|
||||
|
||||
for schema in schemas:
|
||||
tables = []
|
||||
|
||||
# 获取表列表
|
||||
sql = """
|
||||
SELECT
|
||||
t.table_name,
|
||||
COALESCE(s.n_live_tup, 0) as row_count
|
||||
FROM information_schema.tables t
|
||||
LEFT JOIN pg_stat_user_tables s
|
||||
ON t.table_name = s.relname
|
||||
AND t.table_schema = s.schemaname
|
||||
WHERE t.table_schema = %s
|
||||
AND t.table_type = 'BASE TABLE'
|
||||
ORDER BY t.table_name
|
||||
"""
|
||||
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute(sql, (schema,))
|
||||
for row in cur.fetchall():
|
||||
table_name = row[0]
|
||||
row_count = row[1] or 0
|
||||
|
||||
# 尝试获取最新更新时间
|
||||
updated_at = None
|
||||
try:
|
||||
# 尝试 fetched_at 字段
|
||||
cur.execute(f'SELECT MAX(fetched_at) FROM "{schema}"."{table_name}"')
|
||||
result_row = cur.fetchone()
|
||||
if result_row and result_row[0]:
|
||||
updated_at = str(result_row[0])[:19]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not updated_at:
|
||||
try:
|
||||
# 尝试 updated_at 字段
|
||||
cur.execute(f'SELECT MAX(updated_at) FROM "{schema}"."{table_name}"')
|
||||
result_row = cur.fetchone()
|
||||
if result_row and result_row[0]:
|
||||
updated_at = str(result_row[0])[:19]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
tables.append((table_name, row_count, updated_at or "-"))
|
||||
|
||||
result[schema] = tables
|
||||
|
||||
self.tables_loaded.emit(result)
|
||||
except Exception as e:
|
||||
self.query_error.emit(f"加载表列表失败: {e}")
|
||||
|
||||
def is_connected(self) -> bool:
|
||||
"""检查是否已连接"""
|
||||
if not self.conn:
|
||||
return False
|
||||
try:
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("SELECT 1")
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
317
etl_billiards/gui/workers/task_worker.py
Normal file
317
etl_billiards/gui/workers/task_worker.py
Normal file
@@ -0,0 +1,317 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""任务执行工作线程"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Dict
|
||||
|
||||
from PySide6.QtCore import QThread, Signal
|
||||
|
||||
from ..utils.app_settings import app_settings
|
||||
|
||||
|
||||
class TaskWorker(QThread):
|
||||
"""任务执行工作线程"""
|
||||
|
||||
# 信号
|
||||
output_received = Signal(str) # 收到输出行
|
||||
task_finished = Signal(int, str) # 任务完成 (exit_code, summary)
|
||||
error_occurred = Signal(str) # 发生错误
|
||||
progress_updated = Signal(int, int) # 进度更新 (current, total)
|
||||
|
||||
def __init__(self, command: List[str], working_dir: Optional[str] = None,
|
||||
extra_env: Optional[Dict[str, str]] = None, parent=None):
|
||||
super().__init__(parent)
|
||||
self.command = command
|
||||
self.extra_env = extra_env or {}
|
||||
|
||||
# 工作目录优先级: 参数 > 应用设置 > 自动检测
|
||||
if working_dir is not None:
|
||||
self.working_dir = working_dir
|
||||
elif app_settings.etl_project_path:
|
||||
self.working_dir = app_settings.etl_project_path
|
||||
else:
|
||||
# 回退到源码目录
|
||||
self.working_dir = str(Path(__file__).resolve().parents[2])
|
||||
|
||||
self.process: Optional[subprocess.Popen] = None
|
||||
self._stop_requested = False
|
||||
self._exit_code: Optional[int] = None
|
||||
self._output_lines: List[str] = []
|
||||
|
||||
def run(self):
|
||||
"""执行任务"""
|
||||
try:
|
||||
self._stop_requested = False
|
||||
self._output_lines = []
|
||||
|
||||
# 设置环境变量
|
||||
env = os.environ.copy()
|
||||
env["PYTHONIOENCODING"] = "utf-8"
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
|
||||
# 添加项目根目录到 PYTHONPATH
|
||||
project_root = self.working_dir
|
||||
existing_path = env.get("PYTHONPATH", "")
|
||||
if existing_path:
|
||||
env["PYTHONPATH"] = f"{project_root}{os.pathsep}{existing_path}"
|
||||
else:
|
||||
env["PYTHONPATH"] = project_root
|
||||
|
||||
# 添加额外的环境变量
|
||||
if self.extra_env:
|
||||
for key, value in self.extra_env.items():
|
||||
env[key] = str(value)
|
||||
self.output_received.emit(f"[环境变量] {key}={value}")
|
||||
|
||||
self.output_received.emit(f"[工作目录] {self.working_dir}")
|
||||
self.output_received.emit(f"[执行命令] {' '.join(self.command)}")
|
||||
|
||||
# 启动进程
|
||||
self.process = subprocess.Popen(
|
||||
self.command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
cwd=self.working_dir,
|
||||
env=env,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == "win32" else 0,
|
||||
)
|
||||
|
||||
# 读取输出
|
||||
if self.process.stdout:
|
||||
for line in iter(self.process.stdout.readline, ""):
|
||||
if self._stop_requested:
|
||||
break
|
||||
|
||||
line = line.rstrip("\n\r")
|
||||
if line:
|
||||
self._output_lines.append(line)
|
||||
self.output_received.emit(line)
|
||||
|
||||
# 解析进度信息(如果有)
|
||||
self._parse_progress(line)
|
||||
|
||||
# 等待进程结束
|
||||
if self.process:
|
||||
self.process.wait()
|
||||
self._exit_code = self.process.returncode
|
||||
|
||||
# 生成摘要
|
||||
summary = self._generate_summary()
|
||||
self.task_finished.emit(self._exit_code or 0, summary)
|
||||
|
||||
except FileNotFoundError as e:
|
||||
self.error_occurred.emit(f"找不到 Python 解释器: {e}")
|
||||
self.task_finished.emit(-1, f"执行失败: {e}")
|
||||
except Exception as e:
|
||||
self.error_occurred.emit(f"执行出错: {e}")
|
||||
self.task_finished.emit(-1, f"执行失败: {e}")
|
||||
finally:
|
||||
self.process = None
|
||||
|
||||
def stop(self):
|
||||
"""停止任务"""
|
||||
self._stop_requested = True
|
||||
if self.process:
|
||||
try:
|
||||
self.process.terminate()
|
||||
# 给进程一些时间来终止
|
||||
try:
|
||||
self.process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.process.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _parse_progress(self, line: str):
|
||||
"""解析进度信息"""
|
||||
# 尝试从日志中解析进度
|
||||
# 示例: "[INFO] 处理进度: 50/100"
|
||||
import re
|
||||
match = re.search(r'进度[:\s]*(\d+)/(\d+)', line)
|
||||
if match:
|
||||
current = int(match.group(1))
|
||||
total = int(match.group(2))
|
||||
self.progress_updated.emit(current, total)
|
||||
|
||||
def _generate_summary(self) -> str:
|
||||
"""生成执行摘要"""
|
||||
if not self._output_lines:
|
||||
return "无输出"
|
||||
|
||||
return self._parse_detailed_summary()
|
||||
|
||||
def _parse_detailed_summary(self) -> str:
|
||||
"""解析详细的执行摘要"""
|
||||
import re
|
||||
import json
|
||||
|
||||
summary_parts = []
|
||||
|
||||
# 统计各类信息
|
||||
ods_stats = [] # ODS 抓取统计
|
||||
dwd_stats = [] # DWD 装载统计
|
||||
integrity_stats = {} # 数据校验统计
|
||||
errors = [] # 错误信息
|
||||
task_results = [] # 任务结果
|
||||
|
||||
for line in self._output_lines:
|
||||
# 1. 解析 ODS 抓取完成信息
|
||||
# 格式: "xxx: 抓取完成,文件=xxx,记录数=123"
|
||||
match = re.search(r'(\w+): 抓取完成.*记录数[=:]\s*(\d+)', line)
|
||||
if match:
|
||||
task_name = match.group(1)
|
||||
record_count = int(match.group(2))
|
||||
if record_count > 0:
|
||||
ods_stats.append(f"{task_name}: {record_count}条")
|
||||
continue
|
||||
|
||||
# 2. 解析 DWD 装载完成信息
|
||||
# 格式: "DWD 装载完成:xxx,用时 1.02s"
|
||||
match = re.search(r'DWD 装载完成[::]\s*(\S+).*用时\s*([\d.]+)s', line)
|
||||
if match:
|
||||
table_name = match.group(1).replace('billiards_dwd.', '')
|
||||
continue
|
||||
|
||||
# 3. 解析任务完成统计 (JSON格式)
|
||||
# 格式: "xxx: 完成,统计={'tables': [...]}"
|
||||
if "完成,统计=" in line or "完成,统计=" in line:
|
||||
try:
|
||||
match = re.search(r"统计=(\{.+\})", line)
|
||||
if match:
|
||||
stats_str = match.group(1).replace("'", '"')
|
||||
stats = json.loads(stats_str)
|
||||
|
||||
# 解析 DWD 装载统计
|
||||
if 'tables' in stats:
|
||||
total_processed = 0
|
||||
total_inserted = 0
|
||||
tables_with_data = []
|
||||
|
||||
for tbl in stats['tables']:
|
||||
table_name = tbl.get('table', '').replace('billiards_dwd.', '')
|
||||
processed = tbl.get('processed', 0)
|
||||
inserted = tbl.get('inserted', 0)
|
||||
|
||||
if processed > 0:
|
||||
total_processed += processed
|
||||
tables_with_data.append(f"{table_name}({processed})")
|
||||
elif inserted > 0:
|
||||
total_inserted += inserted
|
||||
tables_with_data.append(f"{table_name}(+{inserted})")
|
||||
|
||||
if total_processed > 0 or total_inserted > 0:
|
||||
dwd_stats.append(f"处理维度: {total_processed}条, 新增事实: {total_inserted}条")
|
||||
if len(tables_with_data) <= 5:
|
||||
dwd_stats.append(f"涉及表: {', '.join(tables_with_data)}")
|
||||
else:
|
||||
dwd_stats.append(f"涉及 {len(tables_with_data)} 张表")
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
|
||||
# 4. 解析数据校验结果
|
||||
# 格式: "CHECK_DONE task=xxx missing=1 records=136 errors=0"
|
||||
match = re.search(r'CHECK_DONE task=(\w+) missing=(\d+) records=(\d+)', line)
|
||||
if match:
|
||||
task_name = match.group(1)
|
||||
missing = int(match.group(2))
|
||||
records = int(match.group(3))
|
||||
if missing > 0:
|
||||
if 'missing_tasks' not in integrity_stats:
|
||||
integrity_stats['missing_tasks'] = []
|
||||
integrity_stats['missing_tasks'].append(f"{task_name}: 缺失{missing}/{records}")
|
||||
integrity_stats['total_records'] = integrity_stats.get('total_records', 0) + records
|
||||
integrity_stats['total_missing'] = integrity_stats.get('total_missing', 0) + missing
|
||||
continue
|
||||
|
||||
# 5. 解析数据校验最终结果
|
||||
# 格式: "结果统计: {'missing': 463, 'errors': 0, 'backfilled': 0}"
|
||||
if "结果统计:" in line or "结果统计:" in line:
|
||||
try:
|
||||
match = re.search(r"\{.+\}", line)
|
||||
if match:
|
||||
stats_str = match.group(0).replace("'", '"')
|
||||
stats = json.loads(stats_str)
|
||||
integrity_stats['final_missing'] = stats.get('missing', 0)
|
||||
integrity_stats['final_errors'] = stats.get('errors', 0)
|
||||
integrity_stats['backfilled'] = stats.get('backfilled', 0)
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
|
||||
# 6. 解析错误信息
|
||||
if "[ERROR]" in line or "错误" in line.lower() or "error" in line.lower():
|
||||
if "Traceback" not in line and "File " not in line:
|
||||
errors.append(line.strip()[:100])
|
||||
|
||||
# 7. 解析任务完成信息
|
||||
if "任务执行成功" in line or "ETL运行完成" in line:
|
||||
task_results.append("✓ " + line.split("]")[-1].strip() if "]" in line else line.strip())
|
||||
elif "任务执行失败" in line:
|
||||
task_results.append("✗ " + line.split("]")[-1].strip() if "]" in line else line.strip())
|
||||
|
||||
# 构建摘要
|
||||
if ods_stats:
|
||||
summary_parts.append("【ODS 抓取】" + ", ".join(ods_stats[:5]))
|
||||
if len(ods_stats) > 5:
|
||||
summary_parts[-1] += f" 等{len(ods_stats)}项"
|
||||
|
||||
if dwd_stats:
|
||||
summary_parts.append("【DWD 装载】" + "; ".join(dwd_stats))
|
||||
|
||||
if integrity_stats:
|
||||
total_missing = integrity_stats.get('final_missing', integrity_stats.get('total_missing', 0))
|
||||
total_records = integrity_stats.get('total_records', 0)
|
||||
backfilled = integrity_stats.get('backfilled', 0)
|
||||
|
||||
int_summary = f"【数据校验】检查 {total_records} 条记录"
|
||||
if total_missing > 0:
|
||||
int_summary += f", 发现 {total_missing} 条缺失"
|
||||
if backfilled > 0:
|
||||
int_summary += f", 已补全 {backfilled} 条"
|
||||
else:
|
||||
int_summary += ", 数据完整"
|
||||
summary_parts.append(int_summary)
|
||||
|
||||
# 显示缺失详情
|
||||
if integrity_stats.get('missing_tasks'):
|
||||
missing_detail = integrity_stats['missing_tasks'][:3]
|
||||
summary_parts.append(" 缺失: " + "; ".join(missing_detail))
|
||||
if len(integrity_stats['missing_tasks']) > 3:
|
||||
summary_parts[-1] += f" 等{len(integrity_stats['missing_tasks'])}项"
|
||||
|
||||
if errors:
|
||||
summary_parts.append("【错误】" + "; ".join(errors[:3]))
|
||||
|
||||
if task_results:
|
||||
summary_parts.append("【结果】" + " | ".join(task_results))
|
||||
|
||||
if summary_parts:
|
||||
return "\n".join(summary_parts)
|
||||
|
||||
# 如果没有解析到任何信息,返回最后几行关键信息
|
||||
key_lines = []
|
||||
for line in self._output_lines[-10:]:
|
||||
if "完成" in line or "成功" in line or "失败" in line:
|
||||
key_lines.append(line.strip()[:80])
|
||||
|
||||
if key_lines:
|
||||
return "\n".join(key_lines[-3:])
|
||||
|
||||
return self._output_lines[-1] if self._output_lines else "执行完成"
|
||||
|
||||
@property
|
||||
def exit_code(self) -> Optional[int]:
|
||||
"""获取退出码"""
|
||||
return self._exit_code
|
||||
|
||||
@property
|
||||
def output(self) -> str:
|
||||
"""获取完整输出"""
|
||||
return "\n".join(self._output_lines)
|
||||
Reference in New Issue
Block a user