This commit is contained in:
Neo
2026-01-27 22:47:05 +08:00
parent a6ad343092
commit f5f9a7eb66
476 changed files with 381543 additions and 5819 deletions

View File

@@ -0,0 +1,18 @@
# -*- coding: utf-8 -*-
"""GUI 组件模块"""
from .task_panel import TaskPanel
from .env_editor import EnvEditor
from .log_viewer import LogViewer
from .db_viewer import DBViewer
from .status_panel import StatusPanel
from .task_manager import TaskManager
__all__ = [
"TaskPanel",
"EnvEditor",
"LogViewer",
"DBViewer",
"StatusPanel",
"TaskManager",
]

View File

@@ -0,0 +1,390 @@
# -*- coding: utf-8 -*-
"""数据库查看器"""
from PySide6.QtWidgets import (
QWidget, QVBoxLayout, QHBoxLayout, QSplitter,
QGroupBox, QLabel, QPushButton, QLineEdit, QPlainTextEdit,
QTableWidget, QTableWidgetItem, QTreeWidget, QTreeWidgetItem,
QHeaderView, QComboBox, QTabWidget, QMessageBox, QFrame
)
from PySide6.QtCore import Qt, Signal
from PySide6.QtGui import QFont
from ..workers.db_worker import DBWorker
from ..utils.config_helper import ConfigHelper
# 常用查询模板
QUERY_TEMPLATES = {
"ODS 行数统计": """
SELECT
table_name,
(xpath('/row/cnt/text()',
query_to_xml('SELECT COUNT(*) AS cnt FROM ' || table_schema || '.' || table_name, false, false, ''))
)[1]::text::bigint AS row_count
FROM information_schema.tables
WHERE table_schema = 'billiards_ods'
ORDER BY table_name;
""",
"DWD 行数统计": """
SELECT
table_name,
(xpath('/row/cnt/text()',
query_to_xml('SELECT COUNT(*) AS cnt FROM ' || table_schema || '.' || table_name, false, false, ''))
)[1]::text::bigint AS row_count
FROM information_schema.tables
WHERE table_schema = 'billiards_dwd'
ORDER BY table_name;
""",
"ETL 游标状态": """
SELECT
task_code,
last_start,
last_end,
last_run_id,
updated_at
FROM etl_admin.etl_cursor
ORDER BY task_code;
""",
"最近运行记录": """
SELECT
run_id,
task_code,
status,
started_at,
finished_at,
EXTRACT(EPOCH FROM (finished_at - started_at))::int AS duration_sec,
rows_affected
FROM etl_admin.run_tracker
ORDER BY started_at DESC
LIMIT 50;
""",
"ODS 最新入库时间": """
SELECT
'payment_transactions' AS table_name, MAX(fetched_at) AS max_fetched_at FROM billiards_ods.payment_transactions
UNION ALL
SELECT 'member_profiles', MAX(fetched_at) FROM billiards_ods.member_profiles
UNION ALL
SELECT 'settlement_records', MAX(fetched_at) FROM billiards_ods.settlement_records
UNION ALL
SELECT 'recharge_settlements', MAX(fetched_at) FROM billiards_ods.recharge_settlements
ORDER BY table_name;
""",
}
class DBViewer(QWidget):
"""数据库查看器"""
# 信号
connection_changed = Signal(bool, str) # 连接状态变化
def __init__(self, parent=None):
super().__init__(parent)
self.config_helper = ConfigHelper()
self.db_worker = DBWorker(self)
self._connected = False
self._init_ui()
self._connect_signals()
self._load_dsn_from_env()
def _init_ui(self):
"""初始化界面"""
layout = QVBoxLayout(self)
layout.setContentsMargins(16, 16, 16, 16)
layout.setSpacing(16)
# 标题
title = QLabel("数据库查看器")
title.setProperty("heading", True)
layout.addWidget(title)
# 连接配置
conn_group = QGroupBox("数据库连接")
conn_layout = QHBoxLayout(conn_group)
conn_layout.addWidget(QLabel("DSN:"))
self.dsn_edit = QLineEdit()
self.dsn_edit.setPlaceholderText("postgresql://user:password@host:5432/dbname")
self.dsn_edit.setEchoMode(QLineEdit.Password)
conn_layout.addWidget(self.dsn_edit, 1)
self.show_dsn_btn = QPushButton("显示")
self.show_dsn_btn.setProperty("secondary", True)
self.show_dsn_btn.setCheckable(True)
self.show_dsn_btn.setFixedWidth(60)
conn_layout.addWidget(self.show_dsn_btn)
self.connect_btn = QPushButton("连接")
self.connect_btn.setFixedWidth(80)
conn_layout.addWidget(self.connect_btn)
self.disconnect_btn = QPushButton("断开")
self.disconnect_btn.setProperty("secondary", True)
self.disconnect_btn.setFixedWidth(80)
self.disconnect_btn.setEnabled(False)
conn_layout.addWidget(self.disconnect_btn)
layout.addWidget(conn_group)
# 主分割器
main_splitter = QSplitter(Qt.Horizontal)
layout.addWidget(main_splitter, 1)
# 左侧:表浏览器
left_widget = self._create_table_browser()
main_splitter.addWidget(left_widget)
# 右侧:查询和结果
right_widget = self._create_query_area()
main_splitter.addWidget(right_widget)
# 设置分割比例
main_splitter.setSizes([300, 700])
def _create_table_browser(self) -> QWidget:
"""创建表浏览器"""
widget = QWidget()
layout = QVBoxLayout(widget)
layout.setContentsMargins(0, 0, 8, 0)
# 标题和刷新按钮
header_layout = QHBoxLayout()
header_layout.addWidget(QLabel("表结构"))
self.refresh_tables_btn = QPushButton("刷新")
self.refresh_tables_btn.setProperty("secondary", True)
self.refresh_tables_btn.setEnabled(False)
header_layout.addWidget(self.refresh_tables_btn)
layout.addLayout(header_layout)
# 表树形视图
self.table_tree = QTreeWidget()
self.table_tree.setHeaderLabels(["名称", "行数", "最后更新"])
self.table_tree.header().setSectionResizeMode(0, QHeaderView.Stretch)
self.table_tree.setColumnWidth(1, 80)
self.table_tree.setColumnWidth(2, 130)
layout.addWidget(self.table_tree, 1)
return widget
def _create_query_area(self) -> QWidget:
"""创建查询区域"""
widget = QWidget()
layout = QVBoxLayout(widget)
layout.setContentsMargins(8, 0, 0, 0)
# 查询输入区
query_group = QGroupBox("SQL 查询")
query_layout = QVBoxLayout(query_group)
# 模板选择
template_layout = QHBoxLayout()
template_layout.addWidget(QLabel("常用查询:"))
self.template_combo = QComboBox()
self.template_combo.addItem("-- 选择模板 --")
for name in QUERY_TEMPLATES.keys():
self.template_combo.addItem(name)
template_layout.addWidget(self.template_combo, 1)
query_layout.addLayout(template_layout)
# SQL 编辑器
self.sql_editor = QPlainTextEdit()
self.sql_editor.setObjectName("sqlEditor")
self.sql_editor.setPlaceholderText("输入 SQL 查询语句...")
self.sql_editor.setFont(QFont("Consolas", 11))
self.sql_editor.setMaximumHeight(150)
query_layout.addWidget(self.sql_editor)
# 执行按钮
exec_layout = QHBoxLayout()
exec_layout.addStretch()
self.exec_btn = QPushButton("执行查询 (Ctrl+Enter)")
self.exec_btn.setEnabled(False)
exec_layout.addWidget(self.exec_btn)
query_layout.addLayout(exec_layout)
layout.addWidget(query_group)
# 结果区域
result_group = QGroupBox("查询结果")
result_layout = QVBoxLayout(result_group)
# 结果表格
self.result_table = QTableWidget()
self.result_table.setAlternatingRowColors(True)
self.result_table.horizontalHeader().setStretchLastSection(True)
result_layout.addWidget(self.result_table, 1)
# 结果统计
self.result_label = QLabel("就绪")
self.result_label.setProperty("subheading", True)
result_layout.addWidget(self.result_label)
layout.addWidget(result_group, 1)
return widget
def _connect_signals(self):
"""连接信号"""
# 连接按钮
self.show_dsn_btn.toggled.connect(self._toggle_dsn_visibility)
self.connect_btn.clicked.connect(self._connect_db)
self.disconnect_btn.clicked.connect(self._disconnect_db)
self.refresh_tables_btn.clicked.connect(self._refresh_tables)
# 模板选择
self.template_combo.currentIndexChanged.connect(self._on_template_selected)
# 执行查询
self.exec_btn.clicked.connect(self._execute_query)
# 表双击
self.table_tree.itemDoubleClicked.connect(self._on_table_double_clicked)
# 工作线程信号
self.db_worker.connection_status.connect(self._on_connection_status)
self.db_worker.tables_loaded.connect(self._on_tables_loaded)
self.db_worker.query_finished.connect(self._on_query_finished)
self.db_worker.query_error.connect(self._on_query_error)
def _load_dsn_from_env(self):
"""从环境变量加载 DSN"""
env_vars = self.config_helper.load_env()
dsn = env_vars.get("PG_DSN", "")
if dsn:
self.dsn_edit.setText(dsn)
def _toggle_dsn_visibility(self, checked: bool):
"""切换 DSN 可见性"""
self.dsn_edit.setEchoMode(
QLineEdit.Normal if checked else QLineEdit.Password
)
self.show_dsn_btn.setText("隐藏" if checked else "显示")
def _connect_db(self):
"""连接数据库"""
dsn = self.dsn_edit.text().strip()
if not dsn:
QMessageBox.warning(self, "提示", "请输入数据库连接字符串")
return
self.connect_btn.setEnabled(False)
self.connect_btn.setText("连接中...")
self.db_worker.connect_db(dsn)
def _disconnect_db(self):
"""断开数据库连接"""
self.db_worker.disconnect_db()
def _refresh_tables(self):
"""刷新表列表"""
self.db_worker.load_tables()
def _on_connection_status(self, connected: bool, message: str):
"""处理连接状态变化"""
self._connected = connected
self.connect_btn.setEnabled(not connected)
self.connect_btn.setText("连接")
self.disconnect_btn.setEnabled(connected)
self.refresh_tables_btn.setEnabled(connected)
self.exec_btn.setEnabled(connected)
self.connection_changed.emit(connected, message)
if connected:
# 自动加载表列表
self._refresh_tables()
def _on_tables_loaded(self, tables_dict: dict):
"""处理表列表加载完成"""
self.table_tree.clear()
for schema, tables in tables_dict.items():
schema_item = QTreeWidgetItem([schema, "", ""])
schema_item.setExpanded(True)
for table_name, row_count, updated_at in tables:
table_item = QTreeWidgetItem([table_name, str(row_count), updated_at])
table_item.setData(0, Qt.UserRole, f"{schema}.{table_name}")
schema_item.addChild(table_item)
self.table_tree.addTopLevelItem(schema_item)
def _on_template_selected(self, index: int):
"""模板选择变化"""
if index <= 0:
return
template_name = self.template_combo.currentText()
if template_name in QUERY_TEMPLATES:
self.sql_editor.setPlainText(QUERY_TEMPLATES[template_name].strip())
# 重置选择
self.template_combo.setCurrentIndex(0)
def _on_table_double_clicked(self, item: QTreeWidgetItem, column: int):
"""表双击事件"""
full_name = item.data(0, Qt.UserRole)
if full_name:
# 生成预览查询
sql = f"SELECT * FROM {full_name} LIMIT 100;"
self.sql_editor.setPlainText(sql)
self._execute_query()
def _execute_query(self):
"""执行查询"""
sql = self.sql_editor.toPlainText().strip()
if not sql:
QMessageBox.warning(self, "提示", "请输入 SQL 语句")
return
self.exec_btn.setEnabled(False)
self.exec_btn.setText("执行中...")
self.result_label.setText("正在查询...")
self.db_worker.execute_query(sql)
def _on_query_finished(self, columns: list, rows: list):
"""查询完成"""
self.exec_btn.setEnabled(True)
self.exec_btn.setText("执行查询 (Ctrl+Enter)")
# 更新结果表格
self.result_table.clear()
self.result_table.setColumnCount(len(columns))
self.result_table.setRowCount(len(rows))
self.result_table.setHorizontalHeaderLabels(columns)
for row_idx, row_data in enumerate(rows):
for col_idx, col_name in enumerate(columns):
value = row_data.get(col_name, "")
item = QTableWidgetItem(str(value) if value is not None else "NULL")
if value is None:
item.setForeground(Qt.gray)
self.result_table.setItem(row_idx, col_idx, item)
# 更新统计
self.result_label.setText(f"返回 {len(rows)} 行, {len(columns)}")
def _on_query_error(self, error: str):
"""查询错误"""
self.exec_btn.setEnabled(True)
self.exec_btn.setText("执行查询 (Ctrl+Enter)")
self.result_label.setText(f"错误: {error}")
QMessageBox.critical(self, "查询错误", error)
def close_connection(self):
"""关闭连接"""
if self._connected:
self.db_worker.disconnect_db()
def keyPressEvent(self, event):
"""键盘事件"""
# Ctrl+Enter 执行查询
if event.modifiers() == Qt.ControlModifier and event.key() == Qt.Key_Return:
if self._connected:
self._execute_query()
else:
super().keyPressEvent(event)

View File

@@ -0,0 +1,318 @@
# -*- coding: utf-8 -*-
"""环境变量编辑器"""
from PySide6.QtWidgets import (
QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
QGroupBox, QLabel, QLineEdit, QPushButton, QScrollArea,
QFrame, QMessageBox, QFileDialog, QCheckBox
)
from PySide6.QtCore import Qt, Signal
from ..utils.config_helper import ConfigHelper, ENV_GROUPS
class EnvEditor(QWidget):
"""环境变量编辑器"""
# 信号
config_saved = Signal() # 配置保存成功
def __init__(self, parent=None):
super().__init__(parent)
self.config_helper = ConfigHelper()
self.field_widgets = {} # 存储字段控件
self.show_sensitive = False
self._init_ui()
self.load_config()
def _init_ui(self):
"""初始化界面"""
layout = QVBoxLayout(self)
layout.setContentsMargins(16, 16, 16, 16)
layout.setSpacing(16)
# 标题和按钮
header_layout = QHBoxLayout()
title = QLabel("环境配置")
title.setProperty("heading", True)
header_layout.addWidget(title)
header_layout.addStretch()
self.show_sensitive_check = QCheckBox("显示敏感信息")
self.show_sensitive_check.stateChanged.connect(self._toggle_sensitive)
header_layout.addWidget(self.show_sensitive_check)
self.import_btn = QPushButton("导入")
self.import_btn.setProperty("secondary", True)
self.import_btn.clicked.connect(self._import_config)
header_layout.addWidget(self.import_btn)
self.export_btn = QPushButton("导出")
self.export_btn.setProperty("secondary", True)
self.export_btn.clicked.connect(self._export_config)
header_layout.addWidget(self.export_btn)
self.reload_btn = QPushButton("重新加载")
self.reload_btn.setProperty("secondary", True)
self.reload_btn.clicked.connect(self.load_config)
header_layout.addWidget(self.reload_btn)
self.save_btn = QPushButton("保存")
self.save_btn.clicked.connect(self._save_config)
header_layout.addWidget(self.save_btn)
layout.addLayout(header_layout)
# 配置文件路径
path_layout = QHBoxLayout()
path_layout.addWidget(QLabel("配置文件:"))
self.path_label = QLabel(str(self.config_helper.env_path))
self.path_label.setProperty("subheading", True)
path_layout.addWidget(self.path_label, 1)
layout.addLayout(path_layout)
# 滚动区域
scroll_area = QScrollArea()
scroll_area.setWidgetResizable(True)
scroll_area.setFrameShape(QFrame.NoFrame)
layout.addWidget(scroll_area, 1)
# 配置组容器
config_widget = QWidget()
self.config_layout = QVBoxLayout(config_widget)
self.config_layout.setSpacing(16)
# 创建各配置组
self._create_config_groups()
# 弹性空间
self.config_layout.addStretch()
scroll_area.setWidget(config_widget)
# 验证结果
self.validation_label = QLabel()
self.validation_label.setWordWrap(True)
layout.addWidget(self.validation_label)
def _create_config_groups(self):
"""创建配置分组"""
for group_id, group_info in ENV_GROUPS.items():
group = QGroupBox(group_info["title"])
grid_layout = QGridLayout(group)
for row, key in enumerate(group_info["keys"]):
# 标签
label = QLabel(f"{key}:")
label.setMinimumWidth(180)
grid_layout.addWidget(label, row, 0)
# 输入框
edit = QLineEdit()
edit.setPlaceholderText(self._get_placeholder(key))
# 敏感字段处理
if key in group_info.get("sensitive", []):
edit.setEchoMode(QLineEdit.Password)
edit.setProperty("sensitive", True)
edit.textChanged.connect(self._on_value_changed)
grid_layout.addWidget(edit, row, 1)
# 存储控件引用
self.field_widgets[key] = edit
self.config_layout.addWidget(group)
# 其他配置组(动态添加)
self.other_group = QGroupBox("其他配置")
self.other_layout = QGridLayout(self.other_group)
self.other_group.setVisible(False)
self.config_layout.addWidget(self.other_group)
def load_config(self):
"""加载配置"""
env_vars = self.config_helper.load_env()
# 更新已知字段
for key, edit in self.field_widgets.items():
value = env_vars.get(key, "")
edit.blockSignals(True)
edit.setText(value)
edit.blockSignals(False)
# 处理其他字段
known_keys = set(self.field_widgets.keys())
other_keys = [k for k in env_vars.keys() if k not in known_keys]
# 清除旧的其他字段
while self.other_layout.count():
item = self.other_layout.takeAt(0)
if item.widget():
item.widget().deleteLater()
# 添加其他字段
if other_keys:
self.other_group.setVisible(True)
for row, key in enumerate(sorted(other_keys)):
label = QLabel(f"{key}:")
self.other_layout.addWidget(label, row, 0)
edit = QLineEdit(env_vars[key])
edit.textChanged.connect(self._on_value_changed)
self.other_layout.addWidget(edit, row, 1)
self.field_widgets[key] = edit
else:
self.other_group.setVisible(False)
self._validate()
def _save_config(self):
"""保存配置"""
# 收集所有值
env_vars = {}
for key, edit in self.field_widgets.items():
value = edit.text().strip()
if value:
env_vars[key] = value
# 验证
errors = self.config_helper.validate_env(env_vars)
if errors:
reply = QMessageBox.question(
self,
"验证警告",
"配置存在以下问题:\n\n" + "\n".join(f"{e}" for e in errors) + "\n\n是否仍要保存?",
QMessageBox.Yes | QMessageBox.No,
QMessageBox.No
)
if reply == QMessageBox.No:
return
# 保存
if self.config_helper.save_env(env_vars):
QMessageBox.information(self, "成功", "配置已保存")
self.config_saved.emit()
else:
QMessageBox.critical(self, "错误", "保存配置失败")
def _import_config(self):
"""导入配置"""
file_path, _ = QFileDialog.getOpenFileName(
self,
"导入配置文件",
"",
"环境文件 (*.env);;所有文件 (*.*)"
)
if not file_path:
return
try:
from pathlib import Path
temp_helper = ConfigHelper(Path(file_path))
env_vars = temp_helper.load_env()
# 更新字段
for key, value in env_vars.items():
if key in self.field_widgets:
self.field_widgets[key].setText(value)
QMessageBox.information(self, "成功", f"已导入 {len(env_vars)} 个配置项")
except Exception as e:
QMessageBox.critical(self, "错误", f"导入失败: {e}")
def _export_config(self):
"""导出配置"""
file_path, _ = QFileDialog.getSaveFileName(
self,
"导出配置文件",
".env.backup",
"环境文件 (*.env);;所有文件 (*.*)"
)
if not file_path:
return
try:
from pathlib import Path
# 收集当前值
env_vars = {}
for key, edit in self.field_widgets.items():
value = edit.text().strip()
if value:
env_vars[key] = value
# 保存到指定路径
temp_helper = ConfigHelper(Path(file_path))
if temp_helper.save_env(env_vars):
QMessageBox.information(self, "成功", f"配置已导出到:\n{file_path}")
else:
QMessageBox.critical(self, "错误", "导出失败")
except Exception as e:
QMessageBox.critical(self, "错误", f"导出失败: {e}")
def _toggle_sensitive(self, state: int):
"""切换敏感信息显示"""
self.show_sensitive = state == Qt.Checked
for key, edit in self.field_widgets.items():
if edit.property("sensitive"):
edit.setEchoMode(
QLineEdit.Normal if self.show_sensitive else QLineEdit.Password
)
def _on_value_changed(self):
"""值变化时验证"""
self._validate()
def _validate(self):
"""验证配置"""
env_vars = {}
for key, edit in self.field_widgets.items():
value = edit.text().strip()
if value:
env_vars[key] = value
errors = self.config_helper.validate_env(env_vars)
if errors:
self.validation_label.setText("" + "; ".join(errors))
self.validation_label.setProperty("status", "warning")
else:
self.validation_label.setText("✓ 配置验证通过")
self.validation_label.setProperty("status", "success")
self.validation_label.style().unpolish(self.validation_label)
self.validation_label.style().polish(self.validation_label)
@staticmethod
def _get_placeholder(key: str) -> str:
"""获取占位符提示"""
placeholders = {
"PG_DSN": "postgresql://user:password@host:5432/dbname",
"PG_HOST": "localhost",
"PG_PORT": "5432",
"PG_NAME": "billiards",
"PG_USER": "postgres",
"PG_PASSWORD": "密码",
"API_BASE": "https://pc.ficoo.vip/apiprod/admin/v1",
"API_TOKEN": "Bearer token",
"API_TIMEOUT": "20",
"API_PAGE_SIZE": "200",
"STORE_ID": "门店ID (数字)",
"TIMEZONE": "Asia/Taipei",
"EXPORT_ROOT": "export/JSON",
"LOG_ROOT": "export/LOG",
"FETCH_ROOT": "JSON 抓取输出目录",
"INGEST_SOURCE_DIR": "本地 JSON 输入目录",
"PIPELINE_FLOW": "FULL / FETCH_ONLY / INGEST_ONLY",
"RUN_TASKS": "任务列表,逗号分隔",
"OVERLAP_SECONDS": "3600",
"WINDOW_START": "2025-07-01 00:00:00",
"WINDOW_END": "2025-08-01 00:00:00",
}
return placeholders.get(key, "")

View File

@@ -0,0 +1,247 @@
# -*- coding: utf-8 -*-
"""日志查看器"""
import re
from datetime import datetime
from PySide6.QtWidgets import (
QWidget, QVBoxLayout, QHBoxLayout,
QPlainTextEdit, QPushButton, QLineEdit, QLabel,
QComboBox, QCheckBox, QFileDialog, QMessageBox
)
from PySide6.QtCore import Qt, Signal, Slot
from PySide6.QtGui import QTextCharFormat, QColor, QFont, QTextCursor
class LogViewer(QWidget):
"""日志查看器"""
# 信号
log_cleared = Signal()
def __init__(self, parent=None):
super().__init__(parent)
self.max_lines = 10000
self.auto_scroll = True
self.filter_text = ""
self.filter_level = "ALL"
self._all_logs = [] # 存储所有日志
self._init_ui()
self._connect_signals()
def _init_ui(self):
"""初始化界面"""
layout = QVBoxLayout(self)
layout.setContentsMargins(16, 16, 16, 16)
layout.setSpacing(8)
# 标题和工具栏
header_layout = QHBoxLayout()
title = QLabel("执行日志")
title.setProperty("heading", True)
header_layout.addWidget(title)
header_layout.addStretch()
# 日志级别过滤
header_layout.addWidget(QLabel("级别:"))
self.level_combo = QComboBox()
self.level_combo.addItems(["ALL", "INFO", "WARNING", "ERROR", "DEBUG"])
self.level_combo.setFixedWidth(100)
header_layout.addWidget(self.level_combo)
# 搜索框
header_layout.addWidget(QLabel("搜索:"))
self.search_edit = QLineEdit()
self.search_edit.setPlaceholderText("输入关键字...")
self.search_edit.setFixedWidth(200)
header_layout.addWidget(self.search_edit)
# 自动滚动
self.auto_scroll_check = QCheckBox("自动滚动")
self.auto_scroll_check.setChecked(True)
header_layout.addWidget(self.auto_scroll_check)
layout.addLayout(header_layout)
# 日志文本区域
self.log_text = QPlainTextEdit()
self.log_text.setObjectName("logViewer")
self.log_text.setReadOnly(True)
self.log_text.setFont(QFont("Consolas", 10))
self.log_text.setLineWrapMode(QPlainTextEdit.NoWrap)
layout.addWidget(self.log_text, 1)
# 底部工具栏
footer_layout = QHBoxLayout()
self.line_count_label = QLabel("0 行")
self.line_count_label.setProperty("subheading", True)
footer_layout.addWidget(self.line_count_label)
footer_layout.addStretch()
self.copy_btn = QPushButton("复制全部")
self.copy_btn.setProperty("secondary", True)
footer_layout.addWidget(self.copy_btn)
self.export_btn = QPushButton("导出")
self.export_btn.setProperty("secondary", True)
footer_layout.addWidget(self.export_btn)
self.clear_btn = QPushButton("清空")
self.clear_btn.setProperty("secondary", True)
footer_layout.addWidget(self.clear_btn)
layout.addLayout(footer_layout)
def _connect_signals(self):
"""连接信号"""
self.level_combo.currentTextChanged.connect(self._apply_filter)
self.search_edit.textChanged.connect(self._apply_filter)
self.auto_scroll_check.stateChanged.connect(self._toggle_auto_scroll)
self.copy_btn.clicked.connect(self._copy_all)
self.export_btn.clicked.connect(self._export_log)
self.clear_btn.clicked.connect(self._clear_log)
@Slot(str)
def append_log(self, text: str):
"""追加日志"""
# 添加时间戳(如果没有)
if not re.match(r'^\d{4}-\d{2}-\d{2}', text) and not text.startswith('['):
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
text = f"[{timestamp}] {text}"
# 存储到全部日志
self._all_logs.append(text)
# 限制日志行数
if len(self._all_logs) > self.max_lines:
self._all_logs = self._all_logs[-self.max_lines:]
# 检查是否通过过滤器
if self._matches_filter(text):
self._append_formatted_line(text)
# 更新行数
self._update_line_count()
def _append_formatted_line(self, text: str):
"""追加格式化的行"""
cursor = self.log_text.textCursor()
cursor.movePosition(QTextCursor.End)
# 设置格式
fmt = QTextCharFormat()
text_lower = text.lower()
if "[error]" in text_lower or "错误" in text or "失败" in text:
fmt.setForeground(QColor("#d93025"))
fmt.setFontWeight(QFont.Bold)
elif "[warning]" in text_lower or "警告" in text or "warn" in text_lower:
fmt.setForeground(QColor("#f9ab00"))
elif "[info]" in text_lower:
fmt.setForeground(QColor("#1a73e8"))
elif "[debug]" in text_lower:
fmt.setForeground(QColor("#9aa0a6"))
elif "[gui]" in text_lower:
fmt.setForeground(QColor("#1e8e3e"))
else:
fmt.setForeground(QColor("#333333"))
cursor.insertText(text + "\n", fmt)
# 自动滚动
if self.auto_scroll:
self.log_text.verticalScrollBar().setValue(
self.log_text.verticalScrollBar().maximum()
)
def _matches_filter(self, text: str) -> bool:
"""检查是否匹配过滤器"""
# 级别过滤
if self.filter_level != "ALL":
level_marker = f"[{self.filter_level}]"
if level_marker.lower() not in text.lower():
return False
# 文本过滤
if self.filter_text:
if self.filter_text.lower() not in text.lower():
return False
return True
def _apply_filter(self):
"""应用过滤器"""
self.filter_level = self.level_combo.currentText()
self.filter_text = self.search_edit.text().strip()
# 重新显示日志
self.log_text.clear()
for line in self._all_logs:
if self._matches_filter(line):
self._append_formatted_line(line)
self._update_line_count()
def _toggle_auto_scroll(self, state: int):
"""切换自动滚动"""
self.auto_scroll = state == Qt.Checked
def _copy_all(self):
"""复制全部日志"""
from PySide6.QtWidgets import QApplication
text = self.log_text.toPlainText()
QApplication.clipboard().setText(text)
QMessageBox.information(self, "提示", "日志已复制到剪贴板")
def _export_log(self):
"""导出日志"""
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
default_name = f"etl_log_{timestamp}.txt"
file_path, _ = QFileDialog.getSaveFileName(
self,
"导出日志",
default_name,
"文本文件 (*.txt);;日志文件 (*.log);;所有文件 (*.*)"
)
if not file_path:
return
try:
with open(file_path, "w", encoding="utf-8") as f:
f.write(self.log_text.toPlainText())
QMessageBox.information(self, "成功", f"日志已导出到:\n{file_path}")
except Exception as e:
QMessageBox.critical(self, "错误", f"导出失败: {e}")
def _clear_log(self):
"""清空日志"""
reply = QMessageBox.question(
self,
"确认",
"确定要清空所有日志吗?",
QMessageBox.Yes | QMessageBox.No,
QMessageBox.No
)
if reply == QMessageBox.Yes:
self._all_logs.clear()
self.log_text.clear()
self._update_line_count()
self.log_cleared.emit()
def _update_line_count(self):
"""更新行数显示"""
visible_count = self.log_text.document().blockCount() - 1
total_count = len(self._all_logs)
if visible_count < total_count:
self.line_count_label.setText(f"{visible_count} / {total_count}")
else:
self.line_count_label.setText(f"{total_count}")

View File

@@ -0,0 +1,166 @@
# -*- coding: utf-8 -*-
"""应用程序设置对话框"""
from pathlib import Path
from PySide6.QtWidgets import (
QDialog, QVBoxLayout, QHBoxLayout, QGridLayout,
QGroupBox, QLabel, QLineEdit, QPushButton,
QFileDialog, QMessageBox, QDialogButtonBox
)
from PySide6.QtCore import Qt
from ..utils.app_settings import app_settings
class SettingsDialog(QDialog):
"""设置对话框"""
def __init__(self, parent=None):
super().__init__(parent)
self.setWindowTitle("应用程序设置")
self.setMinimumWidth(600)
self._init_ui()
self._load_settings()
def _init_ui(self):
layout = QVBoxLayout(self)
# ETL 项目路径
project_group = QGroupBox("ETL 项目配置")
project_layout = QGridLayout(project_group)
project_layout.addWidget(QLabel("ETL 项目路径:"), 0, 0)
self.project_path_edit = QLineEdit()
self.project_path_edit.setPlaceholderText("例: etl_billiards")
project_layout.addWidget(self.project_path_edit, 0, 1)
browse_project_btn = QPushButton("浏览...")
browse_project_btn.clicked.connect(self._browse_project_path)
project_layout.addWidget(browse_project_btn, 0, 2)
project_layout.addWidget(QLabel(".env 文件路径:"), 1, 0)
self.env_path_edit = QLineEdit()
self.env_path_edit.setPlaceholderText("例: etl_billiards/.env")
project_layout.addWidget(self.env_path_edit, 1, 1)
browse_env_btn = QPushButton("浏览...")
browse_env_btn.clicked.connect(self._browse_env_path)
project_layout.addWidget(browse_env_btn, 1, 2)
# 验证按钮
validate_btn = QPushButton("验证配置")
validate_btn.clicked.connect(self._validate_config)
project_layout.addWidget(validate_btn, 2, 1)
# 验证结果
self.validation_label = QLabel()
self.validation_label.setWordWrap(True)
project_layout.addWidget(self.validation_label, 3, 0, 1, 3)
layout.addWidget(project_group)
# 说明
note = QLabel(
"说明:\n"
"• ETL 项目路径:包含 cli/main.py 的目录\n"
"• .env 文件路径:环境变量配置文件\n"
"• 配置后才能正常执行 ETL 任务"
)
note.setProperty("subheading", True)
note.setWordWrap(True)
layout.addWidget(note)
layout.addStretch()
# 按钮
btn_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
btn_box.accepted.connect(self._save_and_accept)
btn_box.rejected.connect(self.reject)
layout.addWidget(btn_box)
def _load_settings(self):
"""加载设置"""
self.project_path_edit.setText(app_settings.etl_project_path)
self.env_path_edit.setText(app_settings.env_file_path)
self._validate_config()
def _browse_project_path(self):
"""浏览项目路径"""
path = QFileDialog.getExistingDirectory(
self, "选择 ETL 项目目录",
self.project_path_edit.text() or str(Path.home())
)
if path:
self.project_path_edit.setText(path)
# 自动填充 .env 路径
env_path = Path(path) / ".env"
if env_path.exists():
self.env_path_edit.setText(str(env_path))
self._validate_config()
def _browse_env_path(self):
"""浏览 .env 文件"""
path, _ = QFileDialog.getOpenFileName(
self, "选择 .env 文件",
self.env_path_edit.text() or str(Path.home()),
"环境变量文件 (*.env);;所有文件 (*.*)"
)
if path:
self.env_path_edit.setText(path)
self._validate_config()
def _validate_config(self):
"""验证配置"""
project_path = self.project_path_edit.text().strip()
env_path = self.env_path_edit.text().strip()
issues = []
if not project_path:
issues.append("• 未设置 ETL 项目路径")
else:
p = Path(project_path)
if not p.exists():
issues.append(f"• ETL 项目路径不存在")
elif not (p / "cli" / "main.py").exists():
issues.append(f"• 找不到 cli/main.py")
if not env_path:
issues.append("• 未设置 .env 文件路径")
elif not Path(env_path).exists():
issues.append("• .env 文件不存在")
if issues:
self.validation_label.setText("❌ 配置问题:\n" + "\n".join(issues))
self.validation_label.setStyleSheet("color: #d93025;")
else:
self.validation_label.setText("✅ 配置有效")
self.validation_label.setStyleSheet("color: #1e8e3e;")
def _save_and_accept(self):
"""保存并关闭"""
project_path = self.project_path_edit.text().strip()
env_path = self.env_path_edit.text().strip()
# 简单验证
if project_path:
p = Path(project_path)
if not p.exists():
QMessageBox.warning(self, "警告", "ETL 项目路径不存在")
return
if not (p / "cli" / "main.py").exists():
reply = QMessageBox.question(
self, "确认",
"找不到 cli/main.py确定要使用此路径吗",
QMessageBox.Yes | QMessageBox.No
)
if reply == QMessageBox.No:
return
# 保存设置
app_settings.etl_project_path = project_path
if env_path:
app_settings.env_file_path = env_path
self.accept()

View File

@@ -0,0 +1,406 @@
# -*- coding: utf-8 -*-
"""ETL 状态面板"""
from datetime import datetime
from typing import Dict, List, Optional, Any
from PySide6.QtWidgets import (
QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
QGroupBox, QLabel, QPushButton, QTableWidget, QTableWidgetItem,
QHeaderView, QFrame, QScrollArea, QMessageBox
)
from PySide6.QtCore import Qt, Signal, QTimer
from PySide6.QtGui import QColor
from ..workers.db_worker import DBWorker
from ..utils.config_helper import ConfigHelper
class StatusCard(QFrame):
"""状态卡片"""
def __init__(self, title: str, parent=None):
super().__init__(parent)
self.setProperty("card", True)
self.setFrameShape(QFrame.StyledPanel)
layout = QVBoxLayout(self)
layout.setContentsMargins(16, 12, 16, 12)
layout.setSpacing(8)
# 标题
self.title_label = QLabel(title)
self.title_label.setProperty("subheading", True)
layout.addWidget(self.title_label)
# 值
self.value_label = QLabel("-")
self.value_label.setStyleSheet("font-size: 24px; font-weight: bold;")
layout.addWidget(self.value_label)
# 描述
self.desc_label = QLabel("")
self.desc_label.setProperty("subheading", True)
layout.addWidget(self.desc_label)
def set_value(self, value: str, description: str = "", status: str = ""):
"""设置值"""
self.value_label.setText(value)
self.desc_label.setText(description)
if status:
self.value_label.setProperty("status", status)
self.value_label.style().unpolish(self.value_label)
self.value_label.style().polish(self.value_label)
class StatusPanel(QWidget):
"""ETL 状态面板"""
def __init__(self, parent=None):
super().__init__(parent)
self.config_helper = ConfigHelper()
self.db_worker = DBWorker(self)
self._connected = False
self._init_ui()
self._connect_signals()
# 定时刷新
self.refresh_timer = QTimer(self)
self.refresh_timer.timeout.connect(self._auto_refresh)
def _init_ui(self):
"""初始化界面"""
layout = QVBoxLayout(self)
layout.setContentsMargins(16, 16, 16, 16)
layout.setSpacing(16)
# 标题和按钮
header_layout = QHBoxLayout()
title = QLabel("ETL 状态")
title.setProperty("heading", True)
header_layout.addWidget(title)
header_layout.addStretch()
self.auto_refresh_btn = QPushButton("自动刷新: 关")
self.auto_refresh_btn.setProperty("secondary", True)
self.auto_refresh_btn.setCheckable(True)
header_layout.addWidget(self.auto_refresh_btn)
self.refresh_btn = QPushButton("刷新")
self.refresh_btn.clicked.connect(self._refresh_all)
header_layout.addWidget(self.refresh_btn)
layout.addLayout(header_layout)
# 连接状态
self.conn_status_label = QLabel("数据库: 未连接")
self.conn_status_label.setProperty("status", "warning")
layout.addWidget(self.conn_status_label)
# 滚动区域
scroll_area = QScrollArea()
scroll_area.setWidgetResizable(True)
scroll_area.setFrameShape(QFrame.NoFrame)
layout.addWidget(scroll_area, 1)
# 内容容器
content_widget = QWidget()
content_layout = QVBoxLayout(content_widget)
content_layout.setSpacing(16)
# 概览卡片
cards_layout = QHBoxLayout()
self.ods_card = StatusCard("ODS 表数量")
cards_layout.addWidget(self.ods_card)
self.dwd_card = StatusCard("DWD 表数量")
cards_layout.addWidget(self.dwd_card)
self.last_update_card = StatusCard("最后更新")
cards_layout.addWidget(self.last_update_card)
self.task_count_card = StatusCard("今日任务")
cards_layout.addWidget(self.task_count_card)
content_layout.addLayout(cards_layout)
# ODS Cutoff 状态
cutoff_group = QGroupBox("ODS Cutoff 状态")
cutoff_layout = QVBoxLayout(cutoff_group)
self.cutoff_table = QTableWidget()
self.cutoff_table.setColumnCount(4)
self.cutoff_table.setHorizontalHeaderLabels(["表名", "最新 fetched_at", "行数", "状态"])
self.cutoff_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.Stretch)
self.cutoff_table.setMaximumHeight(250)
cutoff_layout.addWidget(self.cutoff_table)
content_layout.addWidget(cutoff_group)
# 最近运行记录
history_group = QGroupBox("最近运行记录")
history_layout = QVBoxLayout(history_group)
self.history_table = QTableWidget()
self.history_table.setColumnCount(6)
self.history_table.setHorizontalHeaderLabels(["运行ID", "任务", "状态", "开始时间", "耗时", "影响行数"])
self.history_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.Stretch)
self.history_table.setMaximumHeight(250)
history_layout.addWidget(self.history_table)
content_layout.addWidget(history_group)
# 弹性空间
content_layout.addStretch()
scroll_area.setWidget(content_widget)
def _connect_signals(self):
"""连接信号"""
self.auto_refresh_btn.toggled.connect(self._toggle_auto_refresh)
self.db_worker.connection_status.connect(self._on_connection_status)
self.db_worker.query_finished.connect(self._on_query_finished)
self.db_worker.query_error.connect(self._on_query_error)
def _toggle_auto_refresh(self, checked: bool):
"""切换自动刷新"""
if checked:
self.auto_refresh_btn.setText("自动刷新: 开")
self.refresh_timer.start(30000) # 30秒刷新一次
self._refresh_all()
else:
self.auto_refresh_btn.setText("自动刷新: 关")
self.refresh_timer.stop()
def _auto_refresh(self):
"""自动刷新"""
if self._connected:
self._refresh_all()
def _refresh_all(self):
"""刷新所有数据"""
# 尝试连接数据库
if not self._connected:
env_vars = self.config_helper.load_env()
dsn = env_vars.get("PG_DSN", "")
if dsn:
self.db_worker.connect_db(dsn)
else:
self.conn_status_label.setText("数据库: 未配置 DSN")
return
else:
self._load_status_data()
def _on_connection_status(self, connected: bool, message: str):
"""处理连接状态"""
self._connected = connected
if connected:
self.conn_status_label.setText(f"数据库: 已连接")
self.conn_status_label.setProperty("status", "success")
self._load_status_data()
else:
self.conn_status_label.setText(f"数据库: {message}")
self.conn_status_label.setProperty("status", "error")
self.conn_status_label.style().unpolish(self.conn_status_label)
self.conn_status_label.style().polish(self.conn_status_label)
def _load_status_data(self):
"""加载状态数据"""
# 加载表统计
self._current_query = "table_count"
self.db_worker.execute_query("""
SELECT
table_schema,
COUNT(*) as table_count
FROM information_schema.tables
WHERE table_schema IN ('billiards_ods', 'billiards_dwd', 'billiards_dws')
GROUP BY table_schema
""")
def _on_query_finished(self, columns: list, rows: list):
"""处理查询结果"""
query_type = getattr(self, '_current_query', '')
if query_type == "table_count":
self._process_table_count(rows)
# 继续加载 cutoff 数据
self._current_query = "cutoff"
self.db_worker.execute_query("""
SELECT
'payment_transactions' AS table_name,
MAX(fetched_at) AS max_fetched_at,
COUNT(*) AS row_count
FROM billiards_ods.payment_transactions
UNION ALL
SELECT 'member_profiles', MAX(fetched_at), COUNT(*)
FROM billiards_ods.member_profiles
UNION ALL
SELECT 'settlement_records', MAX(fetched_at), COUNT(*)
FROM billiards_ods.settlement_records
UNION ALL
SELECT 'recharge_settlements', MAX(fetched_at), COUNT(*)
FROM billiards_ods.recharge_settlements
UNION ALL
SELECT 'assistant_service_records', MAX(fetched_at), COUNT(*)
FROM billiards_ods.assistant_service_records
ORDER BY table_name
""")
elif query_type == "cutoff":
self._process_cutoff_data(rows)
# 继续加载运行历史
self._current_query = "history"
self.db_worker.execute_query("""
SELECT
run_id,
task_code,
status,
started_at,
finished_at,
rows_affected
FROM etl_admin.run_tracker
ORDER BY started_at DESC
LIMIT 20
""")
elif query_type == "history":
self._process_history_data(rows)
self._current_query = ""
def _process_table_count(self, rows: list):
"""处理表数量数据"""
ods_count = 0
dwd_count = 0
for row in rows:
schema = row.get("table_schema", "")
count = row.get("table_count", 0)
if schema == "billiards_ods":
ods_count = count
elif schema == "billiards_dwd":
dwd_count = count
self.ods_card.set_value(str(ods_count), "个表")
self.dwd_card.set_value(str(dwd_count), "个表")
def _process_cutoff_data(self, rows: list):
"""处理 Cutoff 数据"""
self.cutoff_table.setRowCount(len(rows))
latest_time = None
now = datetime.now()
for row_idx, row in enumerate(rows):
table_name = row.get("table_name", "")
max_fetched = row.get("max_fetched_at")
row_count = row.get("row_count", 0)
self.cutoff_table.setItem(row_idx, 0, QTableWidgetItem(table_name))
if max_fetched:
time_str = str(max_fetched)[:19]
self.cutoff_table.setItem(row_idx, 1, QTableWidgetItem(time_str))
# 更新最新时间
if latest_time is None or max_fetched > latest_time:
latest_time = max_fetched
# 计算状态
if isinstance(max_fetched, datetime):
hours_ago = (now - max_fetched).total_seconds() / 3600
if hours_ago < 1:
status = "正常"
status_color = QColor("#1e8e3e")
elif hours_ago < 24:
status = "较新"
status_color = QColor("#1a73e8")
else:
status = f"落后 {int(hours_ago)}h"
status_color = QColor("#f9ab00")
else:
status = "-"
status_color = QColor("#9aa0a6")
else:
self.cutoff_table.setItem(row_idx, 1, QTableWidgetItem("-"))
status = "无数据"
status_color = QColor("#d93025")
self.cutoff_table.setItem(row_idx, 2, QTableWidgetItem(str(row_count)))
status_item = QTableWidgetItem(status)
status_item.setForeground(status_color)
self.cutoff_table.setItem(row_idx, 3, status_item)
# 更新最后更新时间卡片
if latest_time:
time_str = str(latest_time)[:16]
self.last_update_card.set_value(time_str, "")
else:
self.last_update_card.set_value("-", "无数据")
def _process_history_data(self, rows: list):
"""处理运行历史数据"""
self.history_table.setRowCount(len(rows))
today_count = 0
today = datetime.now().date()
for row_idx, row in enumerate(rows):
run_id = row.get("run_id", "")
task_code = row.get("task_code", "")
status = row.get("status", "")
started_at = row.get("started_at")
finished_at = row.get("finished_at")
rows_affected = row.get("rows_affected", 0)
# 统计今日任务
if started_at and isinstance(started_at, datetime):
if started_at.date() == today:
today_count += 1
self.history_table.setItem(row_idx, 0, QTableWidgetItem(str(run_id)[:8] if run_id else "-"))
self.history_table.setItem(row_idx, 1, QTableWidgetItem(task_code))
# 状态
status_item = QTableWidgetItem(status)
if status and "success" in status.lower():
status_item.setForeground(QColor("#1e8e3e"))
elif status and ("fail" in status.lower() or "error" in status.lower()):
status_item.setForeground(QColor("#d93025"))
self.history_table.setItem(row_idx, 2, status_item)
# 开始时间
time_str = str(started_at)[:19] if started_at else "-"
self.history_table.setItem(row_idx, 3, QTableWidgetItem(time_str))
# 耗时
if started_at and finished_at:
try:
duration = (finished_at - started_at).total_seconds()
if duration < 60:
duration_str = f"{duration:.1f}"
else:
duration_str = f"{int(duration // 60)}{int(duration % 60)}"
except:
duration_str = "-"
else:
duration_str = "-"
self.history_table.setItem(row_idx, 4, QTableWidgetItem(duration_str))
# 影响行数
self.history_table.setItem(row_idx, 5, QTableWidgetItem(str(rows_affected or 0)))
# 更新今日任务卡片
self.task_count_card.set_value(str(today_count), "次执行")
def _on_query_error(self, error: str):
"""处理查询错误"""
self._current_query = ""
# 可能是表不存在,忽略错误继续
pass

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff