1282 lines
55 KiB
Python
1282 lines
55 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""任务配置面板"""
|
||
|
||
from datetime import datetime, timedelta
|
||
|
||
from PySide6.QtWidgets import (
|
||
QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||
QGroupBox, QLabel, QLineEdit, QComboBox, QCheckBox,
|
||
QPushButton, QPlainTextEdit, QListWidget, QListWidgetItem,
|
||
QSplitter, QFrame, QFileDialog, QMessageBox, QScrollArea,
|
||
QSpinBox, QTabWidget, QDateTimeEdit
|
||
)
|
||
from PySide6.QtCore import Qt, Signal, QDateTime, QTimer
|
||
from PySide6.QtGui import QFont
|
||
|
||
from ..models.task_model import TaskItem, TaskConfig, TaskCategory, TASK_CATEGORIES, get_task_category
|
||
from ..models.task_registry import (
|
||
task_registry, get_all_task_tuples, get_fact_ods_task_codes,
|
||
get_dimension_ods_task_codes, BusinessDomain, DOMAIN_LABELS
|
||
)
|
||
from ..utils.cli_builder import CLIBuilder
|
||
from ..utils.app_settings import app_settings
|
||
from ..workers.task_worker import TaskWorker
|
||
from .task_selector import TaskSelectorWidget
|
||
|
||
|
||
def _get_all_tasks():
|
||
"""从注册表动态获取所有任务"""
|
||
return get_all_task_tuples()
|
||
|
||
|
||
# 数据校验相关任务
|
||
INTEGRITY_CHECK_TASKS = [
|
||
"DATA_INTEGRITY_CHECK",
|
||
]
|
||
|
||
|
||
class TaskPanel(QWidget):
|
||
"""任务配置面板"""
|
||
|
||
# 信号
|
||
task_started = Signal(str) # 任务开始信号
|
||
task_finished = Signal(bool, str) # 任务完成信号 (success, message)
|
||
log_message = Signal(str) # 日志消息信号
|
||
add_to_queue = Signal(object) # 添加到队列信号 (TaskConfig)
|
||
create_schedule = Signal(str, list, dict) # 创建调度任务信号 (name, task_codes, task_config)
|
||
|
||
def __init__(self, parent=None):
|
||
super().__init__(parent)
|
||
self.cli_builder = CLIBuilder()
|
||
self.worker = None
|
||
self._init_ui()
|
||
self._connect_signals()
|
||
self.refresh_tasks()
|
||
self._load_settings() # 启动时加载保存的设置
|
||
|
||
# 定时器：每秒更新时间预览
|
||
self._time_preview_timer = QTimer(self)
|
||
self._time_preview_timer.timeout.connect(self._update_sync_time_preview)
|
||
self._time_preview_timer.start(1000) # 每秒更新
|
||
|
||
def _init_ui(self):
|
||
"""初始化界面"""
|
||
layout = QVBoxLayout(self)
|
||
layout.setContentsMargins(16, 16, 16, 16)
|
||
layout.setSpacing(16)
|
||
|
||
# 标题
|
||
title = QLabel("任务配置")
|
||
title.setProperty("heading", True)
|
||
layout.addWidget(title)
|
||
|
||
# 创建分割器
|
||
splitter = QSplitter(Qt.Horizontal)
|
||
layout.addWidget(splitter, 1)
|
||
|
||
# 左侧：任务选择
|
||
left_widget = self._create_task_selection()
|
||
splitter.addWidget(left_widget)
|
||
|
||
# 右侧：参数配置
|
||
right_widget = self._create_config_area()
|
||
splitter.addWidget(right_widget)
|
||
|
||
# 设置分割比例
|
||
splitter.setSizes([400, 600])
|
||
|
||
# 底部：CLI 预览和执行按钮
|
||
bottom_widget = self._create_bottom_area()
|
||
layout.addWidget(bottom_widget)
|
||
|
||
def _create_task_selection(self) -> QWidget:
|
||
"""创建任务选择区域"""
|
||
widget = QWidget()
|
||
layout = QVBoxLayout(widget)
|
||
layout.setContentsMargins(0, 0, 8, 0)
|
||
|
||
# 任务分类过滤
|
||
filter_layout = QHBoxLayout()
|
||
filter_layout.addWidget(QLabel("分类:"))
|
||
self.category_combo = QComboBox()
|
||
self.category_combo.addItem("全部", None)
|
||
self.category_combo.addItem("ODS 数据抓取", TaskCategory.ODS)
|
||
self.category_combo.addItem("DWD 装载", TaskCategory.DWD)
|
||
self.category_combo.addItem("DWS 汇总", TaskCategory.DWS)
|
||
self.category_combo.addItem("Schema 初始化", TaskCategory.SCHEMA)
|
||
self.category_combo.addItem("质量检查", TaskCategory.QUALITY)
|
||
self.category_combo.addItem("其他", TaskCategory.OTHER)
|
||
filter_layout.addWidget(self.category_combo, 1)
|
||
layout.addLayout(filter_layout)
|
||
|
||
# 快捷操作按钮
|
||
btn_layout = QHBoxLayout()
|
||
self.select_all_btn = QPushButton("全选")
|
||
self.select_all_btn.setProperty("secondary", True)
|
||
self.deselect_all_btn = QPushButton("全不选")
|
||
self.deselect_all_btn.setProperty("secondary", True)
|
||
btn_layout.addWidget(self.select_all_btn)
|
||
btn_layout.addWidget(self.deselect_all_btn)
|
||
layout.addLayout(btn_layout)
|
||
|
||
# 任务列表
|
||
self.task_list = QListWidget()
|
||
self.task_list.setSelectionMode(QListWidget.MultiSelection)
|
||
layout.addWidget(self.task_list, 1)
|
||
|
||
# 已选任务数
|
||
self.selected_count_label = QLabel("已选: 0 个任务")
|
||
self.selected_count_label.setProperty("subheading", True)
|
||
layout.addWidget(self.selected_count_label)
|
||
|
||
return widget
|
||
|
||
def _create_config_area(self) -> QWidget:
|
||
"""创建参数配置区域"""
|
||
# 使用选项卡区分快捷操作和高级配置
|
||
tab_widget = QTabWidget()
|
||
|
||
# 快捷操作选项卡
|
||
quick_tab = self._create_quick_actions_tab()
|
||
tab_widget.addTab(quick_tab, "快捷操作")
|
||
|
||
# 高级配置选项卡
|
||
advanced_tab = self._create_advanced_config_tab()
|
||
tab_widget.addTab(advanced_tab, "高级配置")
|
||
|
||
return tab_widget
|
||
|
||
def _create_quick_actions_tab(self) -> QWidget:
|
||
"""创建快捷操作选项卡"""
|
||
scroll_area = QScrollArea()
|
||
scroll_area.setWidgetResizable(True)
|
||
scroll_area.setFrameShape(QFrame.NoFrame)
|
||
|
||
widget = QWidget()
|
||
layout = QVBoxLayout(widget)
|
||
layout.setContentsMargins(8, 8, 8, 8)
|
||
layout.setSpacing(12)
|
||
|
||
# ====== 1. 日常增量同步 ======
|
||
sync_group = QGroupBox("日常增量同步")
|
||
sync_layout = QVBoxLayout(sync_group)
|
||
sync_layout.setSpacing(8)
|
||
|
||
# 说明
|
||
sync_desc = QLabel("从 API 抓取最新数据到 ODS，然后装载到 DWD。适用于日常增量更新、数据修复和历史补全。")
|
||
sync_desc.setProperty("subheading", True)
|
||
sync_desc.setWordWrap(True)
|
||
sync_layout.addWidget(sync_desc)
|
||
|
||
# 时间范围
|
||
sync_time_layout = QHBoxLayout()
|
||
sync_time_layout.addWidget(QLabel("回溯时间:"))
|
||
self.sync_hours = QSpinBox()
|
||
self.sync_hours.setRange(1, 720)
|
||
self.sync_hours.setValue(24)
|
||
self.sync_hours.setSuffix(" 小时")
|
||
self.sync_hours.setToolTip("从当前时间向前回溯的小时数")
|
||
sync_time_layout.addWidget(self.sync_hours)
|
||
|
||
sync_time_layout.addWidget(QLabel("冗余:"))
|
||
self.sync_overlap = QSpinBox()
|
||
self.sync_overlap.setRange(0, 7200)
|
||
self.sync_overlap.setValue(3600)
|
||
self.sync_overlap.setSuffix(" 秒")
|
||
self.sync_overlap.setToolTip("额外向前多抓取的时间，避免边界数据丢失")
|
||
sync_time_layout.addWidget(self.sync_overlap)
|
||
sync_time_layout.addStretch()
|
||
sync_layout.addLayout(sync_time_layout)
|
||
|
||
# 时间范围预览（精确到秒）
|
||
self.sync_time_preview = QLabel()
|
||
self.sync_time_preview.setStyleSheet("color: #666; font-size: 12px;")
|
||
sync_layout.addWidget(self.sync_time_preview)
|
||
self._update_sync_time_preview()
|
||
|
||
# 任务选择（使用新组件）
|
||
sync_task_label = QLabel("选择 ODS 任务（按业务域）:")
|
||
sync_layout.addWidget(sync_task_label)
|
||
|
||
self.sync_task_selector = TaskSelectorWidget(
|
||
show_dimensions=True,
|
||
show_facts=True,
|
||
default_select_facts=True,
|
||
default_select_dimensions=True, # 默认包含维度表
|
||
compact=True,
|
||
max_height=200,
|
||
)
|
||
sync_layout.addWidget(self.sync_task_selector)
|
||
|
||
# 附加选项
|
||
sync_options_layout = QHBoxLayout()
|
||
self.sync_include_dwd = QCheckBox("ODS 完成后装载 DWD")
|
||
self.sync_include_dwd.setChecked(True)
|
||
self.sync_include_dwd.setToolTip("ODS 抓取完成后自动执行 DWD 装载任务")
|
||
sync_options_layout.addWidget(self.sync_include_dwd)
|
||
|
||
self.sync_auto_verify = QCheckBox("完成后校验数据")
|
||
self.sync_auto_verify.setChecked(False)
|
||
self.sync_auto_verify.setToolTip("全部完成后执行数据完整性校验")
|
||
sync_options_layout.addWidget(self.sync_auto_verify)
|
||
sync_options_layout.addStretch()
|
||
sync_layout.addLayout(sync_options_layout)
|
||
|
||
# 操作按钮
|
||
sync_btn_layout = QHBoxLayout()
|
||
sync_btn_layout.addStretch()
|
||
self.sync_run_btn = QPushButton("立即执行")
|
||
self.sync_run_btn.setToolTip("添加到任务队列并执行")
|
||
sync_btn_layout.addWidget(self.sync_run_btn)
|
||
self.sync_schedule_btn = QPushButton("创建调度")
|
||
self.sync_schedule_btn.setProperty("secondary", True)
|
||
self.sync_schedule_btn.setToolTip("创建定时调度任务")
|
||
sync_btn_layout.addWidget(self.sync_schedule_btn)
|
||
sync_layout.addLayout(sync_btn_layout)
|
||
|
||
layout.addWidget(sync_group)
|
||
|
||
# ====== 2. 全量重刷 ======
|
||
reload_group = QGroupBox("全量重刷")
|
||
reload_layout = QVBoxLayout(reload_group)
|
||
reload_layout.setSpacing(8)
|
||
|
||
# 说明
|
||
reload_desc = QLabel("指定精确时间窗口，重新抓取 ODS 数据并装载到 DWD。适用于大范围历史数据重刷、跨月补跑等场景。")
|
||
reload_desc.setProperty("subheading", True)
|
||
reload_desc.setWordWrap(True)
|
||
reload_layout.addWidget(reload_desc)
|
||
|
||
# 时间窗口
|
||
reload_time_layout = QGridLayout()
|
||
reload_time_layout.setColumnStretch(1, 1)
|
||
|
||
reload_time_layout.addWidget(QLabel("开始时间:"), 0, 0)
|
||
self.reload_start = QDateTimeEdit()
|
||
self.reload_start.setDisplayFormat("yyyy-MM-dd HH:mm:ss")
|
||
self.reload_start.setCalendarPopup(True)
|
||
self.reload_start.setDateTime(QDateTime.currentDateTime().addDays(-7))
|
||
reload_time_layout.addWidget(self.reload_start, 0, 1)
|
||
|
||
reload_time_layout.addWidget(QLabel("结束时间:"), 1, 0)
|
||
self.reload_end = QDateTimeEdit()
|
||
self.reload_end.setDisplayFormat("yyyy-MM-dd HH:mm:ss")
|
||
self.reload_end.setCalendarPopup(True)
|
||
self.reload_end.setDateTime(QDateTime.currentDateTime())
|
||
reload_time_layout.addWidget(self.reload_end, 1, 1)
|
||
|
||
reload_layout.addLayout(reload_time_layout)
|
||
|
||
# 窗口切分
|
||
reload_split_layout = QHBoxLayout()
|
||
reload_split_layout.addWidget(QLabel("窗口切分:"))
|
||
self.reload_split_combo = QComboBox()
|
||
self.reload_split_combo.addItem("不切分", "none")
|
||
self.reload_split_combo.addItem("按天切分", "day")
|
||
self.reload_split_combo.addItem("按周切分", "week")
|
||
self.reload_split_combo.addItem("按月切分", "month")
|
||
self.reload_split_combo.setToolTip("长时间窗口建议切分执行，避免单次请求过大")
|
||
reload_split_layout.addWidget(self.reload_split_combo)
|
||
reload_split_layout.addStretch()
|
||
reload_layout.addLayout(reload_split_layout)
|
||
|
||
# 任务选择
|
||
reload_task_label = QLabel("选择 ODS 任务（按业务域）:")
|
||
reload_layout.addWidget(reload_task_label)
|
||
|
||
self.reload_task_selector = TaskSelectorWidget(
|
||
show_dimensions=True,
|
||
show_facts=True,
|
||
default_select_facts=True,
|
||
default_select_dimensions=False,
|
||
compact=True,
|
||
max_height=180,
|
||
)
|
||
reload_layout.addWidget(self.reload_task_selector)
|
||
|
||
# 附加选项
|
||
reload_options_layout = QHBoxLayout()
|
||
self.reload_include_dwd = QCheckBox("ODS 完成后装载 DWD")
|
||
self.reload_include_dwd.setChecked(True)
|
||
reload_options_layout.addWidget(self.reload_include_dwd)
|
||
|
||
self.reload_include_dimensions = QCheckBox("包含维度表")
|
||
self.reload_include_dimensions.setChecked(False)
|
||
self.reload_include_dimensions.setToolTip("勾选后会同时重刷维度类 ODS 任务")
|
||
self.reload_include_dimensions.stateChanged.connect(self._on_reload_include_dimensions_changed)
|
||
reload_options_layout.addWidget(self.reload_include_dimensions)
|
||
reload_options_layout.addStretch()
|
||
reload_layout.addLayout(reload_options_layout)
|
||
|
||
# 操作按钮
|
||
reload_btn_layout = QHBoxLayout()
|
||
reload_btn_layout.addStretch()
|
||
self.reload_run_btn = QPushButton("立即执行")
|
||
self.reload_run_btn.setToolTip("添加到任务队列并执行")
|
||
reload_btn_layout.addWidget(self.reload_run_btn)
|
||
reload_layout.addLayout(reload_btn_layout)
|
||
|
||
layout.addWidget(reload_group)
|
||
|
||
# ====== 3. 数据完整性校验 ======
|
||
integrity_group = QGroupBox("数据完整性校验")
|
||
integrity_layout = QVBoxLayout(integrity_group)
|
||
integrity_layout.setSpacing(8)
|
||
|
||
# 说明
|
||
int_desc = QLabel("对比 API 源数据与 ODS 数据，检查是否有缺失或不一致，可选自动补齐。")
|
||
int_desc.setProperty("subheading", True)
|
||
int_desc.setWordWrap(True)
|
||
integrity_layout.addWidget(int_desc)
|
||
|
||
# 校验模式
|
||
mode_layout = QHBoxLayout()
|
||
mode_layout.addWidget(QLabel("校验模式:"))
|
||
self.integrity_mode_combo = QComboBox()
|
||
self.integrity_mode_combo.addItem("历史全量校验", "history")
|
||
self.integrity_mode_combo.addItem("指定时间窗口", "window")
|
||
self.integrity_mode_combo.setToolTip("历史全量：按日期范围校验；指定窗口：精确到秒的时间窗口")
|
||
mode_layout.addWidget(self.integrity_mode_combo, 1)
|
||
integrity_layout.addLayout(mode_layout)
|
||
|
||
# 时间范围 - 历史模式
|
||
self.history_range_widget = QWidget()
|
||
history_layout = QGridLayout(self.history_range_widget)
|
||
history_layout.setContentsMargins(0, 0, 0, 0)
|
||
history_layout.setColumnStretch(1, 1)
|
||
|
||
history_layout.addWidget(QLabel("开始日期:"), 0, 0)
|
||
self.integrity_start_date = QDateTimeEdit()
|
||
self.integrity_start_date.setDisplayFormat("yyyy-MM-dd")
|
||
self.integrity_start_date.setCalendarPopup(True)
|
||
self.integrity_start_date.setDateTime(QDateTime.currentDateTime().addMonths(-6))
|
||
history_layout.addWidget(self.integrity_start_date, 0, 1)
|
||
|
||
history_layout.addWidget(QLabel("结束日期:"), 1, 0)
|
||
self.integrity_end_date = QDateTimeEdit()
|
||
self.integrity_end_date.setDisplayFormat("yyyy-MM-dd")
|
||
self.integrity_end_date.setCalendarPopup(True)
|
||
self.integrity_end_date.setDateTime(QDateTime.currentDateTime())
|
||
history_layout.addWidget(self.integrity_end_date, 1, 1)
|
||
|
||
integrity_layout.addWidget(self.history_range_widget)
|
||
|
||
# 时间范围 - 窗口模式
|
||
self.window_range_widget = QWidget()
|
||
window_layout = QGridLayout(self.window_range_widget)
|
||
window_layout.setContentsMargins(0, 0, 0, 0)
|
||
window_layout.setColumnStretch(1, 1)
|
||
|
||
window_layout.addWidget(QLabel("开始时间:"), 0, 0)
|
||
self.integrity_window_start = QDateTimeEdit()
|
||
self.integrity_window_start.setDisplayFormat("yyyy-MM-dd HH:mm:ss")
|
||
self.integrity_window_start.setCalendarPopup(True)
|
||
self.integrity_window_start.setDateTime(QDateTime.currentDateTime().addDays(-1))
|
||
window_layout.addWidget(self.integrity_window_start, 0, 1)
|
||
|
||
window_layout.addWidget(QLabel("结束时间:"), 1, 0)
|
||
self.integrity_window_end = QDateTimeEdit()
|
||
self.integrity_window_end.setDisplayFormat("yyyy-MM-dd HH:mm:ss")
|
||
self.integrity_window_end.setCalendarPopup(True)
|
||
self.integrity_window_end.setDateTime(QDateTime.currentDateTime())
|
||
window_layout.addWidget(self.integrity_window_end, 1, 1)
|
||
|
||
self.window_range_widget.setVisible(False)
|
||
integrity_layout.addWidget(self.window_range_widget)
|
||
|
||
# 校验选项分组
|
||
int_options_group = QGroupBox("校验选项")
|
||
int_options_layout = QVBoxLayout(int_options_group)
|
||
int_options_layout.setSpacing(4)
|
||
|
||
# 第一行选项
|
||
int_opt_row1 = QHBoxLayout()
|
||
self.integrity_include_dimensions = QCheckBox("包含维度表")
|
||
self.integrity_include_dimensions.setToolTip("校验维度类 ODS 任务（如会员档案、商品档案等）")
|
||
int_opt_row1.addWidget(self.integrity_include_dimensions)
|
||
|
||
self.integrity_compare_content = QCheckBox("对比内容")
|
||
self.integrity_compare_content.setChecked(True)
|
||
self.integrity_compare_content.setToolTip("不仅对比记录数，还对比具体内容是否一致")
|
||
int_opt_row1.addWidget(self.integrity_compare_content)
|
||
int_opt_row1.addStretch()
|
||
int_options_layout.addLayout(int_opt_row1)
|
||
|
||
# 第二行选项
|
||
int_opt_row2 = QHBoxLayout()
|
||
self.integrity_auto_backfill = QCheckBox("自动补齐缺失数据")
|
||
self.integrity_auto_backfill.setToolTip("发现缺失数据后，自动从 API 补充到 ODS")
|
||
int_opt_row2.addWidget(self.integrity_auto_backfill)
|
||
|
||
self.integrity_backfill_mismatch = QCheckBox("补齐不一致数据")
|
||
self.integrity_backfill_mismatch.setToolTip("发现数据不一致时，也进行补齐（需先勾选自动补齐）")
|
||
self.integrity_backfill_mismatch.setEnabled(False)
|
||
int_opt_row2.addWidget(self.integrity_backfill_mismatch)
|
||
int_opt_row2.addStretch()
|
||
int_options_layout.addLayout(int_opt_row2)
|
||
|
||
# 第三行选项
|
||
int_opt_row3 = QHBoxLayout()
|
||
self.integrity_recheck = QCheckBox("补齐后重新校验")
|
||
self.integrity_recheck.setChecked(True)
|
||
self.integrity_recheck.setToolTip("补齐数据后再执行一次校验，确认数据完整")
|
||
self.integrity_recheck.setEnabled(False)
|
||
int_opt_row3.addWidget(self.integrity_recheck)
|
||
int_opt_row3.addStretch()
|
||
int_options_layout.addLayout(int_opt_row3)
|
||
|
||
integrity_layout.addWidget(int_options_group)
|
||
|
||
# 指定任务
|
||
int_task_layout = QHBoxLayout()
|
||
int_task_layout.addWidget(QLabel("指定任务:"))
|
||
self.integrity_ods_tasks = QLineEdit()
|
||
self.integrity_ods_tasks.setPlaceholderText("留空=全部，或输入: ODS_PAYMENT,ODS_MEMBER")
|
||
int_task_layout.addWidget(self.integrity_ods_tasks, 1)
|
||
integrity_layout.addLayout(int_task_layout)
|
||
|
||
# 操作按钮
|
||
int_btn_layout = QHBoxLayout()
|
||
int_btn_layout.addStretch()
|
||
self.integrity_run_btn = QPushButton("开始校验")
|
||
self.integrity_run_btn.setToolTip("添加到任务队列并执行校验")
|
||
int_btn_layout.addWidget(self.integrity_run_btn)
|
||
self.integrity_schedule_btn = QPushButton("创建调度")
|
||
self.integrity_schedule_btn.setProperty("secondary", True)
|
||
int_btn_layout.addWidget(self.integrity_schedule_btn)
|
||
integrity_layout.addLayout(int_btn_layout)
|
||
|
||
layout.addWidget(integrity_group)
|
||
|
||
# 弹性空间
|
||
layout.addStretch()
|
||
|
||
scroll_area.setWidget(widget)
|
||
return scroll_area
|
||
|
||
def _on_reload_include_dimensions_changed(self, state):
|
||
"""全量重刷的"包含维度表"选项变化"""
|
||
include = state == Qt.Checked
|
||
# 更新任务选择器的默认维度选择
|
||
if include:
|
||
# 选中维度类任务
|
||
current_codes = self.reload_task_selector.get_selected_codes()
|
||
dim_codes = get_dimension_ods_task_codes()
|
||
all_codes = list(set(current_codes) | set(dim_codes))
|
||
self.reload_task_selector.set_selected_codes(all_codes)
|
||
else:
|
||
# 取消选中维度类任务
|
||
current_codes = self.reload_task_selector.get_selected_codes()
|
||
dim_codes = set(get_dimension_ods_task_codes())
|
||
fact_codes = [c for c in current_codes if c not in dim_codes]
|
||
self.reload_task_selector.set_selected_codes(fact_codes)
|
||
|
||
def _create_advanced_config_tab(self) -> QWidget:
|
||
"""创建高级配置选项卡"""
|
||
scroll_area = QScrollArea()
|
||
scroll_area.setWidgetResizable(True)
|
||
scroll_area.setFrameShape(QFrame.NoFrame)
|
||
|
||
widget = QWidget()
|
||
layout = QVBoxLayout(widget)
|
||
layout.setContentsMargins(8, 8, 8, 8)
|
||
|
||
# Pipeline 流程配置
|
||
pipeline_group = QGroupBox("流水线配置")
|
||
pipeline_layout = QGridLayout(pipeline_group)
|
||
|
||
pipeline_layout.addWidget(QLabel("运行模式:"), 0, 0)
|
||
self.pipeline_flow_combo = QComboBox()
|
||
self.pipeline_flow_combo.addItem("FULL - 在线抓取 + 入库", "FULL")
|
||
self.pipeline_flow_combo.addItem("FETCH_ONLY - 仅在线抓取落盘", "FETCH_ONLY")
|
||
self.pipeline_flow_combo.addItem("INGEST_ONLY - 仅本地 JSON 入库", "INGEST_ONLY")
|
||
pipeline_layout.addWidget(self.pipeline_flow_combo, 0, 1)
|
||
|
||
self.dry_run_check = QCheckBox("Dry-run 模式（不提交数据库）")
|
||
pipeline_layout.addWidget(self.dry_run_check, 1, 0, 1, 2)
|
||
|
||
layout.addWidget(pipeline_group)
|
||
|
||
# 时间窗口配置
|
||
window_group = QGroupBox("时间窗口（可选）")
|
||
window_layout = QGridLayout(window_group)
|
||
|
||
window_layout.addWidget(QLabel("开始时间:"), 0, 0)
|
||
self.window_start_edit = QLineEdit()
|
||
self.window_start_edit.setPlaceholderText("例: 2025-07-01 00:00:00")
|
||
window_layout.addWidget(self.window_start_edit, 0, 1)
|
||
|
||
window_layout.addWidget(QLabel("结束时间:"), 1, 0)
|
||
self.window_end_edit = QLineEdit()
|
||
self.window_end_edit.setPlaceholderText("例: 2025-08-01 00:00:00")
|
||
window_layout.addWidget(self.window_end_edit, 1, 1)
|
||
|
||
# 窗口切分选项
|
||
window_layout.addWidget(QLabel("切分模式:"), 2, 0)
|
||
self.window_split_combo = QComboBox()
|
||
self.window_split_combo.addItem("不切分", "none")
|
||
self.window_split_combo.addItem("按月切分", "month")
|
||
self.window_split_combo.setToolTip("长时间窗口按月切分执行，避免单次请求过大")
|
||
window_layout.addWidget(self.window_split_combo, 2, 1)
|
||
|
||
window_layout.addWidget(QLabel("补偿小时:"), 3, 0)
|
||
self.window_compensation_spin = QSpinBox()
|
||
self.window_compensation_spin.setRange(0, 168) # 最多7天
|
||
self.window_compensation_spin.setValue(0)
|
||
self.window_compensation_spin.setSuffix(" 小时")
|
||
self.window_compensation_spin.setToolTip("窗口前后扩展的小时数，用于捕获边界数据")
|
||
window_layout.addWidget(self.window_compensation_spin, 3, 1)
|
||
|
||
layout.addWidget(window_group)
|
||
|
||
# 数据源配置
|
||
source_group = QGroupBox("数据源配置（INGEST_ONLY 模式）")
|
||
source_layout = QGridLayout(source_group)
|
||
|
||
source_layout.addWidget(QLabel("JSON 目录:"), 0, 0)
|
||
self.ingest_source_edit = QLineEdit()
|
||
self.ingest_source_edit.setPlaceholderText("本地 JSON 文件目录")
|
||
source_layout.addWidget(self.ingest_source_edit, 0, 1)
|
||
|
||
self.browse_btn = QPushButton("浏览...")
|
||
self.browse_btn.setProperty("secondary", True)
|
||
source_layout.addWidget(self.browse_btn, 0, 2)
|
||
|
||
layout.addWidget(source_group)
|
||
|
||
# 覆盖配置（高级）
|
||
override_group = QGroupBox("覆盖配置（可选）")
|
||
override_layout = QGridLayout(override_group)
|
||
|
||
override_layout.addWidget(QLabel("门店 ID:"), 0, 0)
|
||
self.store_id_edit = QLineEdit()
|
||
self.store_id_edit.setPlaceholderText("使用 .env 中的配置")
|
||
override_layout.addWidget(self.store_id_edit, 0, 1)
|
||
|
||
override_layout.addWidget(QLabel("数据库 DSN:"), 1, 0)
|
||
self.pg_dsn_edit = QLineEdit()
|
||
self.pg_dsn_edit.setPlaceholderText("使用 .env 中的配置")
|
||
override_layout.addWidget(self.pg_dsn_edit, 1, 1)
|
||
|
||
override_layout.addWidget(QLabel("API Token:"), 2, 0)
|
||
self.api_token_edit = QLineEdit()
|
||
self.api_token_edit.setPlaceholderText("使用 .env 中的配置")
|
||
self.api_token_edit.setEchoMode(QLineEdit.Password)
|
||
override_layout.addWidget(self.api_token_edit, 2, 1)
|
||
|
||
layout.addWidget(override_group)
|
||
|
||
# 弹性空间
|
||
layout.addStretch()
|
||
|
||
scroll_area.setWidget(widget)
|
||
return scroll_area
|
||
|
||
def _create_bottom_area(self) -> QWidget:
|
||
"""创建底部区域"""
|
||
widget = QWidget()
|
||
layout = QVBoxLayout(widget)
|
||
layout.setContentsMargins(0, 0, 0, 0)
|
||
|
||
# CLI 预览
|
||
preview_group = QGroupBox("命令行预览")
|
||
preview_layout = QVBoxLayout(preview_group)
|
||
|
||
self.cli_preview = QPlainTextEdit()
|
||
self.cli_preview.setReadOnly(True)
|
||
self.cli_preview.setMaximumHeight(80)
|
||
self.cli_preview.setFont(QFont("Consolas", 10))
|
||
preview_layout.addWidget(self.cli_preview)
|
||
|
||
layout.addWidget(preview_group)
|
||
|
||
# 执行按钮
|
||
btn_layout = QHBoxLayout()
|
||
btn_layout.addStretch()
|
||
|
||
self.add_to_queue_btn = QPushButton("添加到队列")
|
||
self.add_to_queue_btn.setProperty("secondary", True)
|
||
btn_layout.addWidget(self.add_to_queue_btn)
|
||
|
||
self.run_btn = QPushButton("立即执行")
|
||
self.run_btn.setFixedWidth(120)
|
||
btn_layout.addWidget(self.run_btn)
|
||
|
||
self.stop_btn = QPushButton("停止")
|
||
self.stop_btn.setProperty("danger", True)
|
||
self.stop_btn.setEnabled(False)
|
||
self.stop_btn.setFixedWidth(80)
|
||
btn_layout.addWidget(self.stop_btn)
|
||
|
||
layout.addLayout(btn_layout)
|
||
|
||
return widget
|
||
|
||
def _connect_signals(self):
|
||
"""连接信号"""
|
||
# 分类过滤
|
||
self.category_combo.currentIndexChanged.connect(self._filter_tasks)
|
||
|
||
# 任务选择
|
||
self.task_list.itemSelectionChanged.connect(self._on_selection_changed)
|
||
self.select_all_btn.clicked.connect(self._select_all)
|
||
self.deselect_all_btn.clicked.connect(self._deselect_all)
|
||
|
||
# 配置变化
|
||
self.pipeline_flow_combo.currentIndexChanged.connect(self._update_preview)
|
||
self.dry_run_check.stateChanged.connect(self._update_preview)
|
||
self.window_start_edit.textChanged.connect(self._update_preview)
|
||
self.window_end_edit.textChanged.connect(self._update_preview)
|
||
self.ingest_source_edit.textChanged.connect(self._update_preview)
|
||
self.store_id_edit.textChanged.connect(self._update_preview)
|
||
self.pg_dsn_edit.textChanged.connect(self._update_preview)
|
||
self.api_token_edit.textChanged.connect(self._update_preview)
|
||
|
||
# 浏览目录
|
||
self.browse_btn.clicked.connect(self._browse_source_dir)
|
||
|
||
# 执行按钮
|
||
self.run_btn.clicked.connect(self._run_task)
|
||
self.stop_btn.clicked.connect(self._stop_task)
|
||
self.add_to_queue_btn.clicked.connect(self._add_task_to_queue)
|
||
|
||
# ====== 快捷操作 - 日常增量同步 ======
|
||
self.sync_run_btn.clicked.connect(self._run_daily_sync)
|
||
self.sync_schedule_btn.clicked.connect(self._create_sync_schedule)
|
||
|
||
# ====== 快捷操作 - 全量重刷 ======
|
||
self.reload_run_btn.clicked.connect(self._run_full_reload)
|
||
|
||
# ====== 快捷操作 - 数据校验 ======
|
||
self.integrity_run_btn.clicked.connect(self._run_integrity_check)
|
||
self.integrity_schedule_btn.clicked.connect(self._create_integrity_schedule)
|
||
self.integrity_mode_combo.currentIndexChanged.connect(self._on_integrity_mode_changed)
|
||
# 校验选项联动
|
||
self.integrity_auto_backfill.stateChanged.connect(self._on_backfill_option_changed)
|
||
|
||
# ====== 保存设置的信号连接 ======
|
||
# 日常同步设置
|
||
self.sync_hours.valueChanged.connect(self._save_sync_settings)
|
||
self.sync_hours.valueChanged.connect(self._update_sync_time_preview)
|
||
self.sync_overlap.valueChanged.connect(self._save_sync_settings)
|
||
self.sync_overlap.valueChanged.connect(self._update_sync_time_preview)
|
||
self.sync_include_dwd.stateChanged.connect(self._save_sync_settings)
|
||
self.sync_auto_verify.stateChanged.connect(self._save_sync_settings)
|
||
self.sync_task_selector.selection_changed.connect(self._save_sync_settings)
|
||
|
||
# 全量重刷设置
|
||
self.reload_start.dateTimeChanged.connect(self._save_reload_settings)
|
||
self.reload_end.dateTimeChanged.connect(self._save_reload_settings)
|
||
self.reload_split_combo.currentIndexChanged.connect(self._save_reload_settings)
|
||
self.reload_include_dwd.stateChanged.connect(self._save_reload_settings)
|
||
|
||
# 数据校验设置
|
||
self.integrity_mode_combo.currentIndexChanged.connect(self._save_integrity_settings)
|
||
self.integrity_start_date.dateTimeChanged.connect(self._save_integrity_settings)
|
||
self.integrity_end_date.dateTimeChanged.connect(self._save_integrity_settings)
|
||
self.integrity_include_dimensions.stateChanged.connect(self._save_integrity_settings)
|
||
self.integrity_compare_content.stateChanged.connect(self._save_integrity_settings)
|
||
self.integrity_auto_backfill.stateChanged.connect(self._save_integrity_settings)
|
||
self.integrity_backfill_mismatch.stateChanged.connect(self._save_integrity_settings)
|
||
self.integrity_recheck.stateChanged.connect(self._save_integrity_settings)
|
||
self.integrity_ods_tasks.textChanged.connect(self._save_integrity_settings)
|
||
|
||
# 高级设置
|
||
self.pipeline_flow_combo.currentIndexChanged.connect(self._save_advanced_settings)
|
||
self.dry_run_check.stateChanged.connect(self._save_advanced_settings)
|
||
self.window_split_combo.currentIndexChanged.connect(self._save_advanced_settings)
|
||
self.window_compensation_spin.valueChanged.connect(self._save_advanced_settings)
|
||
|
||
def _on_backfill_option_changed(self, state):
|
||
"""自动补齐选项变化时，联动其他选项"""
|
||
enabled = state == Qt.Checked
|
||
self.integrity_backfill_mismatch.setEnabled(enabled)
|
||
self.integrity_recheck.setEnabled(enabled)
|
||
if not enabled:
|
||
self.integrity_backfill_mismatch.setChecked(False)
|
||
self.integrity_recheck.setChecked(True)
|
||
|
||
def refresh_tasks(self):
|
||
"""刷新任务列表"""
|
||
self.task_list.clear()
|
||
|
||
current_category = self.category_combo.currentData()
|
||
|
||
# 从任务注册表动态获取任务列表
|
||
for code, name, desc in _get_all_tasks():
|
||
category = get_task_category(code)
|
||
|
||
# 应用分类过滤
|
||
if current_category is not None and category != current_category:
|
||
continue
|
||
|
||
item = QListWidgetItem(f"{name} ({code})")
|
||
item.setData(Qt.UserRole, code)
|
||
item.setToolTip(desc)
|
||
self.task_list.addItem(item)
|
||
|
||
self._on_selection_changed()
|
||
|
||
def _filter_tasks(self):
|
||
"""过滤任务列表"""
|
||
self.refresh_tasks()
|
||
|
||
def _on_selection_changed(self):
|
||
"""选择变化时"""
|
||
selected = self.task_list.selectedItems()
|
||
self.selected_count_label.setText(f"已选: {len(selected)} 个任务")
|
||
self._update_preview()
|
||
|
||
def _select_all(self):
|
||
"""全选"""
|
||
self.task_list.selectAll()
|
||
|
||
def _deselect_all(self):
|
||
"""全不选"""
|
||
self.task_list.clearSelection()
|
||
|
||
def _browse_source_dir(self):
|
||
"""浏览数据源目录"""
|
||
dir_path = QFileDialog.getExistingDirectory(
|
||
self, "选择 JSON 数据目录"
|
||
)
|
||
if dir_path:
|
||
self.ingest_source_edit.setText(dir_path)
|
||
|
||
def _get_config(self) -> TaskConfig:
|
||
"""获取当前配置"""
|
||
# 获取选中的任务
|
||
selected_tasks = []
|
||
for item in self.task_list.selectedItems():
|
||
task_code = item.data(Qt.UserRole)
|
||
if task_code:
|
||
selected_tasks.append(task_code)
|
||
|
||
# 构建环境变量（窗口切分参数）
|
||
env_vars = {}
|
||
split_unit = self.window_split_combo.currentData() or "none"
|
||
compensation = self.window_compensation_spin.value()
|
||
if split_unit and split_unit != "none":
|
||
env_vars["WINDOW_SPLIT_UNIT"] = split_unit
|
||
if compensation > 0:
|
||
env_vars["WINDOW_COMPENSATION_HOURS"] = str(compensation)
|
||
|
||
# 构建配置
|
||
config = TaskConfig(
|
||
tasks=selected_tasks,
|
||
pipeline_flow=self.pipeline_flow_combo.currentData(),
|
||
dry_run=self.dry_run_check.isChecked(),
|
||
window_start=self.window_start_edit.text().strip() or None,
|
||
window_end=self.window_end_edit.text().strip() or None,
|
||
window_split=split_unit,
|
||
window_compensation=compensation,
|
||
ingest_source=self.ingest_source_edit.text().strip() or None,
|
||
store_id=int(self.store_id_edit.text()) if self.store_id_edit.text().strip().isdigit() else None,
|
||
pg_dsn=self.pg_dsn_edit.text().strip() or None,
|
||
api_token=self.api_token_edit.text().strip() or None,
|
||
env_vars=env_vars,
|
||
)
|
||
|
||
return config
|
||
|
||
def _update_preview(self):
|
||
"""更新命令行预览"""
|
||
config = self._get_config()
|
||
cmd_str = self.cli_builder.build_command_string(config)
|
||
self.cli_preview.setPlainText(cmd_str)
|
||
|
||
def _run_task(self):
|
||
"""执行任务"""
|
||
config = self._get_config()
|
||
|
||
if not config.tasks:
|
||
QMessageBox.warning(self, "提示", "请至少选择一个任务")
|
||
return
|
||
|
||
# 创建工作线程
|
||
cmd = self.cli_builder.build_command(config)
|
||
self.worker = TaskWorker(cmd)
|
||
|
||
# 连接信号
|
||
self.worker.output_received.connect(self._on_output)
|
||
self.worker.task_finished.connect(self._on_finished)
|
||
self.worker.error_occurred.connect(self._on_error)
|
||
|
||
# 更新 UI 状态
|
||
self.run_btn.setEnabled(False)
|
||
self.stop_btn.setEnabled(True)
|
||
|
||
# 发送开始信号
|
||
task_info = ",".join(config.tasks[:3])
|
||
if len(config.tasks) > 3:
|
||
task_info += f" 等{len(config.tasks)}个任务"
|
||
self.task_started.emit(task_info)
|
||
|
||
# 启动
|
||
self.worker.start()
|
||
|
||
def _stop_task(self):
|
||
"""停止任务"""
|
||
if self.worker and self.worker.isRunning():
|
||
self.worker.stop()
|
||
self.log_message.emit("[GUI] 正在停止任务...")
|
||
|
||
def _on_output(self, line: str):
|
||
"""收到输出"""
|
||
self.log_message.emit(line)
|
||
|
||
def _on_finished(self, exit_code: int, summary: str):
|
||
"""任务完成"""
|
||
self.run_btn.setEnabled(True)
|
||
self.stop_btn.setEnabled(False)
|
||
|
||
success = exit_code == 0
|
||
message = summary if summary else ("任务执行成功" if success else f"任务执行失败 (exit={exit_code})")
|
||
self.task_finished.emit(success, message)
|
||
|
||
if success:
|
||
self.log_message.emit(f"[GUI] 任务完成: {message}")
|
||
else:
|
||
self.log_message.emit(f"[GUI] 任务失败: {message}")
|
||
|
||
def _on_error(self, error: str):
|
||
"""发生错误"""
|
||
self.log_message.emit(f"[GUI] 错误: {error}")
|
||
QMessageBox.critical(self, "执行错误", error)
|
||
|
||
def is_running(self) -> bool:
|
||
"""是否正在执行任务"""
|
||
return self.worker is not None and self.worker.isRunning()
|
||
|
||
def _add_task_to_queue(self):
|
||
"""将任务列表中选中的任务添加到队列"""
|
||
config = self._get_config()
|
||
|
||
if not config.tasks:
|
||
QMessageBox.warning(self, "提示", "请至少选择一个任务")
|
||
return
|
||
|
||
# 发送信号添加到队列
|
||
self.add_to_queue.emit(config)
|
||
|
||
task_info = ",".join(config.tasks[:3])
|
||
if len(config.tasks) > 3:
|
||
task_info += f" 等{len(config.tasks)}个"
|
||
|
||
self.log_message.emit(f"[GUI] 已添加到任务队列: {task_info}")
|
||
QMessageBox.information(self, "提示", f"已添加到任务队列\n\n任务: {task_info}\n\n请切换到「任务管理」查看和执行")
|
||
|
||
def _on_integrity_mode_changed(self, index: int):
|
||
"""校验模式变化"""
|
||
mode = self.integrity_mode_combo.currentData()
|
||
self.history_range_widget.setVisible(mode == "history")
|
||
self.window_range_widget.setVisible(mode == "window")
|
||
|
||
def _update_sync_time_preview(self):
|
||
"""更新日常同步的时间范围预览（精确到秒）"""
|
||
now = datetime.now()
|
||
hours = self.sync_hours.value()
|
||
overlap = self.sync_overlap.value()
|
||
start_time = now - timedelta(hours=hours, seconds=overlap)
|
||
|
||
start_str = start_time.strftime("%Y-%m-%d %H:%M:%S")
|
||
end_str = now.strftime("%Y-%m-%d %H:%M:%S")
|
||
self.sync_time_preview.setText(f"时间窗口: {start_str} ~ {end_str}")
|
||
|
||
# ==================== 日常增量同步 ====================
|
||
|
||
def _run_daily_sync(self):
|
||
"""执行日常增量同步"""
|
||
selected_tasks = self.sync_task_selector.get_selected_codes()
|
||
|
||
if not selected_tasks:
|
||
QMessageBox.warning(self, "提示", "请至少选择一个 ODS 任务")
|
||
return
|
||
|
||
hours = self.sync_hours.value()
|
||
overlap = self.sync_overlap.value()
|
||
include_dwd = self.sync_include_dwd.isChecked()
|
||
auto_verify = self.sync_auto_verify.isChecked()
|
||
|
||
now = datetime.now()
|
||
start_time = now - timedelta(hours=hours, seconds=overlap)
|
||
|
||
# 构建完整任务列表
|
||
all_tasks = selected_tasks.copy()
|
||
|
||
# 添加 DWD 装载
|
||
if include_dwd:
|
||
all_tasks.append("DWD_LOAD_FROM_ODS")
|
||
|
||
# 添加数据校验
|
||
if auto_verify:
|
||
all_tasks.append("DATA_INTEGRITY_CHECK")
|
||
|
||
# 构建环境变量
|
||
env_vars = {}
|
||
if auto_verify:
|
||
env_vars["INTEGRITY_MODE"] = "window"
|
||
env_vars["INTEGRITY_AUTO_BACKFILL"] = "1"
|
||
|
||
config = TaskConfig(
|
||
tasks=all_tasks,
|
||
pipeline_flow="FULL",
|
||
window_start=start_time.strftime("%Y-%m-%d %H:%M:%S"),
|
||
window_end=now.strftime("%Y-%m-%d %H:%M:%S"),
|
||
env_vars=env_vars,
|
||
)
|
||
|
||
# 更新预览
|
||
cmd_str = self.cli_builder.build_command_string(config)
|
||
self.cli_preview.setPlainText(cmd_str)
|
||
|
||
# 构建任务描述
|
||
desc_parts = [f"增量同步 ({hours}h)"]
|
||
if include_dwd:
|
||
desc_parts.append("+ DWD")
|
||
if auto_verify:
|
||
desc_parts.append("+ 校验")
|
||
task_desc = " ".join(desc_parts)
|
||
|
||
# 发送信号添加到队列
|
||
self.add_to_queue.emit(config)
|
||
self.log_message.emit(f"[GUI] 已添加到任务队列: {task_desc}")
|
||
QMessageBox.information(
|
||
self, "提示",
|
||
f"已添加到任务队列\n\n"
|
||
f"任务: {task_desc}\n"
|
||
f"ODS 任务数: {len(selected_tasks)}\n"
|
||
f"总任务数: {len(all_tasks)}\n\n"
|
||
f"请切换到「任务管理」查看和执行"
|
||
)
|
||
|
||
def _create_sync_schedule(self):
|
||
"""创建日常同步调度任务"""
|
||
selected_tasks = self.sync_task_selector.get_selected_codes()
|
||
|
||
if not selected_tasks:
|
||
QMessageBox.warning(self, "提示", "请至少选择一个 ODS 任务")
|
||
return
|
||
|
||
hours = self.sync_hours.value()
|
||
include_dwd = self.sync_include_dwd.isChecked()
|
||
|
||
# 构建任务列表
|
||
all_tasks = selected_tasks.copy()
|
||
if include_dwd:
|
||
all_tasks.append("DWD_LOAD_FROM_ODS")
|
||
|
||
task_config = {
|
||
"pipeline_flow": "FULL",
|
||
"lookback_hours": hours,
|
||
"overlap_seconds": self.sync_overlap.value(),
|
||
}
|
||
|
||
# 发送信号创建调度任务
|
||
self.create_schedule.emit(
|
||
f"日常同步 ({hours}h)",
|
||
all_tasks,
|
||
task_config
|
||
)
|
||
self.log_message.emit(f"[GUI] 创建调度任务: 日常同步 ({hours}h)")
|
||
|
||
# ==================== 全量重刷 ====================
|
||
|
||
def _run_full_reload(self):
|
||
"""执行全量重刷"""
|
||
selected_tasks = self.reload_task_selector.get_selected_codes()
|
||
|
||
if not selected_tasks:
|
||
QMessageBox.warning(self, "提示", "请至少选择一个 ODS 任务")
|
||
return
|
||
|
||
start_dt = self.reload_start.dateTime().toPython()
|
||
end_dt = self.reload_end.dateTime().toPython()
|
||
|
||
if start_dt >= end_dt:
|
||
QMessageBox.warning(self, "提示", "开始时间必须早于结束时间")
|
||
return
|
||
|
||
include_dwd = self.reload_include_dwd.isChecked()
|
||
split_unit = self.reload_split_combo.currentData()
|
||
|
||
# 构建完整任务列表
|
||
all_tasks = selected_tasks.copy()
|
||
if include_dwd:
|
||
all_tasks.append("DWD_LOAD_FROM_ODS")
|
||
|
||
# 构建环境变量
|
||
env_vars = {}
|
||
if split_unit and split_unit != "none":
|
||
env_vars["WINDOW_SPLIT_UNIT"] = split_unit
|
||
|
||
config = TaskConfig(
|
||
tasks=all_tasks,
|
||
pipeline_flow="FULL",
|
||
window_start=start_dt.strftime("%Y-%m-%d %H:%M:%S"),
|
||
window_end=end_dt.strftime("%Y-%m-%d %H:%M:%S"),
|
||
env_vars=env_vars,
|
||
)
|
||
|
||
# 更新预览
|
||
cmd_str = self.cli_builder.build_command_string(config)
|
||
self.cli_preview.setPlainText(cmd_str)
|
||
|
||
# 计算时间跨度
|
||
duration = end_dt - start_dt
|
||
days = duration.days
|
||
|
||
# 构建任务描述
|
||
desc = f"全量重刷 ({days}天)"
|
||
if include_dwd:
|
||
desc += " + DWD"
|
||
|
||
# 发送信号添加到队列
|
||
self.add_to_queue.emit(config)
|
||
self.log_message.emit(f"[GUI] 已添加到任务队列: {desc}")
|
||
QMessageBox.information(
|
||
self, "提示",
|
||
f"已添加到任务队列\n\n"
|
||
f"任务: {desc}\n"
|
||
f"时间窗口: {start_dt.strftime('%Y-%m-%d %H:%M')} ~ {end_dt.strftime('%Y-%m-%d %H:%M')}\n"
|
||
f"ODS 任务数: {len(selected_tasks)}\n"
|
||
f"窗口切分: {split_unit if split_unit != 'none' else '不切分'}\n\n"
|
||
f"请切换到「任务管理」查看和执行"
|
||
)
|
||
|
||
# ==================== 数据校验 ====================
|
||
|
||
def _get_integrity_config(self) -> dict:
|
||
"""获取数据校验配置"""
|
||
mode = self.integrity_mode_combo.currentData()
|
||
|
||
config = {
|
||
"pipeline_flow": "FULL",
|
||
"integrity_mode": mode,
|
||
"integrity_include_dimensions": self.integrity_include_dimensions.isChecked(),
|
||
"integrity_compare_content": self.integrity_compare_content.isChecked(),
|
||
"integrity_auto_backfill": self.integrity_auto_backfill.isChecked(),
|
||
"integrity_backfill_mismatch": self.integrity_backfill_mismatch.isChecked(),
|
||
"integrity_recheck": self.integrity_recheck.isChecked(),
|
||
}
|
||
|
||
if mode == "history":
|
||
config["integrity_history_start"] = self.integrity_start_date.dateTime().toString("yyyy-MM-dd")
|
||
config["integrity_history_end"] = self.integrity_end_date.dateTime().toString("yyyy-MM-dd")
|
||
else:
|
||
config["window_start"] = self.integrity_window_start.dateTime().toPython().strftime("%Y-%m-%d %H:%M:%S")
|
||
config["window_end"] = self.integrity_window_end.dateTime().toPython().strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
ods_tasks = self.integrity_ods_tasks.text().strip()
|
||
if ods_tasks:
|
||
config["integrity_ods_task_codes"] = ods_tasks
|
||
|
||
return config
|
||
|
||
def _run_integrity_check(self):
|
||
"""执行数据完整性校验"""
|
||
mode = self.integrity_mode_combo.currentData()
|
||
int_config = self._get_integrity_config()
|
||
|
||
# 构建环境变量
|
||
env_vars = {
|
||
"INTEGRITY_MODE": int_config.get("integrity_mode", "history"),
|
||
"INTEGRITY_INCLUDE_DIMENSIONS": "1" if int_config.get("integrity_include_dimensions") else "0",
|
||
"INTEGRITY_COMPARE_CONTENT": "1" if int_config.get("integrity_compare_content") else "0",
|
||
"INTEGRITY_AUTO_BACKFILL": "1" if int_config.get("integrity_auto_backfill") else "0",
|
||
"INTEGRITY_BACKFILL_MISMATCH": "1" if int_config.get("integrity_backfill_mismatch") else "0",
|
||
"INTEGRITY_RECHECK_AFTER_BACKFILL": "1" if int_config.get("integrity_recheck") else "0",
|
||
}
|
||
|
||
window_start = None
|
||
window_end = None
|
||
|
||
if mode == "history":
|
||
env_vars["INTEGRITY_HISTORY_START"] = int_config.get("integrity_history_start")
|
||
env_vars["INTEGRITY_HISTORY_END"] = int_config.get("integrity_history_end")
|
||
desc = f"数据校验 ({int_config.get('integrity_history_start')} ~ {int_config.get('integrity_history_end')})"
|
||
else:
|
||
window_start = int_config.get("window_start")
|
||
window_end = int_config.get("window_end")
|
||
desc = f"数据校验 ({window_start} ~ {window_end})"
|
||
|
||
if int_config.get("integrity_ods_task_codes"):
|
||
env_vars["INTEGRITY_ODS_TASK_CODES"] = int_config.get("integrity_ods_task_codes")
|
||
|
||
# 构建描述
|
||
options = []
|
||
if int_config.get("integrity_include_dimensions"):
|
||
options.append("含维度")
|
||
if int_config.get("integrity_auto_backfill"):
|
||
options.append("自动补齐")
|
||
if options:
|
||
desc += " [" + ", ".join(options) + "]"
|
||
|
||
config = TaskConfig(
|
||
tasks=["DATA_INTEGRITY_CHECK"],
|
||
pipeline_flow="FULL",
|
||
window_start=window_start,
|
||
window_end=window_end,
|
||
env_vars=env_vars,
|
||
)
|
||
|
||
# 更新预览
|
||
cmd_str = self.cli_builder.build_command_string(config)
|
||
env_preview = "\n".join(f"# {k}={v}" for k, v in env_vars.items())
|
||
self.cli_preview.setPlainText(f"{cmd_str}\n\n# 环境变量:\n{env_preview}")
|
||
|
||
# 发送信号添加到队列
|
||
self.add_to_queue.emit(config)
|
||
self.log_message.emit(f"[GUI] 已添加到任务队列: {desc}")
|
||
QMessageBox.information(self, "提示", f"已添加到任务队列\n\n任务: {desc}\n\n请切换到「任务管理」查看和执行")
|
||
|
||
def _create_integrity_schedule(self):
|
||
"""创建数据校验调度任务"""
|
||
mode = self.integrity_mode_combo.currentData()
|
||
task_config = self._get_integrity_config()
|
||
|
||
if mode == "history":
|
||
desc = f"数据校验 ({task_config.get('integrity_history_start')} ~ {task_config.get('integrity_history_end')})"
|
||
else:
|
||
desc = "数据校验 (窗口模式)"
|
||
|
||
# 发送信号创建调度任务
|
||
self.create_schedule.emit(
|
||
desc,
|
||
["DATA_INTEGRITY_CHECK"],
|
||
task_config
|
||
)
|
||
self.log_message.emit(f"[GUI] 创建调度任务: {desc}")
|
||
|
||
# ==================== 设置持久化 ====================
|
||
|
||
def _load_settings(self):
|
||
"""从持久化存储加载设置"""
|
||
try:
|
||
# 加载日常同步设置
|
||
if hasattr(app_settings, 'sync_hours'):
|
||
self.sync_hours.setValue(app_settings.sync_hours)
|
||
if hasattr(app_settings, 'sync_overlap'):
|
||
self.sync_overlap.setValue(app_settings.sync_overlap)
|
||
if hasattr(app_settings, 'sync_include_dwd'):
|
||
self.sync_include_dwd.setChecked(app_settings.sync_include_dwd)
|
||
if hasattr(app_settings, 'sync_auto_verify'):
|
||
self.sync_auto_verify.setChecked(app_settings.sync_auto_verify)
|
||
|
||
# 恢复日常同步任务选择
|
||
if hasattr(app_settings, 'sync_selected_tasks'):
|
||
saved_tasks = app_settings.sync_selected_tasks
|
||
if saved_tasks:
|
||
self.sync_task_selector.set_selected_codes(saved_tasks)
|
||
|
||
# 加载数据校验设置
|
||
mode = getattr(app_settings, 'integrity_mode', 'history')
|
||
mode_index = 0 if mode == "history" else 1
|
||
self.integrity_mode_combo.setCurrentIndex(mode_index)
|
||
|
||
# 历史日期
|
||
if hasattr(app_settings, 'integrity_history_start') and app_settings.integrity_history_start:
|
||
try:
|
||
start_date = QDateTime.fromString(app_settings.integrity_history_start, "yyyy-MM-dd")
|
||
if start_date.isValid():
|
||
self.integrity_start_date.setDateTime(start_date)
|
||
except Exception:
|
||
pass
|
||
|
||
if hasattr(app_settings, 'integrity_history_end') and app_settings.integrity_history_end:
|
||
try:
|
||
end_date = QDateTime.fromString(app_settings.integrity_history_end, "yyyy-MM-dd")
|
||
if end_date.isValid():
|
||
self.integrity_end_date.setDateTime(end_date)
|
||
except Exception:
|
||
pass
|
||
|
||
if hasattr(app_settings, 'integrity_include_dimensions'):
|
||
self.integrity_include_dimensions.setChecked(app_settings.integrity_include_dimensions)
|
||
if hasattr(app_settings, 'integrity_compare_content'):
|
||
self.integrity_compare_content.setChecked(app_settings.integrity_compare_content)
|
||
if hasattr(app_settings, 'integrity_auto_backfill'):
|
||
self.integrity_auto_backfill.setChecked(app_settings.integrity_auto_backfill)
|
||
# 加载补齐相关子选项（需要在 auto_backfill 之后加载，因为它们的启用状态依赖 auto_backfill）
|
||
if hasattr(app_settings, 'integrity_backfill_mismatch'):
|
||
self.integrity_backfill_mismatch.setChecked(app_settings.integrity_backfill_mismatch)
|
||
if hasattr(app_settings, 'integrity_recheck'):
|
||
self.integrity_recheck.setChecked(app_settings.integrity_recheck)
|
||
if hasattr(app_settings, 'integrity_ods_tasks'):
|
||
self.integrity_ods_tasks.setText(app_settings.integrity_ods_tasks)
|
||
|
||
# 加载高级设置
|
||
pipeline_flow = getattr(app_settings, 'advanced_pipeline_flow', 'FULL')
|
||
flow_map = {"FULL": 0, "FETCH_ONLY": 1, "INGEST_ONLY": 2}
|
||
self.pipeline_flow_combo.setCurrentIndex(flow_map.get(pipeline_flow, 0))
|
||
|
||
if hasattr(app_settings, 'advanced_dry_run'):
|
||
self.dry_run_check.setChecked(app_settings.advanced_dry_run)
|
||
if hasattr(app_settings, 'advanced_window_start'):
|
||
self.window_start_edit.setText(app_settings.advanced_window_start)
|
||
if hasattr(app_settings, 'advanced_window_end'):
|
||
self.window_end_edit.setText(app_settings.advanced_window_end)
|
||
if hasattr(app_settings, 'advanced_ingest_source'):
|
||
self.ingest_source_edit.setText(app_settings.advanced_ingest_source)
|
||
|
||
# 加载窗口切分设置
|
||
split_map = {"none": 0, "day": 1, "week": 2, "month": 3}
|
||
if hasattr(app_settings, 'advanced_window_split'):
|
||
self.window_split_combo.setCurrentIndex(split_map.get(app_settings.advanced_window_split, 0))
|
||
if hasattr(app_settings, 'advanced_window_compensation'):
|
||
self.window_compensation_spin.setValue(app_settings.advanced_window_compensation)
|
||
|
||
# 更新 UI 状态
|
||
self._on_integrity_mode_changed(mode_index)
|
||
|
||
except Exception as e:
|
||
print(f"加载设置失败: {e}")
|
||
|
||
def _save_sync_settings(self):
|
||
"""保存日常同步设置"""
|
||
try:
|
||
app_settings.sync_hours = self.sync_hours.value()
|
||
app_settings.sync_overlap = self.sync_overlap.value()
|
||
app_settings.sync_include_dwd = self.sync_include_dwd.isChecked()
|
||
app_settings.sync_auto_verify = self.sync_auto_verify.isChecked()
|
||
app_settings.sync_selected_tasks = self.sync_task_selector.get_selected_codes()
|
||
except Exception as e:
|
||
print(f"保存日常同步设置失败: {e}")
|
||
|
||
def _save_reload_settings(self):
|
||
"""保存全量重刷设置"""
|
||
try:
|
||
app_settings.reload_split = self.reload_split_combo.currentData() or "none"
|
||
app_settings.reload_include_dwd = self.reload_include_dwd.isChecked()
|
||
except Exception as e:
|
||
print(f"保存全量重刷设置失败: {e}")
|
||
|
||
def _save_integrity_settings(self):
|
||
"""保存数据校验设置"""
|
||
try:
|
||
mode = self.integrity_mode_combo.currentData()
|
||
app_settings.integrity_mode = mode or "history"
|
||
app_settings.integrity_history_start = self.integrity_start_date.dateTime().toString("yyyy-MM-dd")
|
||
app_settings.integrity_history_end = self.integrity_end_date.dateTime().toString("yyyy-MM-dd")
|
||
app_settings.integrity_include_dimensions = self.integrity_include_dimensions.isChecked()
|
||
app_settings.integrity_compare_content = self.integrity_compare_content.isChecked()
|
||
app_settings.integrity_auto_backfill = self.integrity_auto_backfill.isChecked()
|
||
app_settings.integrity_backfill_mismatch = self.integrity_backfill_mismatch.isChecked()
|
||
app_settings.integrity_recheck = self.integrity_recheck.isChecked()
|
||
app_settings.integrity_ods_tasks = self.integrity_ods_tasks.text().strip()
|
||
except Exception as e:
|
||
print(f"保存数据校验设置失败: {e}")
|
||
|
||
def _save_advanced_settings(self):
|
||
"""保存高级设置"""
|
||
try:
|
||
app_settings.advanced_pipeline_flow = self.pipeline_flow_combo.currentData() or "FULL"
|
||
app_settings.advanced_dry_run = self.dry_run_check.isChecked()
|
||
app_settings.advanced_window_start = self.window_start_edit.text().strip()
|
||
app_settings.advanced_window_end = self.window_end_edit.text().strip()
|
||
app_settings.advanced_ingest_source = self.ingest_source_edit.text().strip()
|
||
app_settings.advanced_window_split = self.window_split_combo.currentData() or "none"
|
||
app_settings.advanced_window_compensation = self.window_compensation_spin.value()
|
||
except Exception as e:
|
||
print(f"保存高级设置失败: {e}")
|