# -*- coding: utf-8 -*- """任务配置面板""" from datetime import datetime, timedelta from PySide6.QtWidgets import ( QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, QGroupBox, QLabel, QLineEdit, QComboBox, QCheckBox, QPushButton, QPlainTextEdit, QListWidget, QListWidgetItem, QSplitter, QFrame, QFileDialog, QMessageBox, QScrollArea, QSpinBox, QTabWidget, QDateTimeEdit ) from PySide6.QtCore import Qt, Signal, QDateTime, QTimer from PySide6.QtGui import QFont from ..models.task_model import TaskItem, TaskConfig, TaskCategory, TASK_CATEGORIES, get_task_category from ..models.task_registry import ( task_registry, get_all_task_tuples, get_fact_ods_task_codes, get_dimension_ods_task_codes, BusinessDomain, DOMAIN_LABELS ) from ..utils.cli_builder import CLIBuilder from ..utils.app_settings import app_settings from ..workers.task_worker import TaskWorker from .task_selector import TaskSelectorWidget def _get_all_tasks(): """从注册表动态获取所有任务""" return get_all_task_tuples() # 数据校验相关任务 INTEGRITY_CHECK_TASKS = [ "DATA_INTEGRITY_CHECK", ] class TaskPanel(QWidget): """任务配置面板""" # 信号 task_started = Signal(str) # 任务开始信号 task_finished = Signal(bool, str) # 任务完成信号 (success, message) log_message = Signal(str) # 日志消息信号 add_to_queue = Signal(object) # 添加到队列信号 (TaskConfig) create_schedule = Signal(str, list, dict) # 创建调度任务信号 (name, task_codes, task_config) def __init__(self, parent=None): super().__init__(parent) self.cli_builder = CLIBuilder() self.worker = None self._init_ui() self._connect_signals() self.refresh_tasks() self._load_settings() # 启动时加载保存的设置 # 定时器：每秒更新时间预览 self._time_preview_timer = QTimer(self) self._time_preview_timer.timeout.connect(self._update_sync_time_preview) self._time_preview_timer.start(1000) # 每秒更新 def _init_ui(self): """初始化界面""" layout = QVBoxLayout(self) layout.setContentsMargins(16, 16, 16, 16) layout.setSpacing(16) # 标题 title = QLabel("任务配置") title.setProperty("heading", True) layout.addWidget(title) # 创建分割器 splitter = QSplitter(Qt.Horizontal) layout.addWidget(splitter, 1) # 左侧：任务选择 left_widget = self._create_task_selection() splitter.addWidget(left_widget) # 右侧：参数配置 right_widget = self._create_config_area() splitter.addWidget(right_widget) # 设置分割比例 splitter.setSizes([400, 600]) # 底部：CLI 预览和执行按钮 bottom_widget = self._create_bottom_area() layout.addWidget(bottom_widget) def _create_task_selection(self) -> QWidget: """创建任务选择区域""" widget = QWidget() layout = QVBoxLayout(widget) layout.setContentsMargins(0, 0, 8, 0) # 任务分类过滤 filter_layout = QHBoxLayout() filter_layout.addWidget(QLabel("分类:")) self.category_combo = QComboBox() self.category_combo.addItem("全部", None) self.category_combo.addItem("ODS 数据抓取", TaskCategory.ODS) self.category_combo.addItem("DWD 装载", TaskCategory.DWD) self.category_combo.addItem("DWS 汇总", TaskCategory.DWS) self.category_combo.addItem("Schema 初始化", TaskCategory.SCHEMA) self.category_combo.addItem("质量检查", TaskCategory.QUALITY) self.category_combo.addItem("其他", TaskCategory.OTHER) filter_layout.addWidget(self.category_combo, 1) layout.addLayout(filter_layout) # 快捷操作按钮 btn_layout = QHBoxLayout() self.select_all_btn = QPushButton("全选") self.select_all_btn.setProperty("secondary", True) self.deselect_all_btn = QPushButton("全不选") self.deselect_all_btn.setProperty("secondary", True) btn_layout.addWidget(self.select_all_btn) btn_layout.addWidget(self.deselect_all_btn) layout.addLayout(btn_layout) # 任务列表 self.task_list = QListWidget() self.task_list.setSelectionMode(QListWidget.MultiSelection) layout.addWidget(self.task_list, 1) # 已选任务数 self.selected_count_label = QLabel("已选: 0 个任务") self.selected_count_label.setProperty("subheading", True) layout.addWidget(self.selected_count_label) return widget def _create_config_area(self) -> QWidget: """创建参数配置区域""" # 使用选项卡区分快捷操作和高级配置 tab_widget = QTabWidget() # 快捷操作选项卡 quick_tab = self._create_quick_actions_tab() tab_widget.addTab(quick_tab, "快捷操作") # 高级配置选项卡 advanced_tab = self._create_advanced_config_tab() tab_widget.addTab(advanced_tab, "高级配置") return tab_widget def _create_quick_actions_tab(self) -> QWidget: """创建快捷操作选项卡""" scroll_area = QScrollArea() scroll_area.setWidgetResizable(True) scroll_area.setFrameShape(QFrame.NoFrame) widget = QWidget() layout = QVBoxLayout(widget) layout.setContentsMargins(8, 8, 8, 8) layout.setSpacing(12) # ====== 1. 日常增量同步 ====== sync_group = QGroupBox("日常增量同步") sync_layout = QVBoxLayout(sync_group) sync_layout.setSpacing(8) # 说明 sync_desc = QLabel("从 API 抓取最新数据到 ODS，然后装载到 DWD。适用于日常增量更新。") sync_desc.setProperty("subheading", True) sync_desc.setWordWrap(True) sync_layout.addWidget(sync_desc) # 时间范围 sync_time_layout = QHBoxLayout() sync_time_layout.addWidget(QLabel("回溯时间:")) self.sync_hours = QSpinBox() self.sync_hours.setRange(1, 720) self.sync_hours.setValue(24) self.sync_hours.setSuffix(" 小时") self.sync_hours.setToolTip("从当前时间向前回溯的小时数") sync_time_layout.addWidget(self.sync_hours) sync_time_layout.addWidget(QLabel("冗余:")) self.sync_overlap = QSpinBox() self.sync_overlap.setRange(0, 7200) self.sync_overlap.setValue(3600) self.sync_overlap.setSuffix(" 秒") self.sync_overlap.setToolTip("额外向前多抓取的时间，避免边界数据丢失") sync_time_layout.addWidget(self.sync_overlap) sync_time_layout.addStretch() sync_layout.addLayout(sync_time_layout) # 时间范围预览（精确到秒） self.sync_time_preview = QLabel() self.sync_time_preview.setStyleSheet("color: #666; font-size: 12px;") sync_layout.addWidget(self.sync_time_preview) self._update_sync_time_preview() # 任务选择（使用新组件） sync_task_label = QLabel("选择 ODS 任务（按业务域）:") sync_layout.addWidget(sync_task_label) self.sync_task_selector = TaskSelectorWidget( show_dimensions=True, show_facts=True, default_select_facts=True, default_select_dimensions=False, compact=True, max_height=200, ) sync_layout.addWidget(self.sync_task_selector) # 附加选项 sync_options_layout = QHBoxLayout() self.sync_include_dwd = QCheckBox("ODS 完成后装载 DWD") self.sync_include_dwd.setChecked(True) self.sync_include_dwd.setToolTip("ODS 抓取完成后自动执行 DWD 装载任务") sync_options_layout.addWidget(self.sync_include_dwd) self.sync_auto_verify = QCheckBox("完成后校验数据") self.sync_auto_verify.setChecked(False) self.sync_auto_verify.setToolTip("全部完成后执行数据完整性校验") sync_options_layout.addWidget(self.sync_auto_verify) sync_options_layout.addStretch() sync_layout.addLayout(sync_options_layout) # 操作按钮 sync_btn_layout = QHBoxLayout() sync_btn_layout.addStretch() self.sync_run_btn = QPushButton("立即执行") self.sync_run_btn.setToolTip("添加到任务队列并执行") sync_btn_layout.addWidget(self.sync_run_btn) self.sync_schedule_btn = QPushButton("创建调度") self.sync_schedule_btn.setProperty("secondary", True) self.sync_schedule_btn.setToolTip("创建定时调度任务") sync_btn_layout.addWidget(self.sync_schedule_btn) sync_layout.addLayout(sync_btn_layout) layout.addWidget(sync_group) # ====== 2. 全量重刷 ====== reload_group = QGroupBox("全量重刷") reload_layout = QVBoxLayout(reload_group) reload_layout.setSpacing(8) # 说明 reload_desc = QLabel("指定时间窗口，重新抓取 ODS 数据并装载到 DWD。适用于数据修复或历史回补。") reload_desc.setProperty("subheading", True) reload_desc.setWordWrap(True) reload_layout.addWidget(reload_desc) # 时间窗口 reload_time_layout = QGridLayout() reload_time_layout.setColumnStretch(1, 1) reload_time_layout.addWidget(QLabel("开始时间:"), 0, 0) self.reload_start = QDateTimeEdit() self.reload_start.setDisplayFormat("yyyy-MM-dd HH:mm:ss") self.reload_start.setCalendarPopup(True) self.reload_start.setDateTime(QDateTime.currentDateTime().addDays(-7)) reload_time_layout.addWidget(self.reload_start, 0, 1) reload_time_layout.addWidget(QLabel("结束时间:"), 1, 0) self.reload_end = QDateTimeEdit() self.reload_end.setDisplayFormat("yyyy-MM-dd HH:mm:ss") self.reload_end.setCalendarPopup(True) self.reload_end.setDateTime(QDateTime.currentDateTime()) reload_time_layout.addWidget(self.reload_end, 1, 1) reload_layout.addLayout(reload_time_layout) # 窗口切分 reload_split_layout = QHBoxLayout() reload_split_layout.addWidget(QLabel("窗口切分:")) self.reload_split_combo = QComboBox() self.reload_split_combo.addItem("不切分", "none") self.reload_split_combo.addItem("按天切分", "day") self.reload_split_combo.addItem("按周切分", "week") self.reload_split_combo.addItem("按月切分", "month") self.reload_split_combo.setToolTip("长时间窗口建议切分执行，避免单次请求过大") reload_split_layout.addWidget(self.reload_split_combo) reload_split_layout.addStretch() reload_layout.addLayout(reload_split_layout) # 任务选择 reload_task_label = QLabel("选择 ODS 任务（按业务域）:") reload_layout.addWidget(reload_task_label) self.reload_task_selector = TaskSelectorWidget( show_dimensions=True, show_facts=True, default_select_facts=True, default_select_dimensions=False, compact=True, max_height=180, ) reload_layout.addWidget(self.reload_task_selector) # 附加选项 reload_options_layout = QHBoxLayout() self.reload_include_dwd = QCheckBox("ODS 完成后装载 DWD") self.reload_include_dwd.setChecked(True) reload_options_layout.addWidget(self.reload_include_dwd) self.reload_include_dimensions = QCheckBox("包含维度表") self.reload_include_dimensions.setChecked(False) self.reload_include_dimensions.setToolTip("勾选后会同时重刷维度类 ODS 任务") self.reload_include_dimensions.stateChanged.connect(self._on_reload_include_dimensions_changed) reload_options_layout.addWidget(self.reload_include_dimensions) reload_options_layout.addStretch() reload_layout.addLayout(reload_options_layout) # 操作按钮 reload_btn_layout = QHBoxLayout() reload_btn_layout.addStretch() self.reload_run_btn = QPushButton("立即执行") self.reload_run_btn.setToolTip("添加到任务队列并执行") reload_btn_layout.addWidget(self.reload_run_btn) reload_layout.addLayout(reload_btn_layout) layout.addWidget(reload_group) # ====== 3. 数据完整性校验 ====== integrity_group = QGroupBox("数据完整性校验") integrity_layout = QVBoxLayout(integrity_group) integrity_layout.setSpacing(8) # 说明 int_desc = QLabel("对比 API 源数据与 ODS 数据，检查是否有缺失或不一致，可选自动补齐。") int_desc.setProperty("subheading", True) int_desc.setWordWrap(True) integrity_layout.addWidget(int_desc) # 校验模式 mode_layout = QHBoxLayout() mode_layout.addWidget(QLabel("校验模式:")) self.integrity_mode_combo = QComboBox() self.integrity_mode_combo.addItem("历史全量校验", "history") self.integrity_mode_combo.addItem("指定时间窗口", "window") self.integrity_mode_combo.setToolTip("历史全量：按日期范围校验；指定窗口：精确到秒的时间窗口") mode_layout.addWidget(self.integrity_mode_combo, 1) integrity_layout.addLayout(mode_layout) # 时间范围 - 历史模式 self.history_range_widget = QWidget() history_layout = QGridLayout(self.history_range_widget) history_layout.setContentsMargins(0, 0, 0, 0) history_layout.setColumnStretch(1, 1) history_layout.addWidget(QLabel("开始日期:"), 0, 0) self.integrity_start_date = QDateTimeEdit() self.integrity_start_date.setDisplayFormat("yyyy-MM-dd") self.integrity_start_date.setCalendarPopup(True) self.integrity_start_date.setDateTime(QDateTime.currentDateTime().addMonths(-6)) history_layout.addWidget(self.integrity_start_date, 0, 1) history_layout.addWidget(QLabel("结束日期:"), 1, 0) self.integrity_end_date = QDateTimeEdit() self.integrity_end_date.setDisplayFormat("yyyy-MM-dd") self.integrity_end_date.setCalendarPopup(True) self.integrity_end_date.setDateTime(QDateTime.currentDateTime()) history_layout.addWidget(self.integrity_end_date, 1, 1) integrity_layout.addWidget(self.history_range_widget) # 时间范围 - 窗口模式 self.window_range_widget = QWidget() window_layout = QGridLayout(self.window_range_widget) window_layout.setContentsMargins(0, 0, 0, 0) window_layout.setColumnStretch(1, 1) window_layout.addWidget(QLabel("开始时间:"), 0, 0) self.integrity_window_start = QDateTimeEdit() self.integrity_window_start.setDisplayFormat("yyyy-MM-dd HH:mm:ss") self.integrity_window_start.setCalendarPopup(True) self.integrity_window_start.setDateTime(QDateTime.currentDateTime().addDays(-1)) window_layout.addWidget(self.integrity_window_start, 0, 1) window_layout.addWidget(QLabel("结束时间:"), 1, 0) self.integrity_window_end = QDateTimeEdit() self.integrity_window_end.setDisplayFormat("yyyy-MM-dd HH:mm:ss") self.integrity_window_end.setCalendarPopup(True) self.integrity_window_end.setDateTime(QDateTime.currentDateTime()) window_layout.addWidget(self.integrity_window_end, 1, 1) self.window_range_widget.setVisible(False) integrity_layout.addWidget(self.window_range_widget) # 校验选项分组 int_options_group = QGroupBox("校验选项") int_options_layout = QVBoxLayout(int_options_group) int_options_layout.setSpacing(4) # 第一行选项 int_opt_row1 = QHBoxLayout() self.integrity_include_dimensions = QCheckBox("包含维度表") self.integrity_include_dimensions.setToolTip("校验维度类 ODS 任务（如会员档案、商品档案等）") int_opt_row1.addWidget(self.integrity_include_dimensions) self.integrity_compare_content = QCheckBox("对比内容") self.integrity_compare_content.setChecked(True) self.integrity_compare_content.setToolTip("不仅对比记录数，还对比具体内容是否一致") int_opt_row1.addWidget(self.integrity_compare_content) int_opt_row1.addStretch() int_options_layout.addLayout(int_opt_row1) # 第二行选项 int_opt_row2 = QHBoxLayout() self.integrity_auto_backfill = QCheckBox("自动补齐缺失数据") self.integrity_auto_backfill.setToolTip("发现缺失数据后，自动从 API 补充到 ODS") int_opt_row2.addWidget(self.integrity_auto_backfill) self.integrity_backfill_mismatch = QCheckBox("补齐不一致数据") self.integrity_backfill_mismatch.setToolTip("发现数据不一致时，也进行补齐（需先勾选自动补齐）") self.integrity_backfill_mismatch.setEnabled(False) int_opt_row2.addWidget(self.integrity_backfill_mismatch) int_opt_row2.addStretch() int_options_layout.addLayout(int_opt_row2) # 第三行选项 int_opt_row3 = QHBoxLayout() self.integrity_recheck = QCheckBox("补齐后重新校验") self.integrity_recheck.setChecked(True) self.integrity_recheck.setToolTip("补齐数据后再执行一次校验，确认数据完整") self.integrity_recheck.setEnabled(False) int_opt_row3.addWidget(self.integrity_recheck) int_opt_row3.addStretch() int_options_layout.addLayout(int_opt_row3) integrity_layout.addWidget(int_options_group) # 指定任务 int_task_layout = QHBoxLayout() int_task_layout.addWidget(QLabel("指定任务:")) self.integrity_ods_tasks = QLineEdit() self.integrity_ods_tasks.setPlaceholderText("留空=全部，或输入: ODS_PAYMENT,ODS_MEMBER") int_task_layout.addWidget(self.integrity_ods_tasks, 1) integrity_layout.addLayout(int_task_layout) # 操作按钮 int_btn_layout = QHBoxLayout() int_btn_layout.addStretch() self.integrity_run_btn = QPushButton("开始校验") self.integrity_run_btn.setToolTip("添加到任务队列并执行校验") int_btn_layout.addWidget(self.integrity_run_btn) self.integrity_schedule_btn = QPushButton("创建调度") self.integrity_schedule_btn.setProperty("secondary", True) int_btn_layout.addWidget(self.integrity_schedule_btn) integrity_layout.addLayout(int_btn_layout) layout.addWidget(integrity_group) # 弹性空间 layout.addStretch() scroll_area.setWidget(widget) return scroll_area def _on_reload_include_dimensions_changed(self, state): """全量重刷的"包含维度表"选项变化""" include = state == Qt.Checked # 更新任务选择器的默认维度选择 if include: # 选中维度类任务 current_codes = self.reload_task_selector.get_selected_codes() dim_codes = get_dimension_ods_task_codes() all_codes = list(set(current_codes) | set(dim_codes)) self.reload_task_selector.set_selected_codes(all_codes) else: # 取消选中维度类任务 current_codes = self.reload_task_selector.get_selected_codes() dim_codes = set(get_dimension_ods_task_codes()) fact_codes = [c for c in current_codes if c not in dim_codes] self.reload_task_selector.set_selected_codes(fact_codes) def _create_advanced_config_tab(self) -> QWidget: """创建高级配置选项卡""" scroll_area = QScrollArea() scroll_area.setWidgetResizable(True) scroll_area.setFrameShape(QFrame.NoFrame) widget = QWidget() layout = QVBoxLayout(widget) layout.setContentsMargins(8, 8, 8, 8) # Pipeline 流程配置 pipeline_group = QGroupBox("流水线配置") pipeline_layout = QGridLayout(pipeline_group) pipeline_layout.addWidget(QLabel("运行模式:"), 0, 0) self.pipeline_flow_combo = QComboBox() self.pipeline_flow_combo.addItem("FULL - 在线抓取 + 入库", "FULL") self.pipeline_flow_combo.addItem("FETCH_ONLY - 仅在线抓取落盘", "FETCH_ONLY") self.pipeline_flow_combo.addItem("INGEST_ONLY - 仅本地 JSON 入库", "INGEST_ONLY") pipeline_layout.addWidget(self.pipeline_flow_combo, 0, 1) self.dry_run_check = QCheckBox("Dry-run 模式（不提交数据库）") pipeline_layout.addWidget(self.dry_run_check, 1, 0, 1, 2) layout.addWidget(pipeline_group) # 时间窗口配置 window_group = QGroupBox("时间窗口（可选）") window_layout = QGridLayout(window_group) window_layout.addWidget(QLabel("开始时间:"), 0, 0) self.window_start_edit = QLineEdit() self.window_start_edit.setPlaceholderText("例: 2025-07-01 00:00:00") window_layout.addWidget(self.window_start_edit, 0, 1) window_layout.addWidget(QLabel("结束时间:"), 1, 0) self.window_end_edit = QLineEdit() self.window_end_edit.setPlaceholderText("例: 2025-08-01 00:00:00") window_layout.addWidget(self.window_end_edit, 1, 1) # 窗口切分选项 window_layout.addWidget(QLabel("切分模式:"), 2, 0) self.window_split_combo = QComboBox() self.window_split_combo.addItem("不切分", "none") self.window_split_combo.addItem("按月切分", "month") self.window_split_combo.setToolTip("长时间窗口按月切分执行，避免单次请求过大") window_layout.addWidget(self.window_split_combo, 2, 1) window_layout.addWidget(QLabel("补偿小时:"), 3, 0) self.window_compensation_spin = QSpinBox() self.window_compensation_spin.setRange(0, 168) # 最多7天 self.window_compensation_spin.setValue(0) self.window_compensation_spin.setSuffix(" 小时") self.window_compensation_spin.setToolTip("窗口前后扩展的小时数，用于捕获边界数据") window_layout.addWidget(self.window_compensation_spin, 3, 1) layout.addWidget(window_group) # 数据源配置 source_group = QGroupBox("数据源配置（INGEST_ONLY 模式）") source_layout = QGridLayout(source_group) source_layout.addWidget(QLabel("JSON 目录:"), 0, 0) self.ingest_source_edit = QLineEdit() self.ingest_source_edit.setPlaceholderText("本地 JSON 文件目录") source_layout.addWidget(self.ingest_source_edit, 0, 1) self.browse_btn = QPushButton("浏览...") self.browse_btn.setProperty("secondary", True) source_layout.addWidget(self.browse_btn, 0, 2) layout.addWidget(source_group) # 覆盖配置（高级） override_group = QGroupBox("覆盖配置（可选）") override_layout = QGridLayout(override_group) override_layout.addWidget(QLabel("门店 ID:"), 0, 0) self.store_id_edit = QLineEdit() self.store_id_edit.setPlaceholderText("使用 .env 中的配置") override_layout.addWidget(self.store_id_edit, 0, 1) override_layout.addWidget(QLabel("数据库 DSN:"), 1, 0) self.pg_dsn_edit = QLineEdit() self.pg_dsn_edit.setPlaceholderText("使用 .env 中的配置") override_layout.addWidget(self.pg_dsn_edit, 1, 1) override_layout.addWidget(QLabel("API Token:"), 2, 0) self.api_token_edit = QLineEdit() self.api_token_edit.setPlaceholderText("使用 .env 中的配置") self.api_token_edit.setEchoMode(QLineEdit.Password) override_layout.addWidget(self.api_token_edit, 2, 1) layout.addWidget(override_group) # 弹性空间 layout.addStretch() scroll_area.setWidget(widget) return scroll_area def _create_bottom_area(self) -> QWidget: """创建底部区域""" widget = QWidget() layout = QVBoxLayout(widget) layout.setContentsMargins(0, 0, 0, 0) # CLI 预览 preview_group = QGroupBox("命令行预览") preview_layout = QVBoxLayout(preview_group) self.cli_preview = QPlainTextEdit() self.cli_preview.setReadOnly(True) self.cli_preview.setMaximumHeight(80) self.cli_preview.setFont(QFont("Consolas", 10)) preview_layout.addWidget(self.cli_preview) layout.addWidget(preview_group) # 执行按钮 btn_layout = QHBoxLayout() btn_layout.addStretch() self.add_to_queue_btn = QPushButton("添加到队列") self.add_to_queue_btn.setProperty("secondary", True) btn_layout.addWidget(self.add_to_queue_btn) self.run_btn = QPushButton("立即执行") self.run_btn.setFixedWidth(120) btn_layout.addWidget(self.run_btn) self.stop_btn = QPushButton("停止") self.stop_btn.setProperty("danger", True) self.stop_btn.setEnabled(False) self.stop_btn.setFixedWidth(80) btn_layout.addWidget(self.stop_btn) layout.addLayout(btn_layout) return widget def _connect_signals(self): """连接信号""" # 分类过滤 self.category_combo.currentIndexChanged.connect(self._filter_tasks) # 任务选择 self.task_list.itemSelectionChanged.connect(self._on_selection_changed) self.select_all_btn.clicked.connect(self._select_all) self.deselect_all_btn.clicked.connect(self._deselect_all) # 配置变化 self.pipeline_flow_combo.currentIndexChanged.connect(self._update_preview) self.dry_run_check.stateChanged.connect(self._update_preview) self.window_start_edit.textChanged.connect(self._update_preview) self.window_end_edit.textChanged.connect(self._update_preview) self.ingest_source_edit.textChanged.connect(self._update_preview) self.store_id_edit.textChanged.connect(self._update_preview) self.pg_dsn_edit.textChanged.connect(self._update_preview) self.api_token_edit.textChanged.connect(self._update_preview) # 浏览目录 self.browse_btn.clicked.connect(self._browse_source_dir) # 执行按钮 self.run_btn.clicked.connect(self._run_task) self.stop_btn.clicked.connect(self._stop_task) self.add_to_queue_btn.clicked.connect(self._add_task_to_queue) # ====== 快捷操作 - 日常增量同步 ====== self.sync_run_btn.clicked.connect(self._run_daily_sync) self.sync_schedule_btn.clicked.connect(self._create_sync_schedule) # ====== 快捷操作 - 全量重刷 ====== self.reload_run_btn.clicked.connect(self._run_full_reload) # ====== 快捷操作 - 数据校验 ====== self.integrity_run_btn.clicked.connect(self._run_integrity_check) self.integrity_schedule_btn.clicked.connect(self._create_integrity_schedule) self.integrity_mode_combo.currentIndexChanged.connect(self._on_integrity_mode_changed) # 校验选项联动 self.integrity_auto_backfill.stateChanged.connect(self._on_backfill_option_changed) # ====== 保存设置的信号连接 ====== # 日常同步设置 self.sync_hours.valueChanged.connect(self._save_sync_settings) self.sync_hours.valueChanged.connect(self._update_sync_time_preview) self.sync_overlap.valueChanged.connect(self._save_sync_settings) self.sync_overlap.valueChanged.connect(self._update_sync_time_preview) self.sync_include_dwd.stateChanged.connect(self._save_sync_settings) self.sync_auto_verify.stateChanged.connect(self._save_sync_settings) self.sync_task_selector.selection_changed.connect(self._save_sync_settings) # 全量重刷设置 self.reload_start.dateTimeChanged.connect(self._save_reload_settings) self.reload_end.dateTimeChanged.connect(self._save_reload_settings) self.reload_split_combo.currentIndexChanged.connect(self._save_reload_settings) self.reload_include_dwd.stateChanged.connect(self._save_reload_settings) # 数据校验设置 self.integrity_mode_combo.currentIndexChanged.connect(self._save_integrity_settings) self.integrity_start_date.dateTimeChanged.connect(self._save_integrity_settings) self.integrity_end_date.dateTimeChanged.connect(self._save_integrity_settings) self.integrity_include_dimensions.stateChanged.connect(self._save_integrity_settings) self.integrity_compare_content.stateChanged.connect(self._save_integrity_settings) self.integrity_auto_backfill.stateChanged.connect(self._save_integrity_settings) self.integrity_backfill_mismatch.stateChanged.connect(self._save_integrity_settings) self.integrity_recheck.stateChanged.connect(self._save_integrity_settings) self.integrity_ods_tasks.textChanged.connect(self._save_integrity_settings) # 高级设置 self.pipeline_flow_combo.currentIndexChanged.connect(self._save_advanced_settings) self.dry_run_check.stateChanged.connect(self._save_advanced_settings) self.window_split_combo.currentIndexChanged.connect(self._save_advanced_settings) self.window_compensation_spin.valueChanged.connect(self._save_advanced_settings) def _on_backfill_option_changed(self, state): """自动补齐选项变化时，联动其他选项""" enabled = state == Qt.Checked self.integrity_backfill_mismatch.setEnabled(enabled) self.integrity_recheck.setEnabled(enabled) if not enabled: self.integrity_backfill_mismatch.setChecked(False) self.integrity_recheck.setChecked(True) def refresh_tasks(self): """刷新任务列表""" self.task_list.clear() current_category = self.category_combo.currentData() # 从任务注册表动态获取任务列表 for code, name, desc in _get_all_tasks(): category = get_task_category(code) # 应用分类过滤 if current_category is not None and category != current_category: continue item = QListWidgetItem(f"{name} ({code})") item.setData(Qt.UserRole, code) item.setToolTip(desc) self.task_list.addItem(item) self._on_selection_changed() def _filter_tasks(self): """过滤任务列表""" self.refresh_tasks() def _on_selection_changed(self): """选择变化时""" selected = self.task_list.selectedItems() self.selected_count_label.setText(f"已选: {len(selected)} 个任务") self._update_preview() def _select_all(self): """全选""" self.task_list.selectAll() def _deselect_all(self): """全不选""" self.task_list.clearSelection() def _browse_source_dir(self): """浏览数据源目录""" dir_path = QFileDialog.getExistingDirectory( self, "选择 JSON 数据目录" ) if dir_path: self.ingest_source_edit.setText(dir_path) def _get_config(self) -> TaskConfig: """获取当前配置""" # 获取选中的任务 selected_tasks = [] for item in self.task_list.selectedItems(): task_code = item.data(Qt.UserRole) if task_code: selected_tasks.append(task_code) # 构建环境变量（窗口切分参数） env_vars = {} split_unit = self.window_split_combo.currentData() or "none" compensation = self.window_compensation_spin.value() if split_unit and split_unit != "none": env_vars["WINDOW_SPLIT_UNIT"] = split_unit if compensation > 0: env_vars["WINDOW_COMPENSATION_HOURS"] = str(compensation) # 构建配置 config = TaskConfig( tasks=selected_tasks, pipeline_flow=self.pipeline_flow_combo.currentData(), dry_run=self.dry_run_check.isChecked(), window_start=self.window_start_edit.text().strip() or None, window_end=self.window_end_edit.text().strip() or None, window_split=split_unit, window_compensation=compensation, ingest_source=self.ingest_source_edit.text().strip() or None, store_id=int(self.store_id_edit.text()) if self.store_id_edit.text().strip().isdigit() else None, pg_dsn=self.pg_dsn_edit.text().strip() or None, api_token=self.api_token_edit.text().strip() or None, env_vars=env_vars, ) return config def _update_preview(self): """更新命令行预览""" config = self._get_config() cmd_str = self.cli_builder.build_command_string(config) self.cli_preview.setPlainText(cmd_str) def _run_task(self): """执行任务""" config = self._get_config() if not config.tasks: QMessageBox.warning(self, "提示", "请至少选择一个任务") return # 创建工作线程 cmd = self.cli_builder.build_command(config) self.worker = TaskWorker(cmd) # 连接信号 self.worker.output_received.connect(self._on_output) self.worker.task_finished.connect(self._on_finished) self.worker.error_occurred.connect(self._on_error) # 更新 UI 状态 self.run_btn.setEnabled(False) self.stop_btn.setEnabled(True) # 发送开始信号 task_info = ",".join(config.tasks[:3]) if len(config.tasks) > 3: task_info += f" 等{len(config.tasks)}个任务" self.task_started.emit(task_info) # 启动 self.worker.start() def _stop_task(self): """停止任务""" if self.worker and self.worker.isRunning(): self.worker.stop() self.log_message.emit("[GUI] 正在停止任务...") def _on_output(self, line: str): """收到输出""" self.log_message.emit(line) def _on_finished(self, exit_code: int, summary: str): """任务完成""" self.run_btn.setEnabled(True) self.stop_btn.setEnabled(False) success = exit_code == 0 message = summary if summary else ("任务执行成功" if success else f"任务执行失败 (exit={exit_code})") self.task_finished.emit(success, message) if success: self.log_message.emit(f"[GUI] 任务完成: {message}") else: self.log_message.emit(f"[GUI] 任务失败: {message}") def _on_error(self, error: str): """发生错误""" self.log_message.emit(f"[GUI] 错误: {error}") QMessageBox.critical(self, "执行错误", error) def is_running(self) -> bool: """是否正在执行任务""" return self.worker is not None and self.worker.isRunning() def _add_task_to_queue(self): """将任务列表中选中的任务添加到队列""" config = self._get_config() if not config.tasks: QMessageBox.warning(self, "提示", "请至少选择一个任务") return # 发送信号添加到队列 self.add_to_queue.emit(config) task_info = ",".join(config.tasks[:3]) if len(config.tasks) > 3: task_info += f" 等{len(config.tasks)}个" self.log_message.emit(f"[GUI] 已添加到任务队列: {task_info}") QMessageBox.information(self, "提示", f"已添加到任务队列\n\n任务: {task_info}\n\n请切换到「任务管理」查看和执行") def _on_integrity_mode_changed(self, index: int): """校验模式变化""" mode = self.integrity_mode_combo.currentData() self.history_range_widget.setVisible(mode == "history") self.window_range_widget.setVisible(mode == "window") def _update_sync_time_preview(self): """更新日常同步的时间范围预览（精确到秒）""" now = datetime.now() hours = self.sync_hours.value() overlap = self.sync_overlap.value() start_time = now - timedelta(hours=hours, seconds=overlap) start_str = start_time.strftime("%Y-%m-%d %H:%M:%S") end_str = now.strftime("%Y-%m-%d %H:%M:%S") self.sync_time_preview.setText(f"时间窗口: {start_str} ~ {end_str}") # ==================== 日常增量同步 ==================== def _run_daily_sync(self): """执行日常增量同步""" selected_tasks = self.sync_task_selector.get_selected_codes() if not selected_tasks: QMessageBox.warning(self, "提示", "请至少选择一个 ODS 任务") return hours = self.sync_hours.value() overlap = self.sync_overlap.value() include_dwd = self.sync_include_dwd.isChecked() auto_verify = self.sync_auto_verify.isChecked() now = datetime.now() start_time = now - timedelta(hours=hours, seconds=overlap) # 构建完整任务列表 all_tasks = selected_tasks.copy() # 添加 DWD 装载 if include_dwd: all_tasks.append("DWD_LOAD_FROM_ODS") # 添加数据校验 if auto_verify: all_tasks.append("DATA_INTEGRITY_CHECK") # 构建环境变量 env_vars = {} if auto_verify: env_vars["INTEGRITY_MODE"] = "window" env_vars["INTEGRITY_AUTO_BACKFILL"] = "1" config = TaskConfig( tasks=all_tasks, pipeline_flow="FULL", window_start=start_time.strftime("%Y-%m-%d %H:%M:%S"), window_end=now.strftime("%Y-%m-%d %H:%M:%S"), env_vars=env_vars, ) # 更新预览 cmd_str = self.cli_builder.build_command_string(config) self.cli_preview.setPlainText(cmd_str) # 构建任务描述 desc_parts = [f"增量同步 ({hours}h)"] if include_dwd: desc_parts.append("+ DWD") if auto_verify: desc_parts.append("+ 校验") task_desc = " ".join(desc_parts) # 发送信号添加到队列 self.add_to_queue.emit(config) self.log_message.emit(f"[GUI] 已添加到任务队列: {task_desc}") QMessageBox.information( self, "提示", f"已添加到任务队列\n\n" f"任务: {task_desc}\n" f"ODS 任务数: {len(selected_tasks)}\n" f"总任务数: {len(all_tasks)}\n\n" f"请切换到「任务管理」查看和执行" ) def _create_sync_schedule(self): """创建日常同步调度任务""" selected_tasks = self.sync_task_selector.get_selected_codes() if not selected_tasks: QMessageBox.warning(self, "提示", "请至少选择一个 ODS 任务") return hours = self.sync_hours.value() include_dwd = self.sync_include_dwd.isChecked() # 构建任务列表 all_tasks = selected_tasks.copy() if include_dwd: all_tasks.append("DWD_LOAD_FROM_ODS") task_config = { "pipeline_flow": "FULL", "lookback_hours": hours, "overlap_seconds": self.sync_overlap.value(), } # 发送信号创建调度任务 self.create_schedule.emit( f"日常同步 ({hours}h)", all_tasks, task_config ) self.log_message.emit(f"[GUI] 创建调度任务: 日常同步 ({hours}h)") # ==================== 全量重刷 ==================== def _run_full_reload(self): """执行全量重刷""" selected_tasks = self.reload_task_selector.get_selected_codes() if not selected_tasks: QMessageBox.warning(self, "提示", "请至少选择一个 ODS 任务") return start_dt = self.reload_start.dateTime().toPython() end_dt = self.reload_end.dateTime().toPython() if start_dt >= end_dt: QMessageBox.warning(self, "提示", "开始时间必须早于结束时间") return include_dwd = self.reload_include_dwd.isChecked() split_unit = self.reload_split_combo.currentData() # 构建完整任务列表 all_tasks = selected_tasks.copy() if include_dwd: all_tasks.append("DWD_LOAD_FROM_ODS") # 构建环境变量 env_vars = {} if split_unit and split_unit != "none": env_vars["WINDOW_SPLIT_UNIT"] = split_unit config = TaskConfig( tasks=all_tasks, pipeline_flow="FULL", window_start=start_dt.strftime("%Y-%m-%d %H:%M:%S"), window_end=end_dt.strftime("%Y-%m-%d %H:%M:%S"), env_vars=env_vars, ) # 更新预览 cmd_str = self.cli_builder.build_command_string(config) self.cli_preview.setPlainText(cmd_str) # 计算时间跨度 duration = end_dt - start_dt days = duration.days # 构建任务描述 desc = f"全量重刷 ({days}天)" if include_dwd: desc += " + DWD" # 发送信号添加到队列 self.add_to_queue.emit(config) self.log_message.emit(f"[GUI] 已添加到任务队列: {desc}") QMessageBox.information( self, "提示", f"已添加到任务队列\n\n" f"任务: {desc}\n" f"时间窗口: {start_dt.strftime('%Y-%m-%d %H:%M')} ~ {end_dt.strftime('%Y-%m-%d %H:%M')}\n" f"ODS 任务数: {len(selected_tasks)}\n" f"窗口切分: {split_unit if split_unit != 'none' else '不切分'}\n\n" f"请切换到「任务管理」查看和执行" ) # ==================== 数据校验 ==================== def _get_integrity_config(self) -> dict: """获取数据校验配置""" mode = self.integrity_mode_combo.currentData() config = { "pipeline_flow": "FULL", "integrity_mode": mode, "integrity_include_dimensions": self.integrity_include_dimensions.isChecked(), "integrity_compare_content": self.integrity_compare_content.isChecked(), "integrity_auto_backfill": self.integrity_auto_backfill.isChecked(), "integrity_backfill_mismatch": self.integrity_backfill_mismatch.isChecked(), "integrity_recheck": self.integrity_recheck.isChecked(), } if mode == "history": config["integrity_history_start"] = self.integrity_start_date.dateTime().toString("yyyy-MM-dd") config["integrity_history_end"] = self.integrity_end_date.dateTime().toString("yyyy-MM-dd") else: config["window_start"] = self.integrity_window_start.dateTime().toPython().strftime("%Y-%m-%d %H:%M:%S") config["window_end"] = self.integrity_window_end.dateTime().toPython().strftime("%Y-%m-%d %H:%M:%S") ods_tasks = self.integrity_ods_tasks.text().strip() if ods_tasks: config["integrity_ods_task_codes"] = ods_tasks return config def _run_integrity_check(self): """执行数据完整性校验""" mode = self.integrity_mode_combo.currentData() int_config = self._get_integrity_config() # 构建环境变量 env_vars = { "INTEGRITY_MODE": int_config.get("integrity_mode", "history"), "INTEGRITY_INCLUDE_DIMENSIONS": "1" if int_config.get("integrity_include_dimensions") else "0", "INTEGRITY_COMPARE_CONTENT": "1" if int_config.get("integrity_compare_content") else "0", "INTEGRITY_AUTO_BACKFILL": "1" if int_config.get("integrity_auto_backfill") else "0", "INTEGRITY_BACKFILL_MISMATCH": "1" if int_config.get("integrity_backfill_mismatch") else "0", "INTEGRITY_RECHECK_AFTER_BACKFILL": "1" if int_config.get("integrity_recheck") else "0", } window_start = None window_end = None if mode == "history": env_vars["INTEGRITY_HISTORY_START"] = int_config.get("integrity_history_start") env_vars["INTEGRITY_HISTORY_END"] = int_config.get("integrity_history_end") desc = f"数据校验 ({int_config.get('integrity_history_start')} ~ {int_config.get('integrity_history_end')})" else: window_start = int_config.get("window_start") window_end = int_config.get("window_end") desc = f"数据校验 ({window_start} ~ {window_end})" if int_config.get("integrity_ods_task_codes"): env_vars["INTEGRITY_ODS_TASK_CODES"] = int_config.get("integrity_ods_task_codes") # 构建描述 options = [] if int_config.get("integrity_include_dimensions"): options.append("含维度") if int_config.get("integrity_auto_backfill"): options.append("自动补齐") if options: desc += " [" + ", ".join(options) + "]" config = TaskConfig( tasks=["DATA_INTEGRITY_CHECK"], pipeline_flow="FULL", window_start=window_start, window_end=window_end, env_vars=env_vars, ) # 更新预览 cmd_str = self.cli_builder.build_command_string(config) env_preview = "\n".join(f"# {k}={v}" for k, v in env_vars.items()) self.cli_preview.setPlainText(f"{cmd_str}\n\n# 环境变量:\n{env_preview}") # 发送信号添加到队列 self.add_to_queue.emit(config) self.log_message.emit(f"[GUI] 已添加到任务队列: {desc}") QMessageBox.information(self, "提示", f"已添加到任务队列\n\n任务: {desc}\n\n请切换到「任务管理」查看和执行") def _create_integrity_schedule(self): """创建数据校验调度任务""" mode = self.integrity_mode_combo.currentData() task_config = self._get_integrity_config() if mode == "history": desc = f"数据校验 ({task_config.get('integrity_history_start')} ~ {task_config.get('integrity_history_end')})" else: desc = "数据校验 (窗口模式)" # 发送信号创建调度任务 self.create_schedule.emit( desc, ["DATA_INTEGRITY_CHECK"], task_config ) self.log_message.emit(f"[GUI] 创建调度任务: {desc}") # ==================== 设置持久化 ==================== def _load_settings(self): """从持久化存储加载设置""" try: # 加载日常同步设置 if hasattr(app_settings, 'sync_hours'): self.sync_hours.setValue(app_settings.sync_hours) if hasattr(app_settings, 'sync_overlap'): self.sync_overlap.setValue(app_settings.sync_overlap) if hasattr(app_settings, 'sync_include_dwd'): self.sync_include_dwd.setChecked(app_settings.sync_include_dwd) if hasattr(app_settings, 'sync_auto_verify'): self.sync_auto_verify.setChecked(app_settings.sync_auto_verify) # 恢复日常同步任务选择 if hasattr(app_settings, 'sync_selected_tasks'): saved_tasks = app_settings.sync_selected_tasks if saved_tasks: self.sync_task_selector.set_selected_codes(saved_tasks) # 加载数据校验设置 mode = getattr(app_settings, 'integrity_mode', 'history') mode_index = 0 if mode == "history" else 1 self.integrity_mode_combo.setCurrentIndex(mode_index) # 历史日期 if hasattr(app_settings, 'integrity_history_start') and app_settings.integrity_history_start: try: start_date = QDateTime.fromString(app_settings.integrity_history_start, "yyyy-MM-dd") if start_date.isValid(): self.integrity_start_date.setDateTime(start_date) except Exception: pass if hasattr(app_settings, 'integrity_history_end') and app_settings.integrity_history_end: try: end_date = QDateTime.fromString(app_settings.integrity_history_end, "yyyy-MM-dd") if end_date.isValid(): self.integrity_end_date.setDateTime(end_date) except Exception: pass if hasattr(app_settings, 'integrity_include_dimensions'): self.integrity_include_dimensions.setChecked(app_settings.integrity_include_dimensions) if hasattr(app_settings, 'integrity_compare_content'): self.integrity_compare_content.setChecked(app_settings.integrity_compare_content) if hasattr(app_settings, 'integrity_auto_backfill'): self.integrity_auto_backfill.setChecked(app_settings.integrity_auto_backfill) if hasattr(app_settings, 'integrity_ods_tasks'): self.integrity_ods_tasks.setText(app_settings.integrity_ods_tasks) # 加载高级设置 pipeline_flow = getattr(app_settings, 'advanced_pipeline_flow', 'FULL') flow_map = {"FULL": 0, "FETCH_ONLY": 1, "INGEST_ONLY": 2} self.pipeline_flow_combo.setCurrentIndex(flow_map.get(pipeline_flow, 0)) if hasattr(app_settings, 'advanced_dry_run'): self.dry_run_check.setChecked(app_settings.advanced_dry_run) if hasattr(app_settings, 'advanced_window_start'): self.window_start_edit.setText(app_settings.advanced_window_start) if hasattr(app_settings, 'advanced_window_end'): self.window_end_edit.setText(app_settings.advanced_window_end) if hasattr(app_settings, 'advanced_ingest_source'): self.ingest_source_edit.setText(app_settings.advanced_ingest_source) # 加载窗口切分设置 split_map = {"none": 0, "day": 1, "week": 2, "month": 3} if hasattr(app_settings, 'advanced_window_split'): self.window_split_combo.setCurrentIndex(split_map.get(app_settings.advanced_window_split, 0)) if hasattr(app_settings, 'advanced_window_compensation'): self.window_compensation_spin.setValue(app_settings.advanced_window_compensation) # 更新 UI 状态 self._on_integrity_mode_changed(mode_index) except Exception as e: print(f"加载设置失败: {e}") def _save_sync_settings(self): """保存日常同步设置""" try: app_settings.sync_hours = self.sync_hours.value() app_settings.sync_overlap = self.sync_overlap.value() app_settings.sync_include_dwd = self.sync_include_dwd.isChecked() app_settings.sync_auto_verify = self.sync_auto_verify.isChecked() app_settings.sync_selected_tasks = self.sync_task_selector.get_selected_codes() except Exception as e: print(f"保存日常同步设置失败: {e}") def _save_reload_settings(self): """保存全量重刷设置""" try: app_settings.reload_split = self.reload_split_combo.currentData() or "none" app_settings.reload_include_dwd = self.reload_include_dwd.isChecked() except Exception as e: print(f"保存全量重刷设置失败: {e}") def _save_integrity_settings(self): """保存数据校验设置""" try: mode = self.integrity_mode_combo.currentData() app_settings.integrity_mode = mode or "history" app_settings.integrity_history_start = self.integrity_start_date.dateTime().toString("yyyy-MM-dd") app_settings.integrity_history_end = self.integrity_end_date.dateTime().toString("yyyy-MM-dd") app_settings.integrity_include_dimensions = self.integrity_include_dimensions.isChecked() app_settings.integrity_compare_content = self.integrity_compare_content.isChecked() app_settings.integrity_auto_backfill = self.integrity_auto_backfill.isChecked() app_settings.integrity_ods_tasks = self.integrity_ods_tasks.text().strip() except Exception as e: print(f"保存数据校验设置失败: {e}") def _save_advanced_settings(self): """保存高级设置""" try: app_settings.advanced_pipeline_flow = self.pipeline_flow_combo.currentData() or "FULL" app_settings.advanced_dry_run = self.dry_run_check.isChecked() app_settings.advanced_window_start = self.window_start_edit.text().strip() app_settings.advanced_window_end = self.window_end_edit.text().strip() app_settings.advanced_ingest_source = self.ingest_source_edit.text().strip() app_settings.advanced_window_split = self.window_split_combo.currentData() or "none" app_settings.advanced_window_compensation = self.window_compensation_spin.value() except Exception as e: print(f"保存高级设置失败: {e}")