/** * ETL 任务配置页面。 * * 提供 Flow 选择、处理模式、时间窗口、高级选项等配置区域, * 以及连接器/Store 选择、任务选择、DWD 表选择、CLI 命令预览和任务提交功能。 */ import React, { useState, useEffect, useMemo } from "react"; import { Card, Radio, Checkbox, InputNumber, DatePicker, Button, Space, Typography, Input, message, Row, Col, Badge, Alert, TreeSelect, Tooltip, Segmented, Spin, } from "antd"; import { SendOutlined, ThunderboltOutlined, CodeOutlined, SettingOutlined, ClockCircleOutlined, SyncOutlined, ShopOutlined, ApiOutlined, } from "@ant-design/icons"; import { useNavigate } from "react-router-dom"; import TaskSelector from "../components/TaskSelector"; import { validateTaskConfig, fetchFlows } from "../api/tasks"; import type { FlowDef, ProcessingModeDef } from "../api/tasks"; import { submitToQueue, executeDirectly } from "../api/execution"; import { useAuthStore } from "../store/authStore"; import type { RadioChangeEvent } from "antd"; import type { Dayjs } from "dayjs"; import type { TaskConfig as TaskConfigType } from "../types"; const { Title, Text } = Typography; const { TextArea } = Input; /* ------------------------------------------------------------------ */ /* Flow / 处理模式 — 本地 fallback(API 不可用时兜底) */ /* ------------------------------------------------------------------ */ interface FlowEntry { name: string; layers: string[] } const FALLBACK_FLOWS: Record = { api_ods: { name: "API → ODS", layers: ["ODS"] }, api_ods_dwd: { name: "API → ODS → DWD", layers: ["ODS", "DWD"] }, api_full: { name: "API → ODS → DWD → DWS → INDEX", layers: ["ODS", "DWD", "DWS", "INDEX"] }, ods_dwd: { name: "ODS → DWD", layers: ["DWD"] }, dwd_dws: { name: "DWD → DWS汇总", layers: ["DWS"] }, dwd_dws_index: { name: "DWD → DWS → INDEX", layers: ["DWS", "INDEX"] }, dwd_index: { name: "DWD → INDEX", layers: ["INDEX"] }, }; interface ProcModeEntry { value: string; label: string; desc: string } const FALLBACK_PROCESSING_MODES: ProcModeEntry[] = [ { value: "increment_only", label: "仅增量", desc: "按游标增量抓取和装载" }, { value: "verify_only", label: "校验并修复", desc: "对比源和目标,修复差异" }, { value: "increment_verify", label: "增量+校验", desc: "先增量再校验" }, ]; /** 将 API 返回的 FlowDef[] 转为 Record */ function apiFlowsToRecord(flows: FlowDef[]): Record { const result: Record = {}; for (const f of flows) result[f.id] = { name: f.name, layers: f.layers }; return result; } /** 将 API 返回的 ProcessingModeDef[] 转为 ProcModeEntry[] */ function apiModesToEntries(modes: ProcessingModeDef[]): ProcModeEntry[] { return modes.map((m) => ({ value: m.id, label: m.name, desc: m.description })); } /** 外部可用的 getFlowLayers(使用 fallback,组件内部用动态数据) */ export function getFlowLayers(flowId: string): string[] { return FALLBACK_FLOWS[flowId]?.layers ?? []; } /* ------------------------------------------------------------------ */ /* 时间窗口 */ /* ------------------------------------------------------------------ */ type WindowMode = "lookback" | "custom"; const WINDOW_SPLIT_OPTIONS = [ { value: 0, label: "不切分" }, { value: 1, label: "1天" }, { value: 10, label: "10天" }, { value: 30, label: "30天" }, ] as const; /* ------------------------------------------------------------------ */ /* 连接器 → 门店 树形数据结构 */ /* ------------------------------------------------------------------ */ /** 连接器定义:每个连接器下挂载门店列表 */ interface ConnectorDef { id: string; label: string; icon: React.ReactNode; } const CONNECTOR_DEFS: ConnectorDef[] = [ { id: "feiqiu", label: "飞球", icon: }, ]; /** 构建 TreeSelect 的 treeData,连接器为父节点,门店为子节点 */ function buildConnectorStoreTree( connectors: ConnectorDef[], siteId: number | null, ): { treeData: { title: React.ReactNode; value: string; key: string; children?: { title: React.ReactNode; value: string; key: string }[] }[]; allValues: string[] } { const allValues: string[] = []; const treeData = connectors.map((c) => { // 每个连接器下挂载当前用户的门店(未来可扩展为多门店) const stores = siteId ? [{ title: (门店 {siteId}), value: `${c.id}::${siteId}`, key: `${c.id}::${siteId}` }] : []; stores.forEach((s) => allValues.push(s.value)); return { title: ({c.icon}{c.label}), value: c.id, key: c.id, children: stores, }; }); return { treeData, allValues }; } /** 从选中值中解析出 store_id 列表 */ function parseSelectedStoreIds(selected: string[]): number[] { const ids: number[] = []; for (const v of selected) { // 格式: "connector::storeId" const parts = v.split("::"); if (parts.length === 2) { const num = Number(parts[1]); if (!isNaN(num)) ids.push(num); } } return ids; } /* ------------------------------------------------------------------ */ /* 页面组件 */ /* ------------------------------------------------------------------ */ const TaskConfig: React.FC = () => { const navigate = useNavigate(); const user = useAuthStore((s) => s.user); /* ---------- Flow / 处理模式 动态加载 ---------- */ const [flowDefs, setFlowDefs] = useState>(FALLBACK_FLOWS); const [procModes, setProcModes] = useState(FALLBACK_PROCESSING_MODES); const [flowsLoading, setFlowsLoading] = useState(true); useEffect(() => { let cancelled = false; fetchFlows() .then(({ flows, processing_modes }) => { if (cancelled) return; if (flows.length > 0) setFlowDefs(apiFlowsToRecord(flows)); if (processing_modes.length > 0) setProcModes(apiModesToEntries(processing_modes)); }) .catch(() => { /* API 不可用,使用 fallback */ }) .finally(() => { if (!cancelled) setFlowsLoading(false); }); return () => { cancelled = true; }; }, []); /* ---------- 连接器 & Store 树形选择 ---------- */ const { treeData: connectorTreeData, allValues: allConnectorStoreValues } = useMemo( () => buildConnectorStoreTree(CONNECTOR_DEFS, user?.site_id ?? null), [user?.site_id], ); // 默认全选 const [selectedConnectorStores, setSelectedConnectorStores] = useState([]); // 初始化时默认全选 useEffect(() => { if (selectedConnectorStores.length === 0 && allConnectorStoreValues.length > 0) { setSelectedConnectorStores(allConnectorStoreValues); } }, [allConnectorStoreValues]); // eslint-disable-line react-hooks/exhaustive-deps // 从选中值解析 store_id(取第一个,当前单门店场景) const selectedStoreIds = useMemo(() => parseSelectedStoreIds(selectedConnectorStores), [selectedConnectorStores]); const effectiveStoreId = selectedStoreIds.length === 1 ? selectedStoreIds[0] : null; /* ---------- Flow ---------- */ const [flow, setFlow] = useState("api_ods_dwd"); /* ---------- 处理模式 ---------- */ const [processingMode, setProcessingMode] = useState("increment_only"); const [fetchBeforeVerify, setFetchBeforeVerify] = useState(false); /* ---------- 时间窗口 ---------- */ const [windowMode, setWindowMode] = useState("lookback"); const [lookbackHours, setLookbackHours] = useState(24); const [overlapSeconds, setOverlapSeconds] = useState(600); const [windowStart, setWindowStart] = useState(null); const [windowEnd, setWindowEnd] = useState(null); const [windowSplitDays, setWindowSplitDays] = useState(0); /* ---------- 任务选择 ---------- */ const [selectedTasks, setSelectedTasks] = useState([]); const [selectedDwdTables, setSelectedDwdTables] = useState([]); /* ---------- 高级选项 ---------- */ const [dryRun, setDryRun] = useState(false); const [forceFull, setForceFull] = useState(false); const [useLocalJson, setUseLocalJson] = useState(false); /* ---------- CLI 预览 ---------- */ const [cliCommand, setCliCommand] = useState(""); const [cliEdited, setCliEdited] = useState(false); const [cliLoading, setCliLoading] = useState(false); /* ---------- 提交状态 ---------- */ const [submitting, setSubmitting] = useState(false); /* ---------- 派生状态 ---------- */ const layers = flowDefs[flow]?.layers ?? []; const showVerifyOption = processingMode === "verify_only"; /* ---------- 构建 TaskConfig 对象 ---------- */ const buildTaskConfig = (): TaskConfigType => { /* layers 包含 DWD 时自动注入 DWD_LOAD_FROM_ODS(UI 上由 DWD 表过滤区块隐含) */ const tasks = layers.includes("DWD") && !selectedTasks.includes("DWD_LOAD_FROM_ODS") ? [...selectedTasks, "DWD_LOAD_FROM_ODS"] : selectedTasks; return { tasks, pipeline: flow, processing_mode: processingMode, pipeline_flow: "FULL", dry_run: dryRun, window_mode: windowMode, window_start: windowMode === "custom" && windowStart ? windowStart.format("YYYY-MM-DD") : null, window_end: windowMode === "custom" && windowEnd ? windowEnd.format("YYYY-MM-DD") : null, window_split: windowSplitDays > 0 ? "day" : null, window_split_days: windowSplitDays > 0 ? windowSplitDays : null, lookback_hours: lookbackHours, overlap_seconds: overlapSeconds, fetch_before_verify: fetchBeforeVerify, skip_ods_when_fetch_before_verify: false, ods_use_local_json: useLocalJson, store_id: effectiveStoreId, dwd_only_tables: selectedDwdTables.length > 0 ? selectedDwdTables : null, force_full: forceFull, extra_args: {}, }; }; /* ---------- 自动刷新 CLI 预览 ---------- */ const refreshCli = async () => { setCliLoading(true); try { const { command } = await validateTaskConfig(buildTaskConfig()); setCliCommand(command); setCliEdited(false); } catch { // 静默失败,保留上次命令 } finally { setCliLoading(false); } }; // 配置变化时自动刷新 CLI(防抖) useEffect(() => { if (cliEdited) return; // 用户手动编辑过则不自动刷新 const timer = setTimeout(refreshCli, 500); return () => clearTimeout(timer); // eslint-disable-next-line react-hooks/exhaustive-deps }, [flow, processingMode, fetchBeforeVerify, windowMode, lookbackHours, overlapSeconds, windowStart, windowEnd, windowSplitDays, selectedTasks, selectedDwdTables, dryRun, forceFull, useLocalJson, selectedConnectorStores]); /* ---------- 事件处理 ---------- */ const handleFlowChange = (e: RadioChangeEvent) => setFlow(e.target.value); const handleSubmitToQueue = async () => { setSubmitting(true); try { await submitToQueue(buildTaskConfig()); message.success("已提交到执行队列"); navigate("/task-manager"); } catch (err: unknown) { const msg = err instanceof Error ? err.message : "提交失败"; message.error(`提交到队列失败:${msg}`); } finally { setSubmitting(false); } }; const handleExecuteDirectly = async () => { setSubmitting(true); try { await executeDirectly(buildTaskConfig()); message.success("任务已开始执行"); navigate("/task-manager"); } catch (err: unknown) { const msg = err instanceof Error ? err.message : "执行失败"; message.error(`直接执行失败:${msg}`); } finally { setSubmitting(false); } }; /* ---------- 样式常量 ---------- */ const cardStyle = { marginBottom: 12 }; const sectionTitleStyle: React.CSSProperties = { fontSize: 13, fontWeight: 500, color: "#666", marginBottom: 8, display: "block", }; return (
{/* ---- 页面标题 ---- */}
<SettingOutlined style={{ marginRight: 8 }} /> 任务配置 已选任务
{/* ---- 第一行:连接器/门店 + Flow ---- */} 连接器 / 门店} style={cardStyle}> `+${omitted.length} 项`} treeCheckStrictly={false} /> {selectedStoreIds.length === 0 ? "未选择门店,将使用 JWT 默认值" : `已选 ${selectedStoreIds.length} 个门店`} 执行流程 (Flow) : "执行流程 (Flow)"} style={cardStyle}> {Object.entries(flowDefs).map(([id, def]) => ( {id} ))}
{layers.join(" → ") || "—"}
{/* ---- 第二行:处理模式 + 时间窗口 ---- */} { setProcessingMode(e.target.value); if (e.target.value === "increment_only") setFetchBeforeVerify(false); }} > {procModes.map((m) => ( {m.label}
{m.desc}
))}
{showVerifyOption && ( setFetchBeforeVerify(e.target.checked)} style={{ marginTop: 8 }} > 校验前从 API 获取 )}
时间窗口} style={cardStyle} > setWindowMode(v as WindowMode)} options={[ { value: "lookback", label: "回溯模式" }, { value: "custom", label: "自定义范围" }, ]} style={{ marginBottom: 12 }} /> {windowMode === "lookback" ? ( 回溯小时数 setLookbackHours(v ?? 24)} style={{ width: "100%" }} addonAfter="小时" /> 冗余秒数 setOverlapSeconds(v ?? 600)} style={{ width: "100%" }} addonAfter="秒" /> ) : ( 开始日期 结束日期 )}
窗口切分 setWindowSplitDays(e.target.value)} > {WINDOW_SPLIT_OPTIONS.map((opt) => ( {opt.label} ))}
{/* ---- 高级选项(带描述) ---- */} setDryRun(e.target.checked)}> dry-run
模拟执行,走完整流程但不写入数据库
setForceFull(e.target.checked)}> force-full
强制全量,跳过 hash 去重和变更对比
setUseLocalJson(e.target.checked)}> 本地 JSON
离线模式,从本地 JSON 回放(等同 --data-source offline)
{/* ---- 任务选择(含 DWD 表过滤) ---- */} {/* ---- CLI 命令预览(内嵌可编辑) ---- */} CLI 命令预览 {cliEdited && (已手动编辑)} } extra={ } style={cardStyle} >