Files
Neo-ZQYY/apps/admin-web/src/pages/TaskConfig.tsx
Neo 254ccb1e77 feat: TaskSelector v2 全链路展示 + 同步检查 + MCP Server + 服务器 Git 排除
- admin-web: TaskSelector 重构为按域+层全链路展示,新增同步检查功能
- admin-web: TaskConfig 动态加载 Flow/处理模式定义,DWD 表过滤内嵌域面板
- admin-web: App hydrate 完成前显示 loading,避免误跳 /login
- backend: 新增 /tasks/sync-check 对比后端与 ETL 真实注册表
- backend: 新增 /tasks/flows 返回 Flow 和处理模式定义
- apps/mcp-server: 新增 MCP Server 模块(百炼 AI PostgreSQL 只读查询)
- scripts/server: 新增 setup-server-git.py + server-exclude.txt
- docs: 更新 LAUNCH-CHECKLIST 添加 Git 排除配置步骤
- pyproject.toml: workspace members 新增 mcp-server
2026-02-19 10:31:16 +08:00

613 lines
23 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* ETL 任务配置页面。
*
* 提供 Flow 选择、处理模式、时间窗口、高级选项等配置区域,
* 以及连接器/Store 选择、任务选择、DWD 表选择、CLI 命令预览和任务提交功能。
*/
import React, { useState, useEffect, useMemo } from "react";
import {
Card,
Radio,
Checkbox,
InputNumber,
DatePicker,
Button,
Space,
Typography,
Input,
message,
Row,
Col,
Badge,
Alert,
TreeSelect,
Tooltip,
Segmented,
Spin,
} from "antd";
import {
SendOutlined,
ThunderboltOutlined,
CodeOutlined,
SettingOutlined,
ClockCircleOutlined,
SyncOutlined,
ShopOutlined,
ApiOutlined,
} from "@ant-design/icons";
import { useNavigate } from "react-router-dom";
import TaskSelector from "../components/TaskSelector";
import { validateTaskConfig, fetchFlows } from "../api/tasks";
import type { FlowDef, ProcessingModeDef } from "../api/tasks";
import { submitToQueue, executeDirectly } from "../api/execution";
import { useAuthStore } from "../store/authStore";
import type { RadioChangeEvent } from "antd";
import type { Dayjs } from "dayjs";
import type { TaskConfig as TaskConfigType } from "../types";
const { Title, Text } = Typography;
const { TextArea } = Input;
/* ------------------------------------------------------------------ */
/* Flow / 处理模式 — 本地 fallbackAPI 不可用时兜底) */
/* ------------------------------------------------------------------ */
interface FlowEntry { name: string; layers: string[] }
const FALLBACK_FLOWS: Record<string, FlowEntry> = {
api_ods: { name: "API → ODS", layers: ["ODS"] },
api_ods_dwd: { name: "API → ODS → DWD", layers: ["ODS", "DWD"] },
api_full: { name: "API → ODS → DWD → DWS → INDEX", layers: ["ODS", "DWD", "DWS", "INDEX"] },
ods_dwd: { name: "ODS → DWD", layers: ["DWD"] },
dwd_dws: { name: "DWD → DWS汇总", layers: ["DWS"] },
dwd_dws_index: { name: "DWD → DWS → INDEX", layers: ["DWS", "INDEX"] },
dwd_index: { name: "DWD → INDEX", layers: ["INDEX"] },
};
interface ProcModeEntry { value: string; label: string; desc: string }
const FALLBACK_PROCESSING_MODES: ProcModeEntry[] = [
{ value: "increment_only", label: "仅增量", desc: "按游标增量抓取和装载" },
{ value: "verify_only", label: "校验并修复", desc: "对比源和目标,修复差异" },
{ value: "increment_verify", label: "增量+校验", desc: "先增量再校验" },
];
/** 将 API 返回的 FlowDef[] 转为 Record<id, FlowEntry> */
function apiFlowsToRecord(flows: FlowDef[]): Record<string, FlowEntry> {
const result: Record<string, FlowEntry> = {};
for (const f of flows) result[f.id] = { name: f.name, layers: f.layers };
return result;
}
/** 将 API 返回的 ProcessingModeDef[] 转为 ProcModeEntry[] */
function apiModesToEntries(modes: ProcessingModeDef[]): ProcModeEntry[] {
return modes.map((m) => ({ value: m.id, label: m.name, desc: m.description }));
}
/** 外部可用的 getFlowLayers使用 fallback组件内部用动态数据 */
export function getFlowLayers(flowId: string): string[] {
return FALLBACK_FLOWS[flowId]?.layers ?? [];
}
/* ------------------------------------------------------------------ */
/* 时间窗口 */
/* ------------------------------------------------------------------ */
type WindowMode = "lookback" | "custom";
const WINDOW_SPLIT_OPTIONS = [
{ value: 0, label: "不切分" },
{ value: 1, label: "1天" },
{ value: 10, label: "10天" },
{ value: 30, label: "30天" },
] as const;
/* ------------------------------------------------------------------ */
/* 连接器 → 门店 树形数据结构 */
/* ------------------------------------------------------------------ */
/** 连接器定义:每个连接器下挂载门店列表 */
interface ConnectorDef {
id: string;
label: string;
icon: React.ReactNode;
}
const CONNECTOR_DEFS: ConnectorDef[] = [
{ id: "feiqiu", label: "飞球", icon: <ApiOutlined /> },
];
/** 构建 TreeSelect 的 treeData连接器为父节点门店为子节点 */
function buildConnectorStoreTree(
connectors: ConnectorDef[],
siteId: number | null,
): { treeData: { title: React.ReactNode; value: string; key: string; children?: { title: React.ReactNode; value: string; key: string }[] }[]; allValues: string[] } {
const allValues: string[] = [];
const treeData = connectors.map((c) => {
// 每个连接器下挂载当前用户的门店(未来可扩展为多门店)
const stores = siteId
? [{ title: (<Space size={4}><ShopOutlined /><span> {siteId}</span></Space>), value: `${c.id}::${siteId}`, key: `${c.id}::${siteId}` }]
: [];
stores.forEach((s) => allValues.push(s.value));
return {
title: (<Space size={4}>{c.icon}<span>{c.label}</span></Space>),
value: c.id,
key: c.id,
children: stores,
};
});
return { treeData, allValues };
}
/** 从选中值中解析出 store_id 列表 */
function parseSelectedStoreIds(selected: string[]): number[] {
const ids: number[] = [];
for (const v of selected) {
// 格式: "connector::storeId"
const parts = v.split("::");
if (parts.length === 2) {
const num = Number(parts[1]);
if (!isNaN(num)) ids.push(num);
}
}
return ids;
}
/* ------------------------------------------------------------------ */
/* 页面组件 */
/* ------------------------------------------------------------------ */
const TaskConfig: React.FC = () => {
const navigate = useNavigate();
const user = useAuthStore((s) => s.user);
/* ---------- Flow / 处理模式 动态加载 ---------- */
const [flowDefs, setFlowDefs] = useState<Record<string, FlowEntry>>(FALLBACK_FLOWS);
const [procModes, setProcModes] = useState<ProcModeEntry[]>(FALLBACK_PROCESSING_MODES);
const [flowsLoading, setFlowsLoading] = useState(true);
useEffect(() => {
let cancelled = false;
fetchFlows()
.then(({ flows, processing_modes }) => {
if (cancelled) return;
if (flows.length > 0) setFlowDefs(apiFlowsToRecord(flows));
if (processing_modes.length > 0) setProcModes(apiModesToEntries(processing_modes));
})
.catch(() => { /* API 不可用,使用 fallback */ })
.finally(() => { if (!cancelled) setFlowsLoading(false); });
return () => { cancelled = true; };
}, []);
/* ---------- 连接器 & Store 树形选择 ---------- */
const { treeData: connectorTreeData, allValues: allConnectorStoreValues } = useMemo(
() => buildConnectorStoreTree(CONNECTOR_DEFS, user?.site_id ?? null),
[user?.site_id],
);
// 默认全选
const [selectedConnectorStores, setSelectedConnectorStores] = useState<string[]>([]);
// 初始化时默认全选
useEffect(() => {
if (selectedConnectorStores.length === 0 && allConnectorStoreValues.length > 0) {
setSelectedConnectorStores(allConnectorStoreValues);
}
}, [allConnectorStoreValues]); // eslint-disable-line react-hooks/exhaustive-deps
// 从选中值解析 store_id取第一个当前单门店场景
const selectedStoreIds = useMemo(() => parseSelectedStoreIds(selectedConnectorStores), [selectedConnectorStores]);
const effectiveStoreId = selectedStoreIds.length === 1 ? selectedStoreIds[0] : null;
/* ---------- Flow ---------- */
const [flow, setFlow] = useState<string>("api_ods_dwd");
/* ---------- 处理模式 ---------- */
const [processingMode, setProcessingMode] = useState<string>("increment_only");
const [fetchBeforeVerify, setFetchBeforeVerify] = useState(false);
/* ---------- 时间窗口 ---------- */
const [windowMode, setWindowMode] = useState<WindowMode>("lookback");
const [lookbackHours, setLookbackHours] = useState<number>(24);
const [overlapSeconds, setOverlapSeconds] = useState<number>(600);
const [windowStart, setWindowStart] = useState<Dayjs | null>(null);
const [windowEnd, setWindowEnd] = useState<Dayjs | null>(null);
const [windowSplitDays, setWindowSplitDays] = useState<number>(0);
/* ---------- 任务选择 ---------- */
const [selectedTasks, setSelectedTasks] = useState<string[]>([]);
const [selectedDwdTables, setSelectedDwdTables] = useState<string[]>([]);
/* ---------- 高级选项 ---------- */
const [dryRun, setDryRun] = useState(false);
const [forceFull, setForceFull] = useState(false);
const [useLocalJson, setUseLocalJson] = useState(false);
/* ---------- CLI 预览 ---------- */
const [cliCommand, setCliCommand] = useState<string>("");
const [cliEdited, setCliEdited] = useState(false);
const [cliLoading, setCliLoading] = useState(false);
/* ---------- 提交状态 ---------- */
const [submitting, setSubmitting] = useState(false);
/* ---------- 派生状态 ---------- */
const layers = flowDefs[flow]?.layers ?? [];
const showVerifyOption = processingMode === "verify_only";
/* ---------- 构建 TaskConfig 对象 ---------- */
const buildTaskConfig = (): TaskConfigType => {
/* layers 包含 DWD 时自动注入 DWD_LOAD_FROM_ODSUI 上由 DWD 表过滤区块隐含) */
const tasks = layers.includes("DWD") && !selectedTasks.includes("DWD_LOAD_FROM_ODS")
? [...selectedTasks, "DWD_LOAD_FROM_ODS"]
: selectedTasks;
return {
tasks,
pipeline: flow,
processing_mode: processingMode,
pipeline_flow: "FULL",
dry_run: dryRun,
window_mode: windowMode,
window_start: windowMode === "custom" && windowStart ? windowStart.format("YYYY-MM-DD") : null,
window_end: windowMode === "custom" && windowEnd ? windowEnd.format("YYYY-MM-DD") : null,
window_split: windowSplitDays > 0 ? "day" : null,
window_split_days: windowSplitDays > 0 ? windowSplitDays : null,
lookback_hours: lookbackHours,
overlap_seconds: overlapSeconds,
fetch_before_verify: fetchBeforeVerify,
skip_ods_when_fetch_before_verify: false,
ods_use_local_json: useLocalJson,
store_id: effectiveStoreId,
dwd_only_tables: selectedDwdTables.length > 0 ? selectedDwdTables : null,
force_full: forceFull,
extra_args: {},
};
};
/* ---------- 自动刷新 CLI 预览 ---------- */
const refreshCli = async () => {
setCliLoading(true);
try {
const { command } = await validateTaskConfig(buildTaskConfig());
setCliCommand(command);
setCliEdited(false);
} catch {
// 静默失败,保留上次命令
} finally {
setCliLoading(false);
}
};
// 配置变化时自动刷新 CLI防抖
useEffect(() => {
if (cliEdited) return; // 用户手动编辑过则不自动刷新
const timer = setTimeout(refreshCli, 500);
return () => clearTimeout(timer);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [flow, processingMode, fetchBeforeVerify, windowMode, lookbackHours, overlapSeconds,
windowStart, windowEnd, windowSplitDays, selectedTasks, selectedDwdTables,
dryRun, forceFull, useLocalJson, selectedConnectorStores]);
/* ---------- 事件处理 ---------- */
const handleFlowChange = (e: RadioChangeEvent) => setFlow(e.target.value);
const handleSubmitToQueue = async () => {
setSubmitting(true);
try {
await submitToQueue(buildTaskConfig());
message.success("已提交到执行队列");
navigate("/task-manager");
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : "提交失败";
message.error(`提交到队列失败:${msg}`);
} finally {
setSubmitting(false);
}
};
const handleExecuteDirectly = async () => {
setSubmitting(true);
try {
await executeDirectly(buildTaskConfig());
message.success("任务已开始执行");
navigate("/task-manager");
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : "执行失败";
message.error(`直接执行失败:${msg}`);
} finally {
setSubmitting(false);
}
};
/* ---------- 样式常量 ---------- */
const cardStyle = { marginBottom: 12 };
const sectionTitleStyle: React.CSSProperties = {
fontSize: 13, fontWeight: 500, color: "#666", marginBottom: 8, display: "block",
};
return (
<div style={{ maxWidth: 960, margin: "0 auto" }}>
{/* ---- 页面标题 ---- */}
<div style={{ marginBottom: 16, display: "flex", alignItems: "center", justifyContent: "space-between" }}>
<Title level={4} style={{ margin: 0 }}>
<SettingOutlined style={{ marginRight: 8 }} />
</Title>
<Space>
<Badge count={selectedTasks.length} size="small" offset={[-4, 0]}>
<Text type="secondary"></Text>
</Badge>
</Space>
</div>
{/* ---- 第一行:连接器/门店 + Flow ---- */}
<Row gutter={12}>
<Col span={8}>
<Card size="small" title={<Space size={4}><ApiOutlined /> / </Space>} style={cardStyle}>
<TreeSelect
treeData={connectorTreeData}
value={selectedConnectorStores}
onChange={setSelectedConnectorStores}
treeCheckable
treeDefaultExpandAll
showCheckedStrategy={TreeSelect.SHOW_CHILD}
placeholder="选择连接器和门店"
style={{ width: "100%" }}
maxTagCount={3}
maxTagPlaceholder={(omitted) => `+${omitted.length}`}
treeCheckStrictly={false}
/>
<Text type="secondary" style={{ fontSize: 11, marginTop: 6, display: "block" }}>
{selectedStoreIds.length === 0
? "未选择门店,将使用 JWT 默认值"
: `已选 ${selectedStoreIds.length} 个门店`}
</Text>
</Card>
</Col>
<Col span={16}>
<Card size="small" title={flowsLoading ? <Space size={4}> (Flow) <Spin size="small" /></Space> : "执行流程 (Flow)"} style={cardStyle}>
<Radio.Group value={flow} onChange={handleFlowChange} style={{ width: "100%" }}>
<Row gutter={[0, 4]}>
{Object.entries(flowDefs).map(([id, def]) => (
<Col span={12} key={id}>
<Tooltip title={def.name}>
<Radio value={id}>
<Text strong style={{ fontSize: 12 }}>{id}</Text>
</Radio>
</Tooltip>
</Col>
))}
</Row>
</Radio.Group>
<div style={{ marginTop: 6, padding: "4px 8px", background: "#f6f8fa", borderRadius: 4 }}>
<Text type="secondary" style={{ fontSize: 12 }}>
{layers.join(" → ") || "—"}
</Text>
</div>
</Card>
</Col>
</Row>
{/* ---- 第二行:处理模式 + 时间窗口 ---- */}
<Row gutter={12}>
<Col span={8}>
<Card size="small" title="处理模式" style={cardStyle}>
<Radio.Group
value={processingMode}
onChange={(e) => {
setProcessingMode(e.target.value);
if (e.target.value === "increment_only") setFetchBeforeVerify(false);
}}
>
<Space direction="vertical" style={{ width: "100%" }}>
{procModes.map((m) => (
<Radio key={m.value} value={m.value}>
<Text strong>{m.label}</Text>
<br />
<Text type="secondary" style={{ fontSize: 12 }}>{m.desc}</Text>
</Radio>
))}
</Space>
</Radio.Group>
{showVerifyOption && (
<Checkbox
checked={fetchBeforeVerify}
onChange={(e) => setFetchBeforeVerify(e.target.checked)}
style={{ marginTop: 8 }}
>
API
</Checkbox>
)}
</Card>
</Col>
<Col span={16}>
<Card
size="small"
title={<><ClockCircleOutlined style={{ marginRight: 6 }} /></>}
style={cardStyle}
>
<Row gutter={16}>
<Col span={24}>
<Segmented
value={windowMode}
onChange={(v) => setWindowMode(v as WindowMode)}
options={[
{ value: "lookback", label: "回溯模式" },
{ value: "custom", label: "自定义范围" },
]}
style={{ marginBottom: 12 }}
/>
</Col>
</Row>
{windowMode === "lookback" ? (
<Row gutter={16}>
<Col span={12}>
<Text style={sectionTitleStyle}></Text>
<InputNumber
min={1} max={720} value={lookbackHours}
onChange={(v) => setLookbackHours(v ?? 24)}
style={{ width: "100%" }}
addonAfter="小时"
/>
</Col>
<Col span={12}>
<Text style={sectionTitleStyle}></Text>
<InputNumber
min={0} max={7200} value={overlapSeconds}
onChange={(v) => setOverlapSeconds(v ?? 600)}
style={{ width: "100%" }}
addonAfter="秒"
/>
</Col>
</Row>
) : (
<Row gutter={16}>
<Col span={12}>
<Text style={sectionTitleStyle}></Text>
<DatePicker
value={windowStart} onChange={setWindowStart}
placeholder="选择开始日期" style={{ width: "100%" }}
/>
</Col>
<Col span={12}>
<Text style={sectionTitleStyle}></Text>
<DatePicker
value={windowEnd} onChange={setWindowEnd}
placeholder="选择结束日期" style={{ width: "100%" }}
status={windowStart && windowEnd && windowEnd.isBefore(windowStart) ? "error" : undefined}
/>
</Col>
</Row>
)}
<div style={{ marginTop: 12 }}>
<Text style={sectionTitleStyle}></Text>
<Radio.Group
value={windowSplitDays}
onChange={(e) => setWindowSplitDays(e.target.value)}
>
{WINDOW_SPLIT_OPTIONS.map((opt) => (
<Radio.Button key={opt.value} value={opt.value}>{opt.label}</Radio.Button>
))}
</Radio.Group>
</div>
</Card>
</Col>
</Row>
{/* ---- 高级选项(带描述) ---- */}
<Card size="small" title="高级选项" style={cardStyle}>
<Row gutter={[24, 8]}>
<Col span={12}>
<Checkbox checked={dryRun} onChange={(e) => setDryRun(e.target.checked)}>
<Text strong>dry-run</Text>
</Checkbox>
<div style={{ marginLeft: 24 }}>
<Text type="secondary" style={{ fontSize: 12 }}></Text>
</div>
</Col>
<Col span={12}>
<Checkbox checked={forceFull} onChange={(e) => setForceFull(e.target.checked)}>
<Text strong>force-full</Text>
</Checkbox>
<div style={{ marginLeft: 24 }}>
<Text type="secondary" style={{ fontSize: 12 }}> hash </Text>
</div>
</Col>
<Col span={12}>
<Checkbox checked={useLocalJson} onChange={(e) => setUseLocalJson(e.target.checked)}>
<Text strong> JSON</Text>
</Checkbox>
<div style={{ marginLeft: 24 }}>
<Text type="secondary" style={{ fontSize: 12 }}>线 JSON --data-source offline</Text>
</div>
</Col>
</Row>
</Card>
{/* ---- 任务选择(含 DWD 表过滤) ---- */}
<Card size="small" title="任务选择" style={cardStyle}>
<TaskSelector
layers={layers}
selectedTasks={selectedTasks}
onTasksChange={setSelectedTasks}
selectedDwdTables={selectedDwdTables}
onDwdTablesChange={setSelectedDwdTables}
/>
</Card>
{/* ---- CLI 命令预览(内嵌可编辑) ---- */}
<Card
size="small"
title={
<Space>
<CodeOutlined />
<span>CLI </span>
{cliEdited && <Text type="warning" style={{ fontSize: 12 }}></Text>}
</Space>
}
extra={
<Button
size="small"
icon={<SyncOutlined spin={cliLoading} />}
onClick={() => { setCliEdited(false); refreshCli(); }}
>
</Button>
}
style={cardStyle}
>
<TextArea
value={cliCommand}
onChange={(e) => { setCliCommand(e.target.value); setCliEdited(true); }}
autoSize={{ minRows: 2, maxRows: 6 }}
style={{
fontFamily: "'Cascadia Code', 'Fira Code', Consolas, monospace",
fontSize: 13,
background: "#1e1e1e",
color: "#d4d4d4",
border: "none",
borderRadius: 4,
}}
placeholder="配置变更后自动生成 CLI 命令..."
/>
{cliEdited && (
<Alert
type="info"
showIcon
message="已手动编辑命令,配置变更不会自动覆盖。点击「重新生成」恢复自动模式。"
style={{ marginTop: 8 }}
banner
/>
)}
</Card>
{/* ---- 操作按钮 ---- */}
<Card size="small" style={{ marginBottom: 24 }}>
<Space size="middle">
<Button
type="primary"
size="large"
icon={<SendOutlined />}
loading={submitting}
onClick={handleSubmitToQueue}
>
</Button>
<Button
size="large"
icon={<ThunderboltOutlined />}
loading={submitting}
onClick={handleExecuteDirectly}
>
</Button>
</Space>
</Card>
</div>
);
};
export default TaskConfig;