feat: TaskSelector v2 全链路展示 + 同步检查 + MCP Server + 服务器 Git 排除
- admin-web: TaskSelector 重构为按域+层全链路展示，新增同步检查功能 - admin-web: TaskConfig 动态加载 Flow/处理模式定义，DWD 表过滤内嵌域面板 - admin-web: App hydrate 完成前显示 loading，避免误跳 /login - backend: 新增 /tasks/sync-check 对比后端与 ETL 真实注册表 - backend: 新增 /tasks/flows 返回 Flow 和处理模式定义 - apps/mcp-server: 新增 MCP Server 模块（百炼 AI PostgreSQL 只读查询） - scripts/server: 新增 setup-server-git.py + server-exclude.txt - docs: 更新 LAUNCH-CHECKLIST 添加 Git 排除配置步骤 - pyproject.toml: workspace members 新增 mcp-server
This commit is contained in:
@@ -24,6 +24,7 @@ import {
|
||||
TreeSelect,
|
||||
Tooltip,
|
||||
Segmented,
|
||||
Spin,
|
||||
} from "antd";
|
||||
import {
|
||||
SendOutlined,
|
||||
@@ -37,7 +38,8 @@ import {
|
||||
} from "@ant-design/icons";
|
||||
import { useNavigate } from "react-router-dom";
|
||||
import TaskSelector from "../components/TaskSelector";
|
||||
import { validateTaskConfig } from "../api/tasks";
|
||||
import { validateTaskConfig, fetchFlows } from "../api/tasks";
|
||||
import type { FlowDef, ProcessingModeDef } from "../api/tasks";
|
||||
import { submitToQueue, executeDirectly } from "../api/execution";
|
||||
import { useAuthStore } from "../store/authStore";
|
||||
import type { RadioChangeEvent } from "antd";
|
||||
@@ -48,32 +50,45 @@ const { Title, Text } = Typography;
|
||||
const { TextArea } = Input;
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Flow 定义 */
|
||||
/* Flow / 处理模式 — 本地 fallback（API 不可用时兜底） */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const FLOW_DEFINITIONS: Record<string, { name: string; layers: string[]; desc: string }> = {
|
||||
api_ods: { name: "API → ODS", layers: ["ODS"], desc: "仅抓取原始数据" },
|
||||
api_ods_dwd: { name: "API → ODS → DWD", layers: ["ODS", "DWD"], desc: "抓取并清洗装载" },
|
||||
api_full: { name: "API → ODS → DWD → DWS → INDEX", layers: ["ODS", "DWD", "DWS", "INDEX"], desc: "全链路执行" },
|
||||
ods_dwd: { name: "ODS → DWD", layers: ["DWD"], desc: "仅清洗装载" },
|
||||
dwd_dws: { name: "DWD → DWS汇总", layers: ["DWS"], desc: "仅汇总计算" },
|
||||
dwd_dws_index: { name: "DWD → DWS → INDEX", layers: ["DWS", "INDEX"], desc: "汇总+指数" },
|
||||
dwd_index: { name: "DWD → INDEX", layers: ["INDEX"], desc: "仅指数计算" },
|
||||
interface FlowEntry { name: string; layers: string[] }
|
||||
|
||||
const FALLBACK_FLOWS: Record<string, FlowEntry> = {
|
||||
api_ods: { name: "API → ODS", layers: ["ODS"] },
|
||||
api_ods_dwd: { name: "API → ODS → DWD", layers: ["ODS", "DWD"] },
|
||||
api_full: { name: "API → ODS → DWD → DWS → INDEX", layers: ["ODS", "DWD", "DWS", "INDEX"] },
|
||||
ods_dwd: { name: "ODS → DWD", layers: ["DWD"] },
|
||||
dwd_dws: { name: "DWD → DWS汇总", layers: ["DWS"] },
|
||||
dwd_dws_index: { name: "DWD → DWS → INDEX", layers: ["DWS", "INDEX"] },
|
||||
dwd_index: { name: "DWD → INDEX", layers: ["INDEX"] },
|
||||
};
|
||||
|
||||
export function getFlowLayers(flowId: string): string[] {
|
||||
return FLOW_DEFINITIONS[flowId]?.layers ?? [];
|
||||
}
|
||||
interface ProcModeEntry { value: string; label: string; desc: string }
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* 处理模式 */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const PROCESSING_MODES = [
|
||||
const FALLBACK_PROCESSING_MODES: ProcModeEntry[] = [
|
||||
{ value: "increment_only", label: "仅增量", desc: "按游标增量抓取和装载" },
|
||||
{ value: "verify_only", label: "校验并修复", desc: "对比源和目标，修复差异" },
|
||||
{ value: "increment_verify", label: "增量+校验", desc: "先增量再校验" },
|
||||
] as const;
|
||||
];
|
||||
|
||||
/** 将 API 返回的 FlowDef[] 转为 Record<id, FlowEntry> */
|
||||
function apiFlowsToRecord(flows: FlowDef[]): Record<string, FlowEntry> {
|
||||
const result: Record<string, FlowEntry> = {};
|
||||
for (const f of flows) result[f.id] = { name: f.name, layers: f.layers };
|
||||
return result;
|
||||
}
|
||||
|
||||
/** 将 API 返回的 ProcessingModeDef[] 转为 ProcModeEntry[] */
|
||||
function apiModesToEntries(modes: ProcessingModeDef[]): ProcModeEntry[] {
|
||||
return modes.map((m) => ({ value: m.id, label: m.name, desc: m.description }));
|
||||
}
|
||||
|
||||
/** 外部可用的 getFlowLayers（使用 fallback，组件内部用动态数据） */
|
||||
export function getFlowLayers(flowId: string): string[] {
|
||||
return FALLBACK_FLOWS[flowId]?.layers ?? [];
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* 时间窗口 */
|
||||
@@ -147,6 +162,24 @@ const TaskConfig: React.FC = () => {
|
||||
const navigate = useNavigate();
|
||||
const user = useAuthStore((s) => s.user);
|
||||
|
||||
/* ---------- Flow / 处理模式 动态加载 ---------- */
|
||||
const [flowDefs, setFlowDefs] = useState<Record<string, FlowEntry>>(FALLBACK_FLOWS);
|
||||
const [procModes, setProcModes] = useState<ProcModeEntry[]>(FALLBACK_PROCESSING_MODES);
|
||||
const [flowsLoading, setFlowsLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
fetchFlows()
|
||||
.then(({ flows, processing_modes }) => {
|
||||
if (cancelled) return;
|
||||
if (flows.length > 0) setFlowDefs(apiFlowsToRecord(flows));
|
||||
if (processing_modes.length > 0) setProcModes(apiModesToEntries(processing_modes));
|
||||
})
|
||||
.catch(() => { /* API 不可用，使用 fallback */ })
|
||||
.finally(() => { if (!cancelled) setFlowsLoading(false); });
|
||||
return () => { cancelled = true; };
|
||||
}, []);
|
||||
|
||||
/* ---------- 连接器 & Store 树形选择 ---------- */
|
||||
const { treeData: connectorTreeData, allValues: allConnectorStoreValues } = useMemo(
|
||||
() => buildConnectorStoreTree(CONNECTOR_DEFS, user?.site_id ?? null),
|
||||
@@ -199,12 +232,17 @@ const TaskConfig: React.FC = () => {
|
||||
const [submitting, setSubmitting] = useState(false);
|
||||
|
||||
/* ---------- 派生状态 ---------- */
|
||||
const layers = getFlowLayers(flow);
|
||||
const layers = flowDefs[flow]?.layers ?? [];
|
||||
const showVerifyOption = processingMode === "verify_only";
|
||||
|
||||
/* ---------- 构建 TaskConfig 对象 ---------- */
|
||||
const buildTaskConfig = (): TaskConfigType => ({
|
||||
tasks: selectedTasks,
|
||||
const buildTaskConfig = (): TaskConfigType => {
|
||||
/* layers 包含 DWD 时自动注入 DWD_LOAD_FROM_ODS（UI 上由 DWD 表过滤区块隐含） */
|
||||
const tasks = layers.includes("DWD") && !selectedTasks.includes("DWD_LOAD_FROM_ODS")
|
||||
? [...selectedTasks, "DWD_LOAD_FROM_ODS"]
|
||||
: selectedTasks;
|
||||
return {
|
||||
tasks,
|
||||
pipeline: flow,
|
||||
processing_mode: processingMode,
|
||||
pipeline_flow: "FULL",
|
||||
@@ -223,7 +261,8 @@ const TaskConfig: React.FC = () => {
|
||||
dwd_only_tables: selectedDwdTables.length > 0 ? selectedDwdTables : null,
|
||||
force_full: forceFull,
|
||||
extra_args: {},
|
||||
});
|
||||
};
|
||||
};
|
||||
|
||||
/* ---------- 自动刷新 CLI 预览 ---------- */
|
||||
const refreshCli = async () => {
|
||||
@@ -326,12 +365,12 @@ const TaskConfig: React.FC = () => {
|
||||
</Card>
|
||||
</Col>
|
||||
<Col span={16}>
|
||||
<Card size="small" title="执行流程 (Flow)" style={cardStyle}>
|
||||
<Card size="small" title={flowsLoading ? <Space size={4}>执行流程 (Flow) <Spin size="small" /></Space> : "执行流程 (Flow)"} style={cardStyle}>
|
||||
<Radio.Group value={flow} onChange={handleFlowChange} style={{ width: "100%" }}>
|
||||
<Row gutter={[0, 4]}>
|
||||
{Object.entries(FLOW_DEFINITIONS).map(([id, def]) => (
|
||||
{Object.entries(flowDefs).map(([id, def]) => (
|
||||
<Col span={12} key={id}>
|
||||
<Tooltip title={def.desc}>
|
||||
<Tooltip title={def.name}>
|
||||
<Radio value={id}>
|
||||
<Text strong style={{ fontSize: 12 }}>{id}</Text>
|
||||
</Radio>
|
||||
@@ -361,7 +400,7 @@ const TaskConfig: React.FC = () => {
|
||||
}}
|
||||
>
|
||||
<Space direction="vertical" style={{ width: "100%" }}>
|
||||
{PROCESSING_MODES.map((m) => (
|
||||
{procModes.map((m) => (
|
||||
<Radio key={m.value} value={m.value}>
|
||||
<Text strong>{m.label}</Text>
|
||||
<br />
|
||||
|
||||
Reference in New Issue
Block a user