root/Neo-ZQYY
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: TaskSelector v2 全链路展示 + 同步检查 + MCP Server + 服务器 Git 排除

Browse Source 
- admin-web: TaskSelector 重构为按域+层全链路展示，新增同步检查功能
- admin-web: TaskConfig 动态加载 Flow/处理模式定义，DWD 表过滤内嵌域面板
- admin-web: App hydrate 完成前显示 loading，避免误跳 /login
- backend: 新增 /tasks/sync-check 对比后端与 ETL 真实注册表
- backend: 新增 /tasks/flows 返回 Flow 和处理模式定义
- apps/mcp-server: 新增 MCP Server 模块（百炼 AI PostgreSQL 只读查询）
- scripts/server: 新增 setup-server-git.py + server-exclude.txt
- docs: 更新 LAUNCH-CHECKLIST 添加 Git 排除配置步骤
- pyproject.toml: workspace members 新增 mcp-server
This commit is contained in:
Neo
2026-02-19 10:31:16 +08:00
parent 4eac07da47
commit 254ccb1e77
16 changed files with 2375 additions and 1285 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
apps/admin-web/src/App.tsx
View File

@@ -175,8 +175,15 @@ const AppLayout: React.FC = () => {
const App: React.FC = () => {
const hydrate = useAuthStore((s) => s.hydrate);
const [hydrated, setHydrated] = useState(false);
useEffect(() => { hydrate(); }, [hydrate]);
useEffect(() => {
hydrate();
setHydrated(true);
}, [hydrate]);
/* hydrate 完成前不渲染路由，避免 PrivateRoute 误判跳转到 /login */
if (!hydrated) return <Spin style={{ display: "flex", justifyContent: "center", marginTop: 120 }} />;
return (
<Routes>

109
apps/admin-web/src/api/tasks.ts
View File

@@ -1,32 +1,77 @@
/**
* 任务相关 API 调用。
*
* - fetchTaskRegistry：获取按业务域分组的任务注册表
*/
import { apiClient } from './client';
import type { TaskConfig, TaskDefinition } from '../types';
/** 获取按业务域分组的任务注册表 */
export async function fetchTaskRegistry(): Promise<Record<string, TaskDefinition[]>> {
// 后端返回 { groups: { 域名: [TaskItem] } }，需要解包
const { data } = await apiClient.get<{ groups: Record<string, TaskDefinition[]> }>('/tasks/registry');
return data.groups;
}
/** 获取按业务域分组的 DWD 表定义 */
export async function fetchDwdTables(): Promise<Record<string, string[]>> {
// 后端返回 { groups: { 域名: [DwdTableItem] } }，需要解包并提取 table_name
const { data } = await apiClient.get<{ groups: Record<string, { table_name: string }[]> }>('/tasks/dwd-tables');
const result: Record<string, string[]> = {};
for (const [domain, items] of Object.entries(data.groups)) {
result[domain] = items.map((item) => item.table_name);
}
return result;
}
/** 验证任务配置并返回生成的 CLI 命令预览 */
export async function validateTaskConfig(config: TaskConfig): Promise<{ command: string }> {
const { data } = await apiClient.post<{ command: string }>('/tasks/validate', { config });
return data;
}
/**
* 任务相关 API 调用。
*/
import { apiClient } from './client';
import type { TaskConfig, TaskDefinition } from '../types';
/** DWD 表项（后端返回的原始结构） */
export interface DwdTableItem {
table_name: string;
display_name: string;
domain: string;
ods_source: string;
is_dimension: boolean;
}
/** Flow 定义 */
export interface FlowDef {
id: string;
name: string;
layers: string[];
}
/** 处理模式定义 */
export interface ProcessingModeDef {
id: string;
name: string;
description: string;
}
/** 同步检查结果 */
export interface SyncCheckResult {
in_sync: boolean;
backend_only: string[];
etl_only: string[];
error: string | null;
}
/** 获取按业务域分组的任务注册表 */
export async function fetchTaskRegistry(): Promise<Record<string, TaskDefinition[]>> {
const { data } = await apiClient.get<{ groups: Record<string, TaskDefinition[]> }>('/tasks/registry');
return data.groups;
}
/** 获取按业务域分组的 DWD 表定义（保留完整结构） */
export async function fetchDwdTablesRich(): Promise<Record<string, DwdTableItem[]>> {
const { data } = await apiClient.get<{ groups: Record<string, DwdTableItem[]> }>('/tasks/dwd-tables');
return data.groups;
}
/** 获取按业务域分组的 DWD 表定义（仅表名，兼容旧调用） */
export async function fetchDwdTables(): Promise<Record<string, string[]>> {
const groups = await fetchDwdTablesRich();
const result: Record<string, string[]> = {};
for (const [domain, items] of Object.entries(groups)) {
result[domain] = items.map((item) => item.table_name);
}
return result;
}
/** 获取 Flow 定义和处理模式定义 */
export async function fetchFlows(): Promise<{ flows: FlowDef[]; processing_modes: ProcessingModeDef[] }> {
const { data } = await apiClient.get<{ flows: FlowDef[]; processing_modes: ProcessingModeDef[] }>('/tasks/flows');
return data;
}
/** 验证任务配置并返回生成的 CLI 命令预览 */
export async function validateTaskConfig(config: TaskConfig): Promise<{ command: string }> {
const { data } = await apiClient.post<{ command: string }>('/tasks/validate', { config });
return data;
}
/** 对比后端与 ETL 真实注册表的任务列表差异 */
export async function checkTaskSync(): Promise<SyncCheckResult> {
const { data } = await apiClient.get<SyncCheckResult>('/tasks/sync-check');
return data;
}

574
apps/admin-web/src/components/TaskSelector.tsx
View File

@@ -1,307 +1,445 @@
/**
* 按业务域分组的任务选择器。
* 按业务域全链路展示的任务选择器（v2）。
*
* 从 /api/tasks/registry 获取任务注册表，按业务域折叠展示，
* 支持全选/反选和按 Flow 层级过滤。
* 当 Flow 包含 DWD 层时，在 DWD 任务下方内嵌表过滤子选项。
* 每个业务域一个折叠面板，内部按层分组展示完整链路：
* ODS 任务 → DWD 表（该域的） → DWS/INDEX 任务
*
* 功能：
* - 同步检查：工具栏右侧 Badge 指示，点击展示差异 Modal
* - 全选常用 / 全选 / 反选 / 清空 按钮
* - DWD 表选中 = 过滤 DWD_LOAD_FROM_ODS 的装载范围
*/
import React, { useEffect, useState, useMemo, useCallback } from "react";
import {
Collapse,
Checkbox,
Spin,
Alert,
Button,
Space,
Typography,
Tag,
Divider,
Collapse, Checkbox, Spin, Alert, Button, Space, Typography,
Tag, Badge, Modal, Tooltip, Divider,
} from "antd";
import {
CheckCircleOutlined, WarningOutlined, SyncOutlined, TableOutlined,
} from "@ant-design/icons";
import type { CheckboxChangeEvent } from "antd/es/checkbox";
import { fetchTaskRegistry, fetchDwdTables } from "../api/tasks";
import type { TaskDefinition } from "../types";
import { fetchTaskRegistry, fetchDwdTablesRich, checkTaskSync } from "../api/tasks";
import type { DwdTableItem as ApiDwdTableItem, SyncCheckResult } from "../api/tasks";
import type { TaskDefinition, DwdTableItem } from "../types";
const { Text } = Typography;
/* ------------------------------------------------------------------ */
/* Props */
/* ------------------------------------------------------------------ */
/* 层排序 / 标签 / 颜色 */
const LAYER_ORDER: Record<string, number> = { ODS: 0, DWD: 1, DWS: 2, INDEX: 3, UTILITY: 4 };
const LAYER_LABELS: Record<string, string> = {
ODS: "ODS 抽取", DWD: "DWD 装载", DWS: "DWS 汇总", INDEX: "DWS 指数", UTILITY: "工具",
};
const LAYER_COLORS: Record<string, string> = {
ODS: "blue", DWD: "green", DWS: "orange", INDEX: "purple", UTILITY: "default",
};
/* 域排序 */
const DOMAIN_ORDER: Record<string, number> = {
助教: 0, 结算: 1, 台桌: 2, 会员: 3, 商品: 4, 团购: 5, 库存: 6, 财务: 7, 指数: 8, 通用: 9, 工具: 10,
};
export interface TaskSelectorProps {
/** 当前 Flow 包含的层（如 ["ODS", "DWD"]） */
layers: string[];
/** 已选中的任务编码列表 */
selectedTasks: string[];
/** 选中任务变化回调 */
onTasksChange: (tasks: string[]) => void;
/** DWD 表过滤：已选中的表名列表 */
selectedDwdTables?: string[];
/** DWD 表过滤变化回调 */
onDwdTablesChange?: (tables: string[]) => void;
}
/* ------------------------------------------------------------------ */
/* 过滤逻辑 */
/* ------------------------------------------------------------------ */
export function filterTasksByLayers(
tasks: TaskDefinition[],
layers: string[],
): TaskDefinition[] {
if (layers.length === 0) return [];
return tasks;
interface DomainGroup {
domain: string;
layerTasks: { layer: string; tasks: TaskDefinition[] }[];
dwdTables: DwdTableItem[];
}
/* ------------------------------------------------------------------ */
/* 组件 */
/* ------------------------------------------------------------------ */
/** 当 layers 包含 DWD 时，DWD_LOAD_FROM_ODS 由 DWD 表过滤区块隐含，不单独显示 */
const HIDDEN_WHEN_DWD_VISIBLE = new Set(["DWD_LOAD_FROM_ODS"]);
/** 按域 + 层构建分组 */
function buildDomainGroups(
registry: Record<string, TaskDefinition[]>,
dwdTableGroups: Record<string, DwdTableItem[]>,
layers: string[],
): DomainGroup[] {
const hideDwdTasks = layers.includes("DWD");
const domainSet = new Set<string>();
const tasksByDomainLayer = new Map<string, Map<string, TaskDefinition[]>>();
for (const tasks of Object.values(registry)) {
for (const t of tasks) {
if (!layers.includes(t.layer)) continue;
if (hideDwdTasks && HIDDEN_WHEN_DWD_VISIBLE.has(t.code)) continue;
domainSet.add(t.domain);
if (!tasksByDomainLayer.has(t.domain)) tasksByDomainLayer.set(t.domain, new Map());
const layerMap = tasksByDomainLayer.get(t.domain)!;
if (!layerMap.has(t.layer)) layerMap.set(t.layer, []);
layerMap.get(t.layer)!.push(t);
}
}
if (layers.includes("DWD")) {
for (const domain of Object.keys(dwdTableGroups)) domainSet.add(domain);
}
const groups: DomainGroup[] = [];
for (const domain of domainSet) {
const layerMap = tasksByDomainLayer.get(domain) ?? new Map<string, TaskDefinition[]>();
const layerTasks: { layer: string; tasks: TaskDefinition[] }[] = [];
const sortedLayers = [...layerMap.keys()].sort(
(a, b) => (LAYER_ORDER[a] ?? 99) - (LAYER_ORDER[b] ?? 99),
);
for (const layer of sortedLayers) {
const tasks = layerMap.get(layer)!;
tasks.sort((a, b) => (a.is_common === b.is_common ? 0 : a.is_common ? -1 : 1));
layerTasks.push({ layer, tasks });
}
const dwdTables = layers.includes("DWD") ? (dwdTableGroups[domain] ?? []) : [];
if (layerTasks.length > 0 || dwdTables.length > 0) {
groups.push({ domain, layerTasks, dwdTables });
}
}
groups.sort((a, b) => (DOMAIN_ORDER[a.domain] ?? 99) - (DOMAIN_ORDER[b.domain] ?? 99));
return groups;
}
const TaskSelector: React.FC<TaskSelectorProps> = ({
layers,
selectedTasks,
onTasksChange,
selectedDwdTables = [],
onDwdTablesChange,
layers, selectedTasks, onTasksChange,
selectedDwdTables = [], onDwdTablesChange,
}) => {
const [registry, setRegistry] = useState<Record<string, TaskDefinition[]>>({});
const [dwdTableGroups, setDwdTableGroups] = useState<Record<string, DwdTableItem[]>>({});
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [syncResult, setSyncResult] = useState<SyncCheckResult | null>(null);
const [syncLoading, setSyncLoading] = useState(false);
const [syncModalOpen, setSyncModalOpen] = useState(false);
// DWD 表定义（按域分组）
const [dwdTableGroups, setDwdTableGroups] = useState<Record<string, string[]>>({});
const showDwdFilter = layers.includes("DWD") && !!onDwdTablesChange;
/* ---------- 加载任务注册表 ---------- */
/* 加载数据 */
useEffect(() => {
let cancelled = false;
setLoading(true);
setError(null);
const promises: Promise<void>[] = [
fetchTaskRegistry()
.then((data) => { if (!cancelled) setRegistry(data); })
.catch((err) => { if (!cancelled) setError(err?.message ?? "获取任务列表失败"); }),
];
// 如果包含 DWD 层，同时加载 DWD 表定义
if (layers.includes("DWD")) {
promises.push(
fetchDwdTables()
.then((data) => { if (!cancelled) setDwdTableGroups(data); })
.catch(() => { /* DWD 表加载失败不阻塞任务列表 */ }),
fetchDwdTablesRich()
.then((data) => {
if (cancelled) return;
const converted: Record<string, DwdTableItem[]> = {};
for (const [domain, items] of Object.entries(data)) {
converted[domain] = items.map((item: ApiDwdTableItem) => ({
table_name: item.table_name, display_name: item.display_name,
domain: item.domain, ods_source: item.ods_source, is_dimension: item.is_dimension,
}));
}
setDwdTableGroups(converted);
})
.catch(() => {}),
);
}
Promise.all(promises).finally(() => { if (!cancelled) setLoading(false); });
return () => { cancelled = true; };
}, [layers]);
/* ---------- 按 layers 过滤后的分组 ---------- */
const filteredGroups = useMemo(() => {
const result: Record<string, TaskDefinition[]> = {};
for (const [domain, tasks] of Object.entries(registry)) {
const visible = filterTasksByLayers(tasks, layers);
if (visible.length > 0) {
result[domain] = [...visible].sort((a, b) => {
if (a.is_common === b.is_common) return 0;
return a.is_common ? -1 : 1;
});
}
}
return result;
}, [registry, layers]);
/* 首次加载后自动同步检查 */
useEffect(() => {
if (Object.keys(registry).length > 0) handleSyncCheck();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [registry]);
const domainGroups = useMemo(
() => buildDomainGroups(registry, dwdTableGroups, layers),
[registry, dwdTableGroups, layers],
);
const allVisibleCodes = useMemo(
() => Object.values(filteredGroups).flatMap((t) => t.map((d) => d.code)),
[filteredGroups],
() => domainGroups.flatMap((g) => g.layerTasks.flatMap((lt) => lt.tasks.map((t) => t.code))),
[domainGroups],
);
const allCommonCodes = useMemo(
() => domainGroups.flatMap((g) =>
g.layerTasks.flatMap((lt) => lt.tasks.filter((t) => t.is_common).map((t) => t.code)),
),
[domainGroups],
);
// DWD 表扁平列表
const allDwdTableNames = useMemo(
() => Object.values(dwdTableGroups).flat(),
[dwdTableGroups],
);
/* 同步检查 */
const handleSyncCheck = useCallback(async () => {
setSyncLoading(true);
try { setSyncResult(await checkTaskSync()); }
catch { setSyncResult({ in_sync: false, backend_only: [], etl_only: [], error: "检查失败" }); }
finally { setSyncLoading(false); }
}, []);
/* ---------- 事件处理 ---------- */
const handleDomainChange = useCallback(
(domain: string, checkedCodes: string[]) => {
const otherDomainCodes = selectedTasks.filter(
(code) => !filteredGroups[domain]?.some((t) => t.code === code),
);
onTasksChange([...otherDomainCodes, ...checkedCodes]);
},
[selectedTasks, filteredGroups, onTasksChange],
);
const handleSelectAll = useCallback(() => {
onTasksChange(allVisibleCodes);
}, [allVisibleCodes, onTasksChange]);
const handleInvertSelection = useCallback(() => {
const currentSet = new Set(selectedTasks);
const inverted = allVisibleCodes.filter((code) => !currentSet.has(code));
onTasksChange(inverted);
/* 任务选择 */
const handleSelectAll = useCallback(() => onTasksChange(allVisibleCodes), [allVisibleCodes, onTasksChange]);
const handleSelectCommon = useCallback(() => onTasksChange(allCommonCodes), [allCommonCodes, onTasksChange]);
const handleInvert = useCallback(() => {
const set = new Set(selectedTasks);
onTasksChange(allVisibleCodes.filter((c) => !set.has(c)));
}, [allVisibleCodes, selectedTasks, onTasksChange]);
const handleClear = useCallback(() => onTasksChange([]), [onTasksChange]);
/* ---------- DWD 表过滤事件 ---------- */
const handleDwdDomainTableChange = useCallback(
(domain: string, checked: string[]) => {
if (!onDwdTablesChange) return;
const domainTables = new Set(dwdTableGroups[domain] ?? []);
const otherSelected = selectedDwdTables.filter((t) => !domainTables.has(t));
onDwdTablesChange([...otherSelected, ...checked]);
const handleDomainToggle = useCallback(
(group: DomainGroup, checked: boolean) => {
const codes = new Set(group.layerTasks.flatMap((lt) => lt.tasks.map((t) => t.code)));
if (checked) {
const merged = new Set(selectedTasks);
codes.forEach((c) => merged.add(c));
onTasksChange([...merged]);
} else {
onTasksChange(selectedTasks.filter((c) => !codes.has(c)));
}
},
[selectedDwdTables, dwdTableGroups, onDwdTablesChange],
[selectedTasks, onTasksChange],
);
const handleDwdSelectAll = useCallback(() => {
onDwdTablesChange?.(allDwdTableNames);
}, [allDwdTableNames, onDwdTablesChange]);
const handleTaskToggle = useCallback(
(code: string, checked: boolean) => {
onTasksChange(checked ? [...selectedTasks, code] : selectedTasks.filter((c) => c !== code));
},
[selectedTasks, onTasksChange],
);
const handleDwdClearAll = useCallback(() => {
onDwdTablesChange?.([]);
}, [onDwdTablesChange]);
/* DWD 表选择 */
const handleDwdTableToggle = useCallback(
(tableName: string, checked: boolean) => {
if (!onDwdTablesChange) return;
onDwdTablesChange(checked
? [...selectedDwdTables, tableName]
: selectedDwdTables.filter((t) => t !== tableName));
},
[selectedDwdTables, onDwdTablesChange],
);
/* ---------- 渲染 ---------- */
const handleDwdDomainToggle = useCallback(
(tables: DwdTableItem[], checked: boolean) => {
if (!onDwdTablesChange) return;
const names = new Set(tables.map((t) => t.table_name));
if (checked) {
const merged = new Set(selectedDwdTables);
names.forEach((n) => merged.add(n));
onDwdTablesChange([...merged]);
} else {
onDwdTablesChange(selectedDwdTables.filter((t) => !names.has(t)));
}
},
[selectedDwdTables, onDwdTablesChange],
);
/* 渲染 */
if (loading) return <Spin tip="加载任务列表…" />;
if (error) return <Alert type="error" message="加载失败" description={error} />;
const domainEntries = Object.entries(filteredGroups);
if (domainEntries.length === 0) return <Text type="secondary">当前 Flow 无可选任务</Text>;
if (domainGroups.length === 0) return <Text type="secondary">当前 Flow 无可选任务</Text>;
const selectedCount = selectedTasks.filter((c) => allVisibleCodes.includes(c)).length;
// DWD 装载任务是否被选中
const dwdLoadSelected = selectedTasks.includes("DWD_LOAD_FROM_ODS");
const showDwdFilter = layers.includes("DWD") && !!onDwdTablesChange;
/** 渲染某个域下的 DWD 表过滤区块 */
const renderDwdTableFilter = (dwdTables: DwdTableItem[]) => {
if (!showDwdFilter || dwdTables.length === 0) return null;
const domainDwdSelected = selectedDwdTables.filter((t) => dwdTables.some((d) => d.table_name === t));
return (
<div style={{
marginTop: 6, marginLeft: 4, padding: "6px 8px",
background: "#f6ffed", borderRadius: 4, border: "1px solid #d9f7be",
}}>
<div style={{ display: "flex", justifyContent: "space-between", alignItems: "center", marginBottom: 4 }}>
<Space size={4}>
<TableOutlined style={{ color: "#52c41a", fontSize: 12 }} />
<Text style={{ fontSize: 12, fontWeight: 500 }}>DWD 表过滤</Text>
<Text type="secondary" style={{ fontSize: 11 }}>
{domainDwdSelected.length === 0 ? "（未选 = 全部装载）" : `${domainDwdSelected.length}/${dwdTables.length}`}
</Text>
</Space>
<Space size={4}>
<Button size="small" type="link" style={{ padding: 0, fontSize: 11, height: "auto" }}
onClick={() => handleDwdDomainToggle(dwdTables, true)}>全选</Button>
<Button size="small" type="link" style={{ padding: 0, fontSize: 11, height: "auto" }}
onClick={() => handleDwdDomainToggle(dwdTables, false)}>清空</Button>
</Space>
</div>
{dwdTables.map((dt) => (
<div key={dt.table_name} style={{ padding: "1px 0" }}>
<Checkbox
checked={selectedDwdTables.includes(dt.table_name)}
onChange={(e) => handleDwdTableToggle(dt.table_name, e.target.checked)}
>
<Text style={{ fontSize: 12 }}>{dt.table_name}</Text>
<Text type="secondary" style={{ marginLeft: 6, fontSize: 11 }}>{dt.display_name}</Text>
{dt.is_dimension && <Tag color="cyan" style={{ marginLeft: 4, fontSize: 10, lineHeight: "16px" }}>维度</Tag>}
</Checkbox>
</div>
))}
</div>
);
};
return (
<div>
<Space style={{ marginBottom: 8 }}>
<Button size="small" onClick={handleSelectAll}>全选</Button>
<Button size="small" onClick={handleInvertSelection}>反选</Button>
<Text type="secondary">已选 {selectedCount} / {allVisibleCodes.length}</Text>
</Space>
{/* 工具栏 */}
<div style={{ display: "flex", justifyContent: "space-between", alignItems: "center", marginBottom: 8 }}>
<Space size={4} wrap>
<Button size="small" onClick={handleSelectCommon}>全选常用</Button>
<Button size="small" onClick={handleSelectAll}>全选</Button>
<Button size="small" onClick={handleInvert}>反选</Button>
<Button size="small" onClick={handleClear}>清空</Button>
<Text type="secondary" style={{ marginLeft: 4 }}>已选 {selectedCount} / {allVisibleCodes.length}</Text>
</Space>
<Tooltip title="对比后端注册表与 ETL 真实任务列表">
{syncLoading ? (
<Button size="small" icon={<SyncOutlined spin />} disabled>检查中…</Button>
) : syncResult === null ? (
<Button size="small" icon={<SyncOutlined />} onClick={handleSyncCheck}>同步检查</Button>
) : syncResult.in_sync ? (
<Button size="small" icon={<CheckCircleOutlined />} style={{ color: "#52c41a", borderColor: "#b7eb8f" }} onClick={handleSyncCheck}>已同步</Button>
) : (
<Badge dot>
<Button size="small" danger icon={<WarningOutlined />} onClick={() => setSyncModalOpen(true)}>有差异</Button>
</Badge>
)}
</Tooltip>
</div>
{/* 域折叠面板 */}
<Collapse
defaultActiveKey={domainEntries.map(([d]) => d)}
items={domainEntries.map(([domain, tasks]) => {
const domainCodes = tasks.map((t) => t.code);
defaultActiveKey={domainGroups.filter((g) => g.domain !== "工具" && g.domain !== "通用").map((g) => g.domain)}
items={domainGroups.map((group) => {
const domainCodes = group.layerTasks.flatMap((lt) => lt.tasks.map((t) => t.code));
const domainSelected = selectedTasks.filter((c) => domainCodes.includes(c));
const allChecked = domainSelected.length === domainCodes.length;
const allChecked = domainCodes.length > 0 && domainSelected.length === domainCodes.length;
const indeterminate = domainSelected.length > 0 && !allChecked;
const handleDomainCheckAll = (e: CheckboxChangeEvent) => {
handleDomainChange(domain, e.target.checked ? domainCodes : []);
};
return {
key: domain,
key: group.domain,
label: (
<span onClick={(e) => e.stopPropagation()}>
<Checkbox
indeterminate={indeterminate}
checked={allChecked}
onChange={handleDomainCheckAll}
indeterminate={indeterminate} checked={allChecked}
onChange={(e: CheckboxChangeEvent) => handleDomainToggle(group, e.target.checked)}
style={{ marginRight: 8 }}
/>
{domain}
<Text type="secondary" style={{ marginLeft: 4 }}>
({domainSelected.length}/{domainCodes.length})
</Text>
{group.domain}
<Text type="secondary" style={{ marginLeft: 4 }}>({domainSelected.length}/{domainCodes.length})</Text>
</span>
),
children: (
<Checkbox.Group
value={domainSelected}
onChange={(checked) => handleDomainChange(domain, checked as string[])}
>
<Space direction="vertical" style={{ width: "100%" }}>
{tasks.map((t) => (
<Checkbox key={t.code} value={t.code}>
<Text strong style={t.is_common === false ? { color: "#999" } : undefined}>{t.code}</Text>
<Text type="secondary" style={{ marginLeft: 8 }}>{t.name}</Text>
{t.is_common === false && (
<Tag color="default" style={{ marginLeft: 6, fontSize: 11 }}>不常用</Tag>
)}
</Checkbox>
))}
</Space>
</Checkbox.Group>
<div>
{(() => {
/* 找到 DWD 表过滤应插入的位置：ODS 之后、DWS/INDEX 之前 */
const hasDwdLayer = group.layerTasks.some((lt) => lt.layer === "DWD");
const shouldInsertDwd = !hasDwdLayer && group.dwdTables.length > 0 && showDwdFilter;
/* 插入点：第一个 DWS/INDEX/UTILITY 层之前，若全是 ODS 则在末尾 */
const insertIdx = shouldInsertDwd
? group.layerTasks.findIndex((lt) => (LAYER_ORDER[lt.layer] ?? 99) >= (LAYER_ORDER["DWS"] ?? 2))
: -1;
const effectiveInsertIdx = shouldInsertDwd && insertIdx === -1 ? group.layerTasks.length : insertIdx;
const elements: React.ReactNode[] = [];
group.layerTasks.forEach((lt, idx) => {
/* 在此位置插入 DWD 表过滤 */
if (shouldInsertDwd && idx === effectiveInsertIdx) {
elements.push(
<div key="__dwd_filter__">
{elements.length > 0 && <Divider style={{ margin: "6px 0" }} />}
<div style={{ marginBottom: 4 }}>
<Tag color="green" style={{ fontSize: 11 }}>DWD 装载</Tag>
</div>
{renderDwdTableFilter(group.dwdTables)}
</div>,
);
}
elements.push(
<div key={lt.layer}>
{elements.length > 0 && <Divider style={{ margin: "6px 0" }} />}
<div style={{ marginBottom: 4 }}>
<Tag color={LAYER_COLORS[lt.layer] ?? "default"} style={{ fontSize: 11 }}>
{LAYER_LABELS[lt.layer] ?? lt.layer}
</Tag>
</div>
<div style={{ paddingLeft: 4 }}>
{lt.tasks.map((t) => (
<div key={t.code} style={{ padding: "2px 0" }}>
<Checkbox
checked={selectedTasks.includes(t.code)}
onChange={(e) => handleTaskToggle(t.code, e.target.checked)}
>
<Text strong style={!t.is_common ? { color: "#999" } : undefined}>{t.code}</Text>
<Text type="secondary" style={{ marginLeft: 8 }}>{t.name}</Text>
{!t.is_common && <Tag color="default" style={{ marginLeft: 6, fontSize: 11 }}>不常用</Tag>}
</Checkbox>
</div>
))}
</div>
{/* DWD 表过滤紧跟 DWD 层任务 */}
{lt.layer === "DWD" && renderDwdTableFilter(group.dwdTables)}
</div>,
);
});
/* 所有层遍历完后，若插入点在末尾 */
if (shouldInsertDwd && effectiveInsertIdx >= group.layerTasks.length) {
elements.push(
<div key="__dwd_filter__">
{elements.length > 0 && <Divider style={{ margin: "6px 0" }} />}
<div style={{ marginBottom: 4 }}>
<Tag color="green" style={{ fontSize: 11 }}>DWD 装载</Tag>
</div>
{renderDwdTableFilter(group.dwdTables)}
</div>,
);
}
return elements;
})()}
</div>
),
};
})}
/>
{/* DWD 表过滤：仅在 DWD 层且 DWD_LOAD_FROM_ODS 被选中时显示 */}
{showDwdFilter && dwdLoadSelected && allDwdTableNames.length > 0 && (
<>
<Divider style={{ margin: "12px 0 8px" }} />
<div style={{ padding: "0 4px" }}>
<Space style={{ marginBottom: 6 }}>
<Text strong style={{ fontSize: 13 }}>DWD 表过滤</Text>
<Text type="secondary" style={{ fontSize: 12 }}>
{selectedDwdTables.length === 0
? "（未选择 = 全部装载）"
: `已选 ${selectedDwdTables.length} / ${allDwdTableNames.length}`}
</Text>
</Space>
<div style={{ marginBottom: 6 }}>
<Space size={4}>
<Button size="small" type="link" style={{ padding: 0, fontSize: 12 }} onClick={handleDwdSelectAll}>
全选
</Button>
<Button size="small" type="link" style={{ padding: 0, fontSize: 12 }} onClick={handleDwdClearAll}>
清空（全部装载）
</Button>
</Space>
</div>
<Collapse
size="small"
items={Object.entries(dwdTableGroups).map(([domain, tables]) => {
const domainSelected = selectedDwdTables.filter((t) => tables.includes(t));
const allDomainChecked = domainSelected.length === tables.length;
const domainIndeterminate = domainSelected.length > 0 && !allDomainChecked;
return {
key: domain,
label: (
<span onClick={(e) => e.stopPropagation()}>
<Checkbox
indeterminate={domainIndeterminate}
checked={allDomainChecked}
onChange={(e: CheckboxChangeEvent) =>
handleDwdDomainTableChange(domain, e.target.checked ? tables : [])
}
style={{ marginRight: 8 }}
/>
{domain}
<Text type="secondary" style={{ marginLeft: 4, fontSize: 12 }}>
({domainSelected.length}/{tables.length})
</Text>
</span>
),
children: (
<Checkbox.Group
value={domainSelected}
onChange={(checked) => handleDwdDomainTableChange(domain, checked as string[])}
>
<Space direction="vertical">
{tables.map((table) => (
<Checkbox key={table} value={table}>
<Text style={{ fontSize: 12 }}>{table}</Text>
</Checkbox>
))}
</Space>
</Checkbox.Group>
),
};
})}
/>
{/* 同步差异 Modal */}
<Modal
title="任务注册表同步检查"
open={syncModalOpen}
onCancel={() => setSyncModalOpen(false)}
footer={[
<Button key="refresh" icon={<SyncOutlined />} onClick={() => { handleSyncCheck(); }}>重新检查</Button>,
<Button key="close" type="primary" onClick={() => setSyncModalOpen(false)}>关闭</Button>,
]}
>
{syncResult?.error ? (
<Alert type="error" message="检查出错" description={syncResult.error} />
) : (
<div>
{syncResult?.backend_only && syncResult.backend_only.length > 0 && (
<div style={{ marginBottom: 12 }}>
<Text strong style={{ color: "#faad14" }}>后端有但 ETL 无（{syncResult.backend_only.length}）：</Text>
<div style={{ marginTop: 4 }}>
{syncResult.backend_only.map((code) => (
<Tag key={code} color="warning" style={{ marginBottom: 4 }}>{code}</Tag>
))}
</div>
</div>
)}
{syncResult?.etl_only && syncResult.etl_only.length > 0 && (
<div>
<Text strong style={{ color: "#ff4d4f" }}>ETL 有但后端无（{syncResult.etl_only.length}）：</Text>
<div style={{ marginTop: 4 }}>
{syncResult.etl_only.map((code) => (
<Tag key={code} color="error" style={{ marginBottom: 4 }}>{code}</Tag>
))}
</div>
</div>
)}
{syncResult?.in_sync && (
<Alert type="success" message="后端与 ETL 任务列表完全一致" />
)}
</div>
</>
)}
)}
</Modal>
</div>
);
};

93
apps/admin-web/src/pages/TaskConfig.tsx
View File

@@ -24,6 +24,7 @@ import {
TreeSelect,
Tooltip,
Segmented,
Spin,
} from "antd";
import {
SendOutlined,
@@ -37,7 +38,8 @@ import {
} from "@ant-design/icons";
import { useNavigate } from "react-router-dom";
import TaskSelector from "../components/TaskSelector";
import { validateTaskConfig } from "../api/tasks";
import { validateTaskConfig, fetchFlows } from "../api/tasks";
import type { FlowDef, ProcessingModeDef } from "../api/tasks";
import { submitToQueue, executeDirectly } from "../api/execution";
import { useAuthStore } from "../store/authStore";
import type { RadioChangeEvent } from "antd";
@@ -48,32 +50,45 @@ const { Title, Text } = Typography;
const { TextArea } = Input;
/* ------------------------------------------------------------------ */
/* Flow 定义 */
/* Flow / 处理模式 — 本地 fallback（API 不可用时兜底） */
/* ------------------------------------------------------------------ */
const FLOW_DEFINITIONS: Record<string, { name: string; layers: string[]; desc: string }> = {
api_ods: { name: "API → ODS", layers: ["ODS"], desc: "仅抓取原始数据" },
api_ods_dwd: { name: "API → ODS → DWD", layers: ["ODS", "DWD"], desc: "抓取并清洗装载" },
api_full: { name: "API → ODS → DWD → DWS → INDEX", layers: ["ODS", "DWD", "DWS", "INDEX"], desc: "全链路执行" },
ods_dwd: { name: "ODS → DWD", layers: ["DWD"], desc: "仅清洗装载" },
dwd_dws: { name: "DWD → DWS汇总", layers: ["DWS"], desc: "仅汇总计算" },
dwd_dws_index: { name: "DWD → DWS → INDEX", layers: ["DWS", "INDEX"], desc: "汇总+指数" },
dwd_index: { name: "DWD → INDEX", layers: ["INDEX"], desc: "仅指数计算" },
interface FlowEntry { name: string; layers: string[] }
const FALLBACK_FLOWS: Record<string, FlowEntry> = {
api_ods: { name: "API → ODS", layers: ["ODS"] },
api_ods_dwd: { name: "API → ODS → DWD", layers: ["ODS", "DWD"] },
api_full: { name: "API → ODS → DWD → DWS → INDEX", layers: ["ODS", "DWD", "DWS", "INDEX"] },
ods_dwd: { name: "ODS → DWD", layers: ["DWD"] },
dwd_dws: { name: "DWD → DWS汇总", layers: ["DWS"] },
dwd_dws_index: { name: "DWD → DWS → INDEX", layers: ["DWS", "INDEX"] },
dwd_index: { name: "DWD → INDEX", layers: ["INDEX"] },
};
export function getFlowLayers(flowId: string): string[] {
return FLOW_DEFINITIONS[flowId]?.layers ?? [];
}
interface ProcModeEntry { value: string; label: string; desc: string }
/* ------------------------------------------------------------------ */
/* 处理模式 */
/* ------------------------------------------------------------------ */
const PROCESSING_MODES = [
const FALLBACK_PROCESSING_MODES: ProcModeEntry[] = [
{ value: "increment_only", label: "仅增量", desc: "按游标增量抓取和装载" },
{ value: "verify_only", label: "校验并修复", desc: "对比源和目标，修复差异" },
{ value: "increment_verify", label: "增量+校验", desc: "先增量再校验" },
] as const;
];
/** 将 API 返回的 FlowDef[] 转为 Record<id, FlowEntry> */
function apiFlowsToRecord(flows: FlowDef[]): Record<string, FlowEntry> {
const result: Record<string, FlowEntry> = {};
for (const f of flows) result[f.id] = { name: f.name, layers: f.layers };
return result;
}
/** 将 API 返回的 ProcessingModeDef[] 转为 ProcModeEntry[] */
function apiModesToEntries(modes: ProcessingModeDef[]): ProcModeEntry[] {
return modes.map((m) => ({ value: m.id, label: m.name, desc: m.description }));
}
/** 外部可用的 getFlowLayers（使用 fallback，组件内部用动态数据） */
export function getFlowLayers(flowId: string): string[] {
return FALLBACK_FLOWS[flowId]?.layers ?? [];
}
/* ------------------------------------------------------------------ */
/* 时间窗口 */
@@ -147,6 +162,24 @@ const TaskConfig: React.FC = () => {
const navigate = useNavigate();
const user = useAuthStore((s) => s.user);
/* ---------- Flow / 处理模式 动态加载 ---------- */
const [flowDefs, setFlowDefs] = useState<Record<string, FlowEntry>>(FALLBACK_FLOWS);
const [procModes, setProcModes] = useState<ProcModeEntry[]>(FALLBACK_PROCESSING_MODES);
const [flowsLoading, setFlowsLoading] = useState(true);
useEffect(() => {
let cancelled = false;
fetchFlows()
.then(({ flows, processing_modes }) => {
if (cancelled) return;
if (flows.length > 0) setFlowDefs(apiFlowsToRecord(flows));
if (processing_modes.length > 0) setProcModes(apiModesToEntries(processing_modes));
})
.catch(() => { /* API 不可用，使用 fallback */ })
.finally(() => { if (!cancelled) setFlowsLoading(false); });
return () => { cancelled = true; };
}, []);
/* ---------- 连接器 & Store 树形选择 ---------- */
const { treeData: connectorTreeData, allValues: allConnectorStoreValues } = useMemo(
() => buildConnectorStoreTree(CONNECTOR_DEFS, user?.site_id ?? null),
@@ -199,12 +232,17 @@ const TaskConfig: React.FC = () => {
const [submitting, setSubmitting] = useState(false);
/* ---------- 派生状态 ---------- */
const layers = getFlowLayers(flow);
const layers = flowDefs[flow]?.layers ?? [];
const showVerifyOption = processingMode === "verify_only";
/* ---------- 构建 TaskConfig 对象 ---------- */
const buildTaskConfig = (): TaskConfigType => ({
tasks: selectedTasks,
const buildTaskConfig = (): TaskConfigType => {
/* layers 包含 DWD 时自动注入 DWD_LOAD_FROM_ODS（UI 上由 DWD 表过滤区块隐含） */
const tasks = layers.includes("DWD") && !selectedTasks.includes("DWD_LOAD_FROM_ODS")
? [...selectedTasks, "DWD_LOAD_FROM_ODS"]
: selectedTasks;
return {
tasks,
pipeline: flow,
processing_mode: processingMode,
pipeline_flow: "FULL",
@@ -223,7 +261,8 @@ const TaskConfig: React.FC = () => {
dwd_only_tables: selectedDwdTables.length > 0 ? selectedDwdTables : null,
force_full: forceFull,
extra_args: {},
});
};
};
/* ---------- 自动刷新 CLI 预览 ---------- */
const refreshCli = async () => {
@@ -326,12 +365,12 @@ const TaskConfig: React.FC = () => {
</Card>
</Col>
<Col span={16}>
<Card size="small" title="执行流程 (Flow)" style={cardStyle}>
<Card size="small" title={flowsLoading ? <Space size={4}>执行流程 (Flow) <Spin size="small" /></Space> : "执行流程 (Flow)"} style={cardStyle}>
<Radio.Group value={flow} onChange={handleFlowChange} style={{ width: "100%" }}>
<Row gutter={[0, 4]}>
{Object.entries(FLOW_DEFINITIONS).map(([id, def]) => (
{Object.entries(flowDefs).map(([id, def]) => (
<Col span={12} key={id}>
<Tooltip title={def.desc}>
<Tooltip title={def.name}>
<Radio value={id}>
<Text strong style={{ fontSize: 12 }}>{id}</Text>
</Radio>
@@ -361,7 +400,7 @@ const TaskConfig: React.FC = () => {
}}
>
<Space direction="vertical" style={{ width: "100%" }}>
{PROCESSING_MODES.map((m) => (
{procModes.map((m) => (
<Radio key={m.value} value={m.value}>
<Text strong>{m.label}</Text>
<br />

277
apps/admin-web/src/types/index.ts
View File

@@ -1,133 +1,144 @@
/**
* 前后端共享的 TypeScript 类型定义。
* 与设计文档中的 Pydantic 模型和数据库表结构对应。
*/
/** ETL 任务执行配置 */
export interface TaskConfig {
tasks: string[];
/** 执行流程 Flow ID（对应 CLI --pipeline） */
pipeline: string;
/** 处理模式 */
processing_mode: string;
/** 传统模式兼容（已弃用） */
pipeline_flow: string;
dry_run: boolean;
/** lookback / custom */
window_mode: string;
window_start: string | null;
window_end: string | null;
/** none / day */
window_split: string | null;
/** 1 / 10 / 30 */
window_split_days: number | null;
lookback_hours: number;
overlap_seconds: number;
fetch_before_verify: boolean;
skip_ods_when_fetch_before_verify: boolean;
ods_use_local_json: boolean;
/** 门店 ID（由后端从 JWT 注入） */
store_id: number | null;
/** DWD 表级选择 */
dwd_only_tables: string[] | null;
/** 强制全量处理（跳过 hash 去重和变更对比） */
force_full: boolean;
extra_args: Record<string, unknown>;
}
/** 执行流程（Flow）定义 */
export interface PipelineDefinition {
id: string;
name: string;
/** 包含的层：ODS / DWD / DWS / INDEX */
layers: string[];
}
/** 处理模式定义 */
export interface ProcessingModeDefinition {
id: string;
name: string;
description: string;
}
/** 任务注册表中的任务定义 */
export interface TaskDefinition {
code: string;
name: string;
description: string;
/** 业务域（会员、结算、助教等） */
domain: string;
requires_window: boolean;
is_ods: boolean;
is_dimension: boolean;
default_enabled: boolean;
/** 常用任务标记，false 表示工具类/手动类任务 */
is_common: boolean;
}
/** 调度配置 */
export interface ScheduleConfig {
schedule_type: "once" | "interval" | "daily" | "weekly" | "cron";
interval_value: number;
interval_unit: "minutes" | "hours" | "days";
daily_time: string;
weekly_days: number[];
weekly_time: string;
cron_expression: string;
enabled: boolean;
start_date: string | null;
end_date: string | null;
}
/** 队列中的任务 */
export interface QueuedTask {
id: string;
site_id: number;
config: TaskConfig;
status: "pending" | "running" | "success" | "failed" | "cancelled";
position: number;
created_at: string;
started_at: string | null;
finished_at: string | null;
exit_code: number | null;
error_message: string | null;
}
/** 执行历史记录 */
export interface ExecutionLog {
id: string;
site_id: number;
task_codes: string[];
status: string;
started_at: string;
finished_at: string | null;
exit_code: number | null;
duration_ms: number | null;
command: string;
summary: Record<string, unknown> | null;
}
/** 调度任务 */
export interface ScheduledTask {
id: string;
site_id: number;
name: string;
task_codes: string[];
task_config: TaskConfig;
schedule_config: ScheduleConfig;
enabled: boolean;
last_run_at: string | null;
next_run_at: string | null;
run_count: number;
last_status: string | null;
created_at: string;
updated_at: string;
}
/** 环境配置项 */
export interface EnvConfigItem {
key: string;
value: string;
is_sensitive: boolean;
}
/**
* 前后端共享的 TypeScript 类型定义。
* 与设计文档中的 Pydantic 模型和数据库表结构对应。
*/
/** ETL 任务执行配置 */
export interface TaskConfig {
tasks: string[];
/** 执行流程 Flow ID（对应 CLI --pipeline） */
pipeline: string;
/** 处理模式 */
processing_mode: string;
/** 传统模式兼容（已弃用） */
pipeline_flow: string;
dry_run: boolean;
/** lookback / custom */
window_mode: string;
window_start: string | null;
window_end: string | null;
/** none / day */
window_split: string | null;
/** 1 / 10 / 30 */
window_split_days: number | null;
lookback_hours: number;
overlap_seconds: number;
fetch_before_verify: boolean;
skip_ods_when_fetch_before_verify: boolean;
ods_use_local_json: boolean;
/** 门店 ID（由后端从 JWT 注入） */
store_id: number | null;
/** DWD 表级选择 */
dwd_only_tables: string[] | null;
/** 强制全量处理（跳过 hash 去重和变更对比） */
force_full: boolean;
extra_args: Record<string, unknown>;
}
/** 执行流程（Flow）定义 */
export interface PipelineDefinition {
id: string;
name: string;
/** 包含的层：ODS / DWD / DWS / INDEX */
layers: string[];
}
/** 处理模式定义 */
export interface ProcessingModeDefinition {
id: string;
name: string;
description: string;
}
/** 任务注册表中的任务定义 */
export interface TaskDefinition {
code: string;
name: string;
description: string;
/** 业务域（会员、结算、助教等） */
domain: string;
/** 所属层：ODS / DWD / DWS / INDEX / UTILITY */
layer: string;
requires_window: boolean;
is_ods: boolean;
is_dimension: boolean;
default_enabled: boolean;
/** 常用任务标记，false 表示工具类/手动类任务 */
is_common: boolean;
}
/** DWD 表定义（后端返回的完整结构） */
export interface DwdTableItem {
table_name: string;
display_name: string;
domain: string;
ods_source: string;
is_dimension: boolean;
}
/** 调度配置 */
export interface ScheduleConfig {
schedule_type: "once" | "interval" | "daily" | "weekly" | "cron";
interval_value: number;
interval_unit: "minutes" | "hours" | "days";
daily_time: string;
weekly_days: number[];
weekly_time: string;
cron_expression: string;
enabled: boolean;
start_date: string | null;
end_date: string | null;
}
/** 队列中的任务 */
export interface QueuedTask {
id: string;
site_id: number;
config: TaskConfig;
status: "pending" | "running" | "success" | "failed" | "cancelled";
position: number;
created_at: string;
started_at: string | null;
finished_at: string | null;
exit_code: number | null;
error_message: string | null;
}
/** 执行历史记录 */
export interface ExecutionLog {
id: string;
site_id: number;
task_codes: string[];
status: string;
started_at: string;
finished_at: string | null;
exit_code: number | null;
duration_ms: number | null;
command: string;
summary: Record<string, unknown> | null;
}
/** 调度任务 */
export interface ScheduledTask {
id: string;
site_id: number;
name: string;
task_codes: string[];
task_config: TaskConfig;
schedule_config: ScheduleConfig;
enabled: boolean;
last_run_at: string | null;
next_run_at: string | null;
run_count: number;
last_status: string | null;
created_at: string;
updated_at: string;
}
/** 环境配置项 */
export interface EnvConfigItem {
key: string;
value: string;
is_sensitive: boolean;
}