在前后端开发联调前 的提交20260223
This commit is contained in:
8
apps/admin-web/.vite/deps/_metadata.json
Normal file
8
apps/admin-web/.vite/deps/_metadata.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"hash": "75f75ae2",
|
||||
"configHash": "3c6579c7",
|
||||
"lockfileHash": "4e1d8c76",
|
||||
"browserHash": "dc64490c",
|
||||
"optimized": {},
|
||||
"chunks": {}
|
||||
}
|
||||
3
apps/admin-web/.vite/deps/package.json
Normal file
3
apps/admin-web/.vite/deps/package.json
Normal file
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"type": "module"
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
DashboardOutlined,
|
||||
FileTextOutlined,
|
||||
LogoutOutlined,
|
||||
DesktopOutlined,
|
||||
} from "@ant-design/icons";
|
||||
import type { MenuProps } from "antd";
|
||||
import { useAuthStore } from "./store/authStore";
|
||||
@@ -29,6 +30,7 @@ import EnvConfig from "./pages/EnvConfig";
|
||||
import DBViewer from "./pages/DBViewer";
|
||||
import ETLStatus from "./pages/ETLStatus";
|
||||
import LogViewer from "./pages/LogViewer";
|
||||
import OpsPanel from "./pages/OpsPanel";
|
||||
|
||||
const { Sider, Content, Footer } = Layout;
|
||||
const { Text } = Typography;
|
||||
@@ -44,6 +46,7 @@ const NAV_ITEMS: MenuProps["items"] = [
|
||||
{ key: "/db-viewer", icon: <DatabaseOutlined />, label: "数据库" },
|
||||
{ key: "/log-viewer", icon: <FileTextOutlined />, label: "日志" },
|
||||
{ key: "/env-config", icon: <ToolOutlined />, label: "环境配置" },
|
||||
{ key: "/ops-panel", icon: <DesktopOutlined />, label: "运维面板" },
|
||||
];
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
@@ -140,6 +143,7 @@ const AppLayout: React.FC = () => {
|
||||
<Route path="/db-viewer" element={<DBViewer />} />
|
||||
<Route path="/etl-status" element={<ETLStatus />} />
|
||||
<Route path="/log-viewer" element={<LogViewer />} />
|
||||
<Route path="/ops-panel" element={<OpsPanel />} />
|
||||
</Routes>
|
||||
</Content>
|
||||
<Footer
|
||||
@@ -154,7 +158,7 @@ const AppLayout: React.FC = () => {
|
||||
<Space size={8}>
|
||||
<Spin size="small" />
|
||||
<Text>执行中</Text>
|
||||
<Tag color="processing">{runningTask.config.pipeline}</Tag>
|
||||
<Tag color="processing">{runningTask.config.flow}</Tag>
|
||||
<Text type="secondary" style={{ fontSize: 12 }}>
|
||||
{runningTask.config.tasks.slice(0, 3).join(", ")}
|
||||
{runningTask.config.tasks.length > 3 && ` +${runningTask.config.tasks.length - 3}`}
|
||||
|
||||
100
apps/admin-web/src/api/opsPanel.ts
Normal file
100
apps/admin-web/src/api/opsPanel.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
/**
|
||||
* 运维控制面板 API
|
||||
*
|
||||
* 对接后端 /api/ops/* 端点,提供服务状态、Git 操作、系统信息等。
|
||||
*/
|
||||
|
||||
import { apiClient } from "./client";
|
||||
|
||||
// ---- 类型定义 ----
|
||||
|
||||
export interface SystemInfo {
|
||||
cpu_percent: number;
|
||||
memory_total_gb: number;
|
||||
memory_used_gb: number;
|
||||
memory_percent: number;
|
||||
disk_total_gb: number;
|
||||
disk_used_gb: number;
|
||||
disk_percent: number;
|
||||
boot_time: string;
|
||||
}
|
||||
|
||||
export interface ServiceStatus {
|
||||
env: string;
|
||||
label: string;
|
||||
running: boolean;
|
||||
pid: number | null;
|
||||
port: number;
|
||||
uptime_seconds: number | null;
|
||||
memory_mb: number | null;
|
||||
cpu_percent: number | null;
|
||||
}
|
||||
|
||||
export interface GitInfo {
|
||||
env: string;
|
||||
branch: string;
|
||||
last_commit_hash: string;
|
||||
last_commit_message: string;
|
||||
last_commit_time: string;
|
||||
has_local_changes: boolean;
|
||||
}
|
||||
|
||||
export interface ActionResult {
|
||||
env: string;
|
||||
action: string;
|
||||
success: boolean;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface GitPullResult {
|
||||
env: string;
|
||||
success: boolean;
|
||||
output: string;
|
||||
}
|
||||
|
||||
// ---- API 调用 ----
|
||||
|
||||
export async function fetchSystemInfo(): Promise<SystemInfo> {
|
||||
const { data } = await apiClient.get<SystemInfo>("/ops/system");
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function fetchServicesStatus(): Promise<ServiceStatus[]> {
|
||||
const { data } = await apiClient.get<ServiceStatus[]>("/ops/services");
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function fetchGitInfo(): Promise<GitInfo[]> {
|
||||
const { data } = await apiClient.get<GitInfo[]>("/ops/git");
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function startService(env: string): Promise<ActionResult> {
|
||||
const { data } = await apiClient.post<ActionResult>(`/ops/services/${env}/start`);
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function stopService(env: string): Promise<ActionResult> {
|
||||
const { data } = await apiClient.post<ActionResult>(`/ops/services/${env}/stop`);
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function restartService(env: string): Promise<ActionResult> {
|
||||
const { data } = await apiClient.post<ActionResult>(`/ops/services/${env}/restart`);
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function gitPull(env: string): Promise<GitPullResult> {
|
||||
const { data } = await apiClient.post<GitPullResult>(`/ops/git/${env}/pull`);
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function syncDeps(env: string): Promise<ActionResult> {
|
||||
const { data } = await apiClient.post<ActionResult>(`/ops/git/${env}/sync-deps`);
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function fetchEnvFile(env: string): Promise<{ env: string; content: string }> {
|
||||
const { data } = await apiClient.get<{ env: string; content: string }>(`/ops/env-file/${env}`);
|
||||
return data;
|
||||
}
|
||||
@@ -240,7 +240,7 @@ const ScheduleTab: React.FC = () => {
|
||||
task_codes: [],
|
||||
task_config: {
|
||||
tasks: [],
|
||||
pipeline: 'api_full',
|
||||
flow: 'api_full',
|
||||
processing_mode: 'increment_only',
|
||||
pipeline_flow: 'FULL',
|
||||
dry_run: false,
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* 功能:
|
||||
* - 同步检查:工具栏右侧 Badge 指示,点击展示差异 Modal
|
||||
* - 全选常用 / 全选 / 反选 / 清空 按钮
|
||||
* - DWD 表选中 = 过滤 DWD_LOAD_FROM_ODS 的装载范围
|
||||
* - DWD 表勾选 = 选择要装载的 DWD 表(正向选择,和 ODS/DWS 一致)
|
||||
*/
|
||||
|
||||
import React, { useEffect, useState, useMemo, useCallback } from "react";
|
||||
@@ -151,6 +151,16 @@ const TaskSelector: React.FC<TaskSelectorProps> = ({
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [registry]);
|
||||
|
||||
/* CHANGE [2026-02-19] intent: DWD 表正向勾选,加载后默认全选 */
|
||||
useEffect(() => {
|
||||
if (!onDwdTablesChange) return;
|
||||
const allTables = Object.values(dwdTableGroups).flat().map((t) => t.table_name);
|
||||
if (allTables.length > 0 && selectedDwdTables.length === 0) {
|
||||
onDwdTablesChange(allTables);
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [dwdTableGroups]);
|
||||
|
||||
const domainGroups = useMemo(
|
||||
() => buildDomainGroups(registry, dwdTableGroups, layers),
|
||||
[registry, dwdTableGroups, layers],
|
||||
@@ -251,9 +261,9 @@ const TaskSelector: React.FC<TaskSelectorProps> = ({
|
||||
<div style={{ display: "flex", justifyContent: "space-between", alignItems: "center", marginBottom: 4 }}>
|
||||
<Space size={4}>
|
||||
<TableOutlined style={{ color: "#52c41a", fontSize: 12 }} />
|
||||
<Text style={{ fontSize: 12, fontWeight: 500 }}>DWD 表过滤</Text>
|
||||
<Text style={{ fontSize: 12, fontWeight: 500 }}>DWD 装载表</Text>
|
||||
<Text type="secondary" style={{ fontSize: 11 }}>
|
||||
{domainDwdSelected.length === 0 ? "(未选 = 全部装载)" : `${domainDwdSelected.length}/${dwdTables.length}`}
|
||||
{`${domainDwdSelected.length}/${dwdTables.length}`}
|
||||
</Text>
|
||||
</Space>
|
||||
<Space size={4}>
|
||||
@@ -370,6 +380,9 @@ const TaskSelector: React.FC<TaskSelectorProps> = ({
|
||||
>
|
||||
<Text strong style={!t.is_common ? { color: "#999" } : undefined}>{t.code}</Text>
|
||||
<Text type="secondary" style={{ marginLeft: 8 }}>{t.name}</Text>
|
||||
{t.description && (t.layer === "DWS" || t.layer === "INDEX") && (
|
||||
<Text type="secondary" style={{ marginLeft: 6, fontSize: 10, color: "#8c8c8c" }}>({t.description})</Text>
|
||||
)}
|
||||
{!t.is_common && <Tag color="default" style={{ marginLeft: 6, fontSize: 11 }}>不常用</Tag>}
|
||||
</Checkbox>
|
||||
</div>
|
||||
|
||||
365
apps/admin-web/src/pages/OpsPanel.tsx
Normal file
365
apps/admin-web/src/pages/OpsPanel.tsx
Normal file
@@ -0,0 +1,365 @@
|
||||
/**
|
||||
* 运维控制面板页面
|
||||
*
|
||||
* 功能:
|
||||
* - 服务器系统资源概况(CPU / 内存 / 磁盘)
|
||||
* - 各环境服务状态 + 启停重启按钮
|
||||
* - 各环境 Git 状态 + pull / 同步依赖按钮
|
||||
* - 各环境 .env 配置查看(敏感值脱敏)
|
||||
*/
|
||||
|
||||
import React, { useEffect, useState, useCallback } from "react";
|
||||
import {
|
||||
Card,
|
||||
Row,
|
||||
Col,
|
||||
Tag,
|
||||
Button,
|
||||
Space,
|
||||
Statistic,
|
||||
Progress,
|
||||
Modal,
|
||||
message,
|
||||
Descriptions,
|
||||
Spin,
|
||||
Tooltip,
|
||||
Typography,
|
||||
Input,
|
||||
} from "antd";
|
||||
import {
|
||||
PlayCircleOutlined,
|
||||
PauseCircleOutlined,
|
||||
ReloadOutlined,
|
||||
CloudDownloadOutlined,
|
||||
SyncOutlined,
|
||||
FileTextOutlined,
|
||||
CheckCircleOutlined,
|
||||
CloseCircleOutlined,
|
||||
ClockCircleOutlined,
|
||||
DesktopOutlined,
|
||||
} from "@ant-design/icons";
|
||||
import type {
|
||||
SystemInfo,
|
||||
ServiceStatus,
|
||||
GitInfo,
|
||||
} from "../api/opsPanel";
|
||||
import {
|
||||
fetchSystemInfo,
|
||||
fetchServicesStatus,
|
||||
fetchGitInfo,
|
||||
startService,
|
||||
stopService,
|
||||
restartService,
|
||||
gitPull,
|
||||
syncDeps,
|
||||
fetchEnvFile,
|
||||
} from "../api/opsPanel";
|
||||
|
||||
const { Text, Title } = Typography;
|
||||
const { TextArea } = Input;
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* 工具函数 */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/** 秒数格式化为 "Xd Xh Xm" */
|
||||
function formatUptime(seconds: number | null): string {
|
||||
if (seconds == null) return "-";
|
||||
const d = Math.floor(seconds / 86400);
|
||||
const h = Math.floor((seconds % 86400) / 3600);
|
||||
const m = Math.floor((seconds % 3600) / 60);
|
||||
const parts: string[] = [];
|
||||
if (d > 0) parts.push(`${d}天`);
|
||||
if (h > 0) parts.push(`${h}时`);
|
||||
parts.push(`${m}分`);
|
||||
return parts.join(" ");
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* 组件 */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const OpsPanel: React.FC = () => {
|
||||
const [system, setSystem] = useState<SystemInfo | null>(null);
|
||||
const [services, setServices] = useState<ServiceStatus[]>([]);
|
||||
const [gitInfos, setGitInfos] = useState<GitInfo[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [actionLoading, setActionLoading] = useState<Record<string, boolean>>({});
|
||||
const [envModalOpen, setEnvModalOpen] = useState(false);
|
||||
const [envModalContent, setEnvModalContent] = useState("");
|
||||
const [envModalTitle, setEnvModalTitle] = useState("");
|
||||
|
||||
// ---- 数据加载 ----
|
||||
|
||||
const loadAll = useCallback(async () => {
|
||||
try {
|
||||
const [sys, svc, git] = await Promise.all([
|
||||
fetchSystemInfo(),
|
||||
fetchServicesStatus(),
|
||||
fetchGitInfo(),
|
||||
]);
|
||||
setSystem(sys);
|
||||
setServices(svc);
|
||||
setGitInfos(git);
|
||||
} catch {
|
||||
message.error("加载运维数据失败");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
loadAll();
|
||||
const timer = setInterval(loadAll, 15_000);
|
||||
return () => clearInterval(timer);
|
||||
}, [loadAll]);
|
||||
|
||||
// ---- 操作处理 ----
|
||||
|
||||
const withAction = async (key: string, fn: () => Promise<void>) => {
|
||||
setActionLoading((prev) => ({ ...prev, [key]: true }));
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
setActionLoading((prev) => ({ ...prev, [key]: false }));
|
||||
}
|
||||
};
|
||||
|
||||
const handleStart = (env: string) =>
|
||||
withAction(`start-${env}`, async () => {
|
||||
const r = await startService(env);
|
||||
r.success ? message.success(r.message) : message.warning(r.message);
|
||||
await loadAll();
|
||||
});
|
||||
|
||||
const handleStop = (env: string) =>
|
||||
withAction(`stop-${env}`, async () => {
|
||||
const r = await stopService(env);
|
||||
r.success ? message.success(r.message) : message.warning(r.message);
|
||||
await loadAll();
|
||||
});
|
||||
|
||||
const handleRestart = (env: string) =>
|
||||
withAction(`restart-${env}`, async () => {
|
||||
const r = await restartService(env);
|
||||
r.success ? message.success(r.message) : message.warning(r.message);
|
||||
await loadAll();
|
||||
});
|
||||
|
||||
const handlePull = (env: string) =>
|
||||
withAction(`pull-${env}`, async () => {
|
||||
const r = await gitPull(env);
|
||||
if (r.success) {
|
||||
message.success("拉取成功");
|
||||
Modal.info({ title: `Git Pull - ${env}`, content: <pre style={{ maxHeight: 300, overflow: "auto", fontSize: 12 }}>{r.output}</pre>, width: 600 });
|
||||
} else {
|
||||
message.error("拉取失败");
|
||||
Modal.error({ title: `Git Pull 失败 - ${env}`, content: <pre style={{ maxHeight: 300, overflow: "auto", fontSize: 12 }}>{r.output}</pre>, width: 600 });
|
||||
}
|
||||
await loadAll();
|
||||
});
|
||||
|
||||
const handleSyncDeps = (env: string) =>
|
||||
withAction(`sync-${env}`, async () => {
|
||||
const r = await syncDeps(env);
|
||||
r.success ? message.success("依赖同步完成") : message.error(r.message);
|
||||
});
|
||||
|
||||
const handleViewEnv = async (env: string, label: string) => {
|
||||
try {
|
||||
const r = await fetchEnvFile(env);
|
||||
setEnvModalTitle(`${label} .env 配置`);
|
||||
setEnvModalContent(r.content);
|
||||
setEnvModalOpen(true);
|
||||
} catch {
|
||||
message.error("读取配置文件失败");
|
||||
}
|
||||
};
|
||||
|
||||
// ---- 渲染 ----
|
||||
|
||||
if (loading) {
|
||||
return <Spin size="large" style={{ display: "flex", justifyContent: "center", marginTop: 120 }} />;
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Title level={4} style={{ marginBottom: 16 }}>
|
||||
<DesktopOutlined style={{ marginRight: 8 }} />
|
||||
运维控制面板
|
||||
</Title>
|
||||
|
||||
{/* ---- 系统资源 ---- */}
|
||||
{system && (
|
||||
<Card size="small" title="服务器资源" style={{ marginBottom: 16 }}>
|
||||
<Row gutter={24}>
|
||||
<Col span={8}>
|
||||
<Statistic title="CPU 使用率" value={system.cpu_percent} suffix="%" />
|
||||
<Progress percent={system.cpu_percent} size="small" status={system.cpu_percent > 80 ? "exception" : "normal"} showInfo={false} />
|
||||
</Col>
|
||||
<Col span={8}>
|
||||
<Statistic title="内存" value={system.memory_used_gb} suffix={`/ ${system.memory_total_gb} GB`} precision={1} />
|
||||
<Progress percent={system.memory_percent} size="small" status={system.memory_percent > 85 ? "exception" : "normal"} showInfo={false} />
|
||||
</Col>
|
||||
<Col span={8}>
|
||||
<Statistic title="磁盘" value={system.disk_used_gb} suffix={`/ ${system.disk_total_gb} GB`} precision={1} />
|
||||
<Progress percent={system.disk_percent} size="small" status={system.disk_percent > 90 ? "exception" : "normal"} showInfo={false} />
|
||||
</Col>
|
||||
</Row>
|
||||
<Text type="secondary" style={{ fontSize: 12, marginTop: 8, display: "block" }}>
|
||||
开机时间:{new Date(system.boot_time).toLocaleString()}
|
||||
</Text>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{/* ---- 服务状态 ---- */}
|
||||
<Card size="small" title="服务状态" style={{ marginBottom: 16 }}>
|
||||
<Row gutter={16}>
|
||||
{services.map((svc) => (
|
||||
<Col span={12} key={svc.env}>
|
||||
<Card
|
||||
size="small"
|
||||
type="inner"
|
||||
title={
|
||||
<Space>
|
||||
{svc.running
|
||||
? <CheckCircleOutlined style={{ color: "#52c41a" }} />
|
||||
: <CloseCircleOutlined style={{ color: "#ff4d4f" }} />}
|
||||
{svc.label}
|
||||
<Tag color={svc.running ? "success" : "error"}>
|
||||
{svc.running ? "运行中" : "已停止"}
|
||||
</Tag>
|
||||
</Space>
|
||||
}
|
||||
extra={<Tag>:{svc.port}</Tag>}
|
||||
>
|
||||
{svc.running && (
|
||||
<Descriptions size="small" column={3} style={{ marginBottom: 12 }}>
|
||||
<Descriptions.Item label="PID">{svc.pid}</Descriptions.Item>
|
||||
<Descriptions.Item label="运行时长">
|
||||
<ClockCircleOutlined style={{ marginRight: 4 }} />
|
||||
{formatUptime(svc.uptime_seconds)}
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item label="内存">{svc.memory_mb ?? "-"} MB</Descriptions.Item>
|
||||
</Descriptions>
|
||||
)}
|
||||
<Space>
|
||||
{!svc.running && (
|
||||
<Button
|
||||
type="primary"
|
||||
size="small"
|
||||
icon={<PlayCircleOutlined />}
|
||||
loading={actionLoading[`start-${svc.env}`]}
|
||||
onClick={() => handleStart(svc.env)}
|
||||
>
|
||||
启动
|
||||
</Button>
|
||||
)}
|
||||
{svc.running && (
|
||||
<>
|
||||
<Button
|
||||
danger
|
||||
size="small"
|
||||
icon={<PauseCircleOutlined />}
|
||||
loading={actionLoading[`stop-${svc.env}`]}
|
||||
onClick={() => handleStop(svc.env)}
|
||||
>
|
||||
停止
|
||||
</Button>
|
||||
<Button
|
||||
size="small"
|
||||
icon={<ReloadOutlined />}
|
||||
loading={actionLoading[`restart-${svc.env}`]}
|
||||
onClick={() => handleRestart(svc.env)}
|
||||
>
|
||||
重启
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
</Space>
|
||||
</Card>
|
||||
</Col>
|
||||
))}
|
||||
</Row>
|
||||
</Card>
|
||||
|
||||
{/* ---- Git 状态 & 配置 ---- */}
|
||||
<Card size="small" title="代码与配置" style={{ marginBottom: 16 }}>
|
||||
<Row gutter={16}>
|
||||
{gitInfos.map((git) => {
|
||||
const envCfg = services.find((s) => s.env === git.env);
|
||||
const label = envCfg?.label ?? git.env;
|
||||
return (
|
||||
<Col span={12} key={git.env}>
|
||||
<Card size="small" type="inner" title={label}>
|
||||
<Descriptions size="small" column={1} style={{ marginBottom: 12 }}>
|
||||
<Descriptions.Item label="分支">
|
||||
<Tag color="blue">{git.branch}</Tag>
|
||||
{git.has_local_changes && (
|
||||
<Tooltip title="工作区有未提交的变更">
|
||||
<Tag color="warning">有变更</Tag>
|
||||
</Tooltip>
|
||||
)}
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item label="最新提交">
|
||||
<Text code style={{ fontSize: 12 }}>{git.last_commit_hash}</Text>
|
||||
<Text type="secondary" style={{ marginLeft: 8, fontSize: 12 }}>
|
||||
{git.last_commit_message}
|
||||
</Text>
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item label="提交时间">
|
||||
<Text type="secondary" style={{ fontSize: 12 }}>{git.last_commit_time}</Text>
|
||||
</Descriptions.Item>
|
||||
</Descriptions>
|
||||
<Space>
|
||||
<Button
|
||||
size="small"
|
||||
icon={<CloudDownloadOutlined />}
|
||||
loading={actionLoading[`pull-${git.env}`]}
|
||||
onClick={() => handlePull(git.env)}
|
||||
>
|
||||
Git Pull
|
||||
</Button>
|
||||
<Button
|
||||
size="small"
|
||||
icon={<SyncOutlined />}
|
||||
loading={actionLoading[`sync-${git.env}`]}
|
||||
onClick={() => handleSyncDeps(git.env)}
|
||||
>
|
||||
同步依赖
|
||||
</Button>
|
||||
<Button
|
||||
size="small"
|
||||
icon={<FileTextOutlined />}
|
||||
onClick={() => handleViewEnv(git.env, label)}
|
||||
>
|
||||
查看配置
|
||||
</Button>
|
||||
</Space>
|
||||
</Card>
|
||||
</Col>
|
||||
);
|
||||
})}
|
||||
</Row>
|
||||
</Card>
|
||||
|
||||
{/* ---- 配置查看弹窗 ---- */}
|
||||
<Modal
|
||||
title={envModalTitle}
|
||||
open={envModalOpen}
|
||||
onCancel={() => setEnvModalOpen(false)}
|
||||
footer={null}
|
||||
width={700}
|
||||
>
|
||||
<TextArea
|
||||
value={envModalContent}
|
||||
readOnly
|
||||
autoSize={{ minRows: 10, maxRows: 30 }}
|
||||
style={{ fontFamily: "monospace", fontSize: 12 }}
|
||||
/>
|
||||
</Modal>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default OpsPanel;
|
||||
@@ -71,6 +71,7 @@ const FALLBACK_PROCESSING_MODES: ProcModeEntry[] = [
|
||||
{ value: "increment_only", label: "仅增量", desc: "按游标增量抓取和装载" },
|
||||
{ value: "verify_only", label: "校验并修复", desc: "对比源和目标,修复差异" },
|
||||
{ value: "increment_verify", label: "增量+校验", desc: "先增量再校验" },
|
||||
{ value: "full_window", label: "全窗口", desc: "用 API 返回数据的时间范围处理所有层" },
|
||||
];
|
||||
|
||||
/** 将 API 返回的 FlowDef[] 转为 Record<id, FlowEntry> */
|
||||
@@ -243,7 +244,7 @@ const TaskConfig: React.FC = () => {
|
||||
: selectedTasks;
|
||||
return {
|
||||
tasks,
|
||||
pipeline: flow,
|
||||
flow: flow,
|
||||
processing_mode: processingMode,
|
||||
pipeline_flow: "FULL",
|
||||
dry_run: dryRun,
|
||||
@@ -258,7 +259,8 @@ const TaskConfig: React.FC = () => {
|
||||
skip_ods_when_fetch_before_verify: false,
|
||||
ods_use_local_json: useLocalJson,
|
||||
store_id: effectiveStoreId,
|
||||
dwd_only_tables: selectedDwdTables.length > 0 ? selectedDwdTables : null,
|
||||
/* CHANGE [2026-02-19] intent: DWD 表正向勾选,选中=装载 */
|
||||
dwd_only_tables: layers.includes("DWD") ? (selectedDwdTables.length > 0 ? selectedDwdTables : null) : null,
|
||||
force_full: forceFull,
|
||||
extra_args: {},
|
||||
};
|
||||
|
||||
@@ -2,22 +2,27 @@
|
||||
* 任务管理页面。
|
||||
*
|
||||
* 三个 Tab:队列、调度、历史
|
||||
* 队列 Tab:running 状态的任务可点击查看实时 WebSocket 日志流
|
||||
* 历史 Tab:点击记录可查看执行详情和历史日志
|
||||
*/
|
||||
|
||||
import React, { useEffect, useState, useCallback } from 'react';
|
||||
import React, { useEffect, useState, useCallback, useRef } from 'react';
|
||||
import {
|
||||
Tabs, Table, Tag, Button, Popconfirm, Space, message, Drawer,
|
||||
Typography, Descriptions, Empty,
|
||||
Typography, Descriptions, Empty, Spin,
|
||||
} from 'antd';
|
||||
import {
|
||||
ReloadOutlined, DeleteOutlined, StopOutlined,
|
||||
UnorderedListOutlined, ClockCircleOutlined, HistoryOutlined,
|
||||
FileTextOutlined,
|
||||
} from '@ant-design/icons';
|
||||
import type { ColumnsType } from 'antd/es/table';
|
||||
import type { QueuedTask, ExecutionLog } from '../types';
|
||||
import {
|
||||
fetchQueue, fetchHistory, deleteFromQueue, cancelExecution,
|
||||
} from '../api/execution';
|
||||
import { apiClient } from '../api/client';
|
||||
import LogStream from '../components/LogStream';
|
||||
import ScheduleTab from '../components/ScheduleTab';
|
||||
|
||||
const { Title, Text } = Typography;
|
||||
@@ -61,6 +66,13 @@ const QueueTab: React.FC = () => {
|
||||
const [data, setData] = useState<QueuedTask[]>([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
/* WebSocket 日志流状态 */
|
||||
const [logDrawerOpen, setLogDrawerOpen] = useState(false);
|
||||
const [logLines, setLogLines] = useState<string[]>([]);
|
||||
const [logTaskId, setLogTaskId] = useState<string | null>(null);
|
||||
const [wsConnected, setWsConnected] = useState(false);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
setLoading(true);
|
||||
try { setData(await fetchQueue()); }
|
||||
@@ -70,6 +82,51 @@ const QueueTab: React.FC = () => {
|
||||
|
||||
useEffect(() => { load(); }, [load]);
|
||||
|
||||
/* 自动轮询队列状态(5 秒间隔),保持状态实时 */
|
||||
useEffect(() => {
|
||||
const timer = setInterval(load, 5_000);
|
||||
return () => clearInterval(timer);
|
||||
}, [load]);
|
||||
|
||||
/* 组件卸载时关闭 WebSocket */
|
||||
useEffect(() => {
|
||||
return () => { wsRef.current?.close(); };
|
||||
}, []);
|
||||
|
||||
/** 打开日志抽屉并建立 WebSocket 连接 */
|
||||
const handleViewLogs = useCallback((taskId: string) => {
|
||||
setLogTaskId(taskId);
|
||||
setLogLines([]);
|
||||
setLogDrawerOpen(true);
|
||||
|
||||
// 关闭旧连接
|
||||
wsRef.current?.close();
|
||||
|
||||
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
const host = window.location.host;
|
||||
const ws = new WebSocket(`${protocol}//${host}/ws/logs/${taskId}`);
|
||||
wsRef.current = ws;
|
||||
|
||||
ws.onopen = () => { setWsConnected(true); };
|
||||
ws.onmessage = (event) => {
|
||||
setLogLines((prev) => [...prev, event.data]);
|
||||
};
|
||||
ws.onclose = () => { setWsConnected(false); };
|
||||
ws.onerror = () => {
|
||||
message.error('WebSocket 连接失败');
|
||||
setWsConnected(false);
|
||||
};
|
||||
}, []);
|
||||
|
||||
/** 关闭日志抽屉 */
|
||||
const handleCloseLogDrawer = useCallback(() => {
|
||||
setLogDrawerOpen(false);
|
||||
wsRef.current?.close();
|
||||
wsRef.current = null;
|
||||
setWsConnected(false);
|
||||
setLogTaskId(null);
|
||||
}, []);
|
||||
|
||||
const handleDelete = async (id: string) => {
|
||||
try { await deleteFromQueue(id); message.success('已删除'); load(); }
|
||||
catch { message.error('删除失败'); }
|
||||
@@ -90,7 +147,7 @@ const QueueTab: React.FC = () => {
|
||||
),
|
||||
},
|
||||
{
|
||||
title: 'Flow', dataIndex: ['config', 'pipeline'], key: 'pipeline', width: 120,
|
||||
title: 'Flow', dataIndex: ['config', 'flow'], key: 'flow', width: 120,
|
||||
render: (v: string) => <Tag>{v}</Tag>,
|
||||
},
|
||||
{
|
||||
@@ -100,7 +157,7 @@ const QueueTab: React.FC = () => {
|
||||
{ title: '位置', dataIndex: 'position', key: 'position', width: 60, align: 'center' },
|
||||
{ title: '创建时间', dataIndex: 'created_at', key: 'created_at', width: 170, render: fmtTime },
|
||||
{
|
||||
title: '操作', key: 'action', width: 100, align: 'center',
|
||||
title: '操作', key: 'action', width: 160, align: 'center',
|
||||
render: (_: unknown, record: QueuedTask) => {
|
||||
if (record.status === 'pending') {
|
||||
return (
|
||||
@@ -111,9 +168,17 @@ const QueueTab: React.FC = () => {
|
||||
}
|
||||
if (record.status === 'running') {
|
||||
return (
|
||||
<Popconfirm title="确认取消执行?" onConfirm={() => handleCancel(record.id)}>
|
||||
<Button type="link" danger icon={<StopOutlined />} size="small">取消</Button>
|
||||
</Popconfirm>
|
||||
<Space size={4}>
|
||||
<Button
|
||||
type="link" icon={<FileTextOutlined />} size="small"
|
||||
onClick={() => handleViewLogs(record.id)}
|
||||
>
|
||||
日志
|
||||
</Button>
|
||||
<Popconfirm title="确认取消执行?" onConfirm={() => handleCancel(record.id)}>
|
||||
<Button type="link" danger icon={<StopOutlined />} size="small">取消</Button>
|
||||
</Popconfirm>
|
||||
</Space>
|
||||
);
|
||||
}
|
||||
return null;
|
||||
@@ -132,6 +197,29 @@ const QueueTab: React.FC = () => {
|
||||
loading={loading} pagination={false} size="small"
|
||||
locale={{ emptyText: <Empty description="队列为空" /> }}
|
||||
/>
|
||||
|
||||
{/* 实时日志抽屉 */}
|
||||
<Drawer
|
||||
title={
|
||||
<Space>
|
||||
<FileTextOutlined />
|
||||
<span>执行日志</span>
|
||||
{wsConnected
|
||||
? <Tag color="processing">实时连接中</Tag>
|
||||
: <Tag>未连接</Tag>}
|
||||
</Space>
|
||||
}
|
||||
open={logDrawerOpen}
|
||||
onClose={handleCloseLogDrawer}
|
||||
width={720}
|
||||
styles={{ body: { padding: 12, display: 'flex', flexDirection: 'column', height: '100%' } }}
|
||||
>
|
||||
{logTaskId && (
|
||||
<div style={{ flex: 1, minHeight: 0 }}>
|
||||
<LogStream executionId={logTaskId} lines={logLines} />
|
||||
</div>
|
||||
)}
|
||||
</Drawer>
|
||||
</>
|
||||
);
|
||||
};
|
||||
@@ -144,6 +232,8 @@ const HistoryTab: React.FC = () => {
|
||||
const [data, setData] = useState<ExecutionLog[]>([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [detail, setDetail] = useState<ExecutionLog | null>(null);
|
||||
const [historyLogLines, setHistoryLogLines] = useState<string[]>([]);
|
||||
const [logLoading, setLogLoading] = useState(false);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
setLoading(true);
|
||||
@@ -154,6 +244,28 @@ const HistoryTab: React.FC = () => {
|
||||
|
||||
useEffect(() => { load(); }, [load]);
|
||||
|
||||
/** 点击行时加载详情和日志 */
|
||||
const handleRowClick = useCallback(async (record: ExecutionLog) => {
|
||||
setDetail(record);
|
||||
setHistoryLogLines([]);
|
||||
setLogLoading(true);
|
||||
try {
|
||||
const { data: logData } = await apiClient.get<{
|
||||
execution_id: string;
|
||||
output_log: string | null;
|
||||
error_log: string | null;
|
||||
}>(`/execution/${record.id}/logs`);
|
||||
const parts: string[] = [];
|
||||
if (logData.output_log) parts.push(logData.output_log);
|
||||
if (logData.error_log) parts.push(logData.error_log);
|
||||
setHistoryLogLines(parts.join('\n').split('\n').filter(Boolean));
|
||||
} catch {
|
||||
/* 日志可能不存在,静默处理 */
|
||||
} finally {
|
||||
setLogLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const columns: ColumnsType<ExecutionLog> = [
|
||||
{
|
||||
title: '任务', dataIndex: 'task_codes', key: 'task_codes',
|
||||
@@ -187,31 +299,44 @@ const HistoryTab: React.FC = () => {
|
||||
rowKey="id" columns={columns} dataSource={data}
|
||||
loading={loading} pagination={{ pageSize: 20, showTotal: (t) => `共 ${t} 条` }}
|
||||
size="small"
|
||||
onRow={(record) => ({ onClick: () => setDetail(record), style: { cursor: 'pointer' } })}
|
||||
onRow={(record) => ({ onClick: () => handleRowClick(record), style: { cursor: 'pointer' } })}
|
||||
/>
|
||||
|
||||
<Drawer
|
||||
title="执行详情" open={!!detail} onClose={() => setDetail(null)}
|
||||
width={520}
|
||||
width={720}
|
||||
styles={{ body: { padding: 12 } }}
|
||||
>
|
||||
{detail && (
|
||||
<Descriptions column={1} bordered size="small">
|
||||
<Descriptions.Item label="任务">{detail.task_codes?.join(', ')}</Descriptions.Item>
|
||||
<Descriptions.Item label="状态">
|
||||
<Tag color={STATUS_COLOR[detail.status] ?? 'default'}>{detail.status}</Tag>
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item label="开始时间">{fmtTime(detail.started_at)}</Descriptions.Item>
|
||||
<Descriptions.Item label="结束时间">{fmtTime(detail.finished_at)}</Descriptions.Item>
|
||||
<Descriptions.Item label="时长">{fmtDuration(detail.duration_ms)}</Descriptions.Item>
|
||||
<Descriptions.Item label="退出码">
|
||||
{detail.exit_code != null ? (
|
||||
<Tag color={detail.exit_code === 0 ? 'success' : 'error'}>{detail.exit_code}</Tag>
|
||||
) : '—'}
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item label="命令">
|
||||
<code style={{ wordBreak: 'break-all', fontSize: 12 }}>{detail.command || '—'}</code>
|
||||
</Descriptions.Item>
|
||||
</Descriptions>
|
||||
<>
|
||||
<Descriptions column={1} bordered size="small" style={{ marginBottom: 16 }}>
|
||||
<Descriptions.Item label="任务">{detail.task_codes?.join(', ')}</Descriptions.Item>
|
||||
<Descriptions.Item label="状态">
|
||||
<Tag color={STATUS_COLOR[detail.status] ?? 'default'}>{detail.status}</Tag>
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item label="开始时间">{fmtTime(detail.started_at)}</Descriptions.Item>
|
||||
<Descriptions.Item label="结束时间">{fmtTime(detail.finished_at)}</Descriptions.Item>
|
||||
<Descriptions.Item label="时长">{fmtDuration(detail.duration_ms)}</Descriptions.Item>
|
||||
<Descriptions.Item label="退出码">
|
||||
{detail.exit_code != null ? (
|
||||
<Tag color={detail.exit_code === 0 ? 'success' : 'error'}>{detail.exit_code}</Tag>
|
||||
) : '—'}
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item label="命令">
|
||||
<code style={{ wordBreak: 'break-all', fontSize: 12 }}>{detail.command || '—'}</code>
|
||||
</Descriptions.Item>
|
||||
</Descriptions>
|
||||
|
||||
{/* 历史日志展示 */}
|
||||
<div style={{ marginBottom: 8, display: 'flex', alignItems: 'center', gap: 8 }}>
|
||||
<FileTextOutlined />
|
||||
<Text strong>执行日志</Text>
|
||||
{logLoading && <Spin size="small" />}
|
||||
</div>
|
||||
<div style={{ height: 400 }}>
|
||||
<LogStream executionId={detail.id} lines={historyLogLines} />
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</Drawer>
|
||||
</>
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
/** ETL 任务执行配置 */
|
||||
export interface TaskConfig {
|
||||
tasks: string[];
|
||||
/** 执行流程 Flow ID(对应 CLI --pipeline) */
|
||||
pipeline: string;
|
||||
/** 执行流程 Flow ID(对应 CLI --flow) */
|
||||
flow: string;
|
||||
/** 处理模式 */
|
||||
processing_mode: string;
|
||||
/** 传统模式兼容(已弃用) */
|
||||
@@ -36,7 +36,7 @@ export interface TaskConfig {
|
||||
}
|
||||
|
||||
/** 执行流程(Flow)定义 */
|
||||
export interface PipelineDefinition {
|
||||
export interface FlowDefinition {
|
||||
id: string;
|
||||
name: string;
|
||||
/** 包含的层:ODS / DWD / DWS / INDEX */
|
||||
|
||||
@@ -12,7 +12,9 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app import config
|
||||
# CHANGE 2026-02-19 | 新增 xcx_test 路由(MVP 验证)+ wx_callback 路由(微信消息推送)
|
||||
from app.routers import auth, execution, schedules, tasks, env_config, db_viewer, etl_status, xcx_test, wx_callback
|
||||
# CHANGE 2026-02-22 | 新增 member_birthday 路由(助教手动补录会员生日)
|
||||
# CHANGE 2026-02-23 | 新增 ops_panel 路由(运维控制面板)
|
||||
from app.routers import auth, execution, schedules, tasks, env_config, db_viewer, etl_status, xcx_test, wx_callback, member_birthday, ops_panel
|
||||
from app.services.scheduler import scheduler
|
||||
from app.services.task_queue import task_queue
|
||||
from app.ws.logs import ws_router
|
||||
@@ -60,6 +62,8 @@ app.include_router(etl_status.router)
|
||||
app.include_router(ws_router)
|
||||
app.include_router(xcx_test.router)
|
||||
app.include_router(wx_callback.router)
|
||||
app.include_router(member_birthday.router)
|
||||
app.include_router(ops_panel.router)
|
||||
|
||||
|
||||
@app.get("/health", tags=["系统"])
|
||||
|
||||
57
apps/backend/app/routers/member_birthday.py
Normal file
57
apps/backend/app/routers/member_birthday.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""
|
||||
会员生日手动补录路由。
|
||||
|
||||
- POST /api/member-birthday — 助教提交会员生日(UPSERT)
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
|
||||
from app.database import get_connection
|
||||
from app.schemas.member_birthday import MemberBirthdaySubmit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["会员生日"])
|
||||
|
||||
|
||||
@router.post("/member-birthday")
|
||||
async def submit_member_birthday(body: MemberBirthdaySubmit):
|
||||
"""
|
||||
助教提交会员生日(UPSERT)。
|
||||
|
||||
同一 (member_id, assistant_id) 组合重复提交时,
|
||||
更新 birthday_value 和 recorded_at,保留其他助教的记录。
|
||||
"""
|
||||
sql = """
|
||||
INSERT INTO member_birthday_manual
|
||||
(member_id, birthday_value, recorded_by_assistant_id, recorded_by_name, site_id)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
ON CONFLICT (member_id, recorded_by_assistant_id)
|
||||
DO UPDATE SET
|
||||
birthday_value = EXCLUDED.birthday_value,
|
||||
recorded_at = NOW()
|
||||
"""
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (
|
||||
body.member_id,
|
||||
body.birthday_value,
|
||||
body.assistant_id,
|
||||
body.assistant_name,
|
||||
body.site_id,
|
||||
))
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
logger.exception("会员生日 UPSERT 失败: member_id=%s", body.member_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="生日提交失败,请稍后重试",
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return {"status": "ok"}
|
||||
372
apps/backend/app/routers/ops_panel.py
Normal file
372
apps/backend/app/routers/ops_panel.py
Normal file
@@ -0,0 +1,372 @@
|
||||
"""
|
||||
运维控制面板 API
|
||||
|
||||
提供服务器各环境的服务状态查看、启停控制、Git 操作和配置管理。
|
||||
面向管理后台的运维面板页面。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import subprocess
|
||||
import platform
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import psutil
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
router = APIRouter(prefix="/api/ops", tags=["运维面板"])
|
||||
|
||||
# ---- 环境定义 ----
|
||||
# 服务器上的两套环境;开发机上回退到本机路径(方便调试)
|
||||
|
||||
_SERVER_BASE = Path("D:/NeoZQYY")
|
||||
|
||||
ENVIRONMENTS: dict[str, dict[str, Any]] = {
|
||||
"test": {
|
||||
"label": "测试环境",
|
||||
"repo_path": str(_SERVER_BASE / "test" / "repo"),
|
||||
"branch": "test",
|
||||
"port": 8001,
|
||||
"bat_script": str(_SERVER_BASE / "scripts" / "start-test-api.bat"),
|
||||
"window_title": "NeoZQYY Test API",
|
||||
},
|
||||
"prod": {
|
||||
"label": "正式环境",
|
||||
"repo_path": str(_SERVER_BASE / "prod" / "repo"),
|
||||
"branch": "master",
|
||||
"port": 8000,
|
||||
"bat_script": str(_SERVER_BASE / "scripts" / "start-prod-api.bat"),
|
||||
"window_title": "NeoZQYY Prod API",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---- 数据模型 ----
|
||||
|
||||
class ServiceStatus(BaseModel):
|
||||
env: str
|
||||
label: str
|
||||
running: bool
|
||||
pid: int | None = None
|
||||
port: int
|
||||
uptime_seconds: float | None = None
|
||||
memory_mb: float | None = None
|
||||
cpu_percent: float | None = None
|
||||
|
||||
|
||||
class GitInfo(BaseModel):
|
||||
env: str
|
||||
branch: str
|
||||
last_commit_hash: str
|
||||
last_commit_message: str
|
||||
last_commit_time: str
|
||||
has_local_changes: bool
|
||||
|
||||
|
||||
class SystemInfo(BaseModel):
|
||||
cpu_percent: float
|
||||
memory_total_gb: float
|
||||
memory_used_gb: float
|
||||
memory_percent: float
|
||||
disk_total_gb: float
|
||||
disk_used_gb: float
|
||||
disk_percent: float
|
||||
boot_time: str
|
||||
|
||||
|
||||
class EnvFileContent(BaseModel):
|
||||
content: str
|
||||
|
||||
|
||||
class GitPullResult(BaseModel):
|
||||
env: str
|
||||
success: bool
|
||||
output: str
|
||||
|
||||
|
||||
class ServiceActionResult(BaseModel):
|
||||
env: str
|
||||
action: str
|
||||
success: bool
|
||||
message: str
|
||||
|
||||
|
||||
# ---- 辅助函数 ----
|
||||
|
||||
def _find_uvicorn_process(port: int) -> psutil.Process | None:
|
||||
"""查找监听指定端口的 uvicorn 进程。"""
|
||||
for proc in psutil.process_iter(["pid", "name", "cmdline"]):
|
||||
try:
|
||||
cmdline = proc.info.get("cmdline") or []
|
||||
cmdline_str = " ".join(cmdline)
|
||||
# 匹配 uvicorn 进程且包含对应端口
|
||||
if "uvicorn" in cmdline_str and str(port) in cmdline_str:
|
||||
return proc
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _run_cmd(cmd: str | list[str], cwd: str | None = None, timeout: int = 30) -> tuple[bool, str]:
|
||||
"""执行命令并返回 (成功, 输出)。"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
shell=isinstance(cmd, str),
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
output = (result.stdout + "\n" + result.stderr).strip()
|
||||
return result.returncode == 0, output
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "命令执行超时"
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
# ---- 系统信息 ----
|
||||
|
||||
@router.get("/system", response_model=SystemInfo)
|
||||
async def get_system_info():
|
||||
"""获取服务器系统资源概况。"""
|
||||
mem = psutil.virtual_memory()
|
||||
disk = psutil.disk_usage("D:\\") if os.path.exists("D:\\") else psutil.disk_usage("/")
|
||||
boot = datetime.fromtimestamp(psutil.boot_time())
|
||||
return SystemInfo(
|
||||
cpu_percent=psutil.cpu_percent(interval=0.5),
|
||||
memory_total_gb=round(mem.total / (1024 ** 3), 2),
|
||||
memory_used_gb=round(mem.used / (1024 ** 3), 2),
|
||||
memory_percent=mem.percent,
|
||||
disk_total_gb=round(disk.total / (1024 ** 3), 2),
|
||||
disk_used_gb=round(disk.used / (1024 ** 3), 2),
|
||||
disk_percent=disk.percent,
|
||||
boot_time=boot.isoformat(),
|
||||
)
|
||||
|
||||
|
||||
# ---- 服务状态 ----
|
||||
|
||||
@router.get("/services", response_model=list[ServiceStatus])
|
||||
async def get_services_status():
|
||||
"""获取所有环境的服务运行状态。"""
|
||||
results = []
|
||||
for env_key, env_cfg in ENVIRONMENTS.items():
|
||||
proc = _find_uvicorn_process(env_cfg["port"])
|
||||
if proc:
|
||||
try:
|
||||
mem_info = proc.memory_info()
|
||||
create_time = proc.create_time()
|
||||
results.append(ServiceStatus(
|
||||
env=env_key,
|
||||
label=env_cfg["label"],
|
||||
running=True,
|
||||
pid=proc.pid,
|
||||
port=env_cfg["port"],
|
||||
uptime_seconds=round(datetime.now().timestamp() - create_time, 1),
|
||||
memory_mb=round(mem_info.rss / (1024 ** 2), 1),
|
||||
cpu_percent=proc.cpu_percent(interval=0.1),
|
||||
))
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
results.append(ServiceStatus(
|
||||
env=env_key, label=env_cfg["label"],
|
||||
running=False, port=env_cfg["port"],
|
||||
))
|
||||
else:
|
||||
results.append(ServiceStatus(
|
||||
env=env_key, label=env_cfg["label"],
|
||||
running=False, port=env_cfg["port"],
|
||||
))
|
||||
return results
|
||||
|
||||
|
||||
# ---- 服务启停 ----
|
||||
|
||||
@router.post("/services/{env}/start", response_model=ServiceActionResult)
|
||||
async def start_service(env: str):
|
||||
"""启动指定环境的后端服务。"""
|
||||
if env not in ENVIRONMENTS:
|
||||
raise HTTPException(404, f"未知环境: {env}")
|
||||
|
||||
cfg = ENVIRONMENTS[env]
|
||||
proc = _find_uvicorn_process(cfg["port"])
|
||||
if proc:
|
||||
return ServiceActionResult(
|
||||
env=env, action="start", success=True,
|
||||
message=f"服务已在运行中 (PID: {proc.pid})",
|
||||
)
|
||||
|
||||
bat_path = cfg["bat_script"]
|
||||
if not os.path.exists(bat_path):
|
||||
raise HTTPException(400, f"启动脚本不存在: {bat_path}")
|
||||
|
||||
# 通过 start 命令在新窗口中启动 bat 脚本
|
||||
try:
|
||||
subprocess.Popen(
|
||||
f'start "{cfg["window_title"]}" cmd /c "{bat_path}"',
|
||||
shell=True,
|
||||
)
|
||||
# 等待进程启动
|
||||
await asyncio.sleep(3)
|
||||
new_proc = _find_uvicorn_process(cfg["port"])
|
||||
if new_proc:
|
||||
return ServiceActionResult(
|
||||
env=env, action="start", success=True,
|
||||
message=f"服务已启动 (PID: {new_proc.pid})",
|
||||
)
|
||||
else:
|
||||
return ServiceActionResult(
|
||||
env=env, action="start", success=False,
|
||||
message="启动命令已执行,但未检测到进程,请检查日志",
|
||||
)
|
||||
except Exception as e:
|
||||
return ServiceActionResult(
|
||||
env=env, action="start", success=False, message=str(e),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/services/{env}/stop", response_model=ServiceActionResult)
|
||||
async def stop_service(env: str):
|
||||
"""停止指定环境的后端服务。"""
|
||||
if env not in ENVIRONMENTS:
|
||||
raise HTTPException(404, f"未知环境: {env}")
|
||||
|
||||
cfg = ENVIRONMENTS[env]
|
||||
proc = _find_uvicorn_process(cfg["port"])
|
||||
if not proc:
|
||||
return ServiceActionResult(
|
||||
env=env, action="stop", success=True, message="服务未在运行",
|
||||
)
|
||||
|
||||
try:
|
||||
# 终止进程树(包括子进程)
|
||||
parent = psutil.Process(proc.pid)
|
||||
children = parent.children(recursive=True)
|
||||
for child in children:
|
||||
child.terminate()
|
||||
parent.terminate()
|
||||
# 等待进程退出
|
||||
gone, alive = psutil.wait_procs([parent] + children, timeout=5)
|
||||
for p in alive:
|
||||
p.kill()
|
||||
return ServiceActionResult(
|
||||
env=env, action="stop", success=True, message="服务已停止",
|
||||
)
|
||||
except Exception as e:
|
||||
return ServiceActionResult(
|
||||
env=env, action="stop", success=False, message=str(e),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/services/{env}/restart", response_model=ServiceActionResult)
|
||||
async def restart_service(env: str):
|
||||
"""重启指定环境的后端服务。"""
|
||||
stop_result = await stop_service(env)
|
||||
if not stop_result.success and "未在运行" not in stop_result.message:
|
||||
return ServiceActionResult(
|
||||
env=env, action="restart", success=False,
|
||||
message=f"停止失败: {stop_result.message}",
|
||||
)
|
||||
await asyncio.sleep(1)
|
||||
start_result = await start_service(env)
|
||||
return ServiceActionResult(
|
||||
env=env, action="restart",
|
||||
success=start_result.success,
|
||||
message=start_result.message,
|
||||
)
|
||||
|
||||
|
||||
# ---- Git 操作 ----
|
||||
|
||||
@router.get("/git", response_model=list[GitInfo])
|
||||
async def get_git_info():
|
||||
"""获取所有环境的 Git 状态。"""
|
||||
results = []
|
||||
for env_key, env_cfg in ENVIRONMENTS.items():
|
||||
repo = env_cfg["repo_path"]
|
||||
if not os.path.isdir(os.path.join(repo, ".git")):
|
||||
results.append(GitInfo(
|
||||
env=env_key, branch="N/A",
|
||||
last_commit_hash="N/A", last_commit_message="仓库不存在",
|
||||
last_commit_time="", has_local_changes=False,
|
||||
))
|
||||
continue
|
||||
|
||||
_, branch = _run_cmd(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=repo)
|
||||
_, log_out = _run_cmd(
|
||||
["git", "log", "-1", "--format=%H|%s|%ci"],
|
||||
cwd=repo,
|
||||
)
|
||||
_, status_out = _run_cmd(["git", "status", "--porcelain"], cwd=repo)
|
||||
|
||||
parts = log_out.strip().split("|", 2) if log_out else ["", "", ""]
|
||||
results.append(GitInfo(
|
||||
env=env_key,
|
||||
branch=branch.strip(),
|
||||
last_commit_hash=parts[0][:8] if parts[0] else "N/A",
|
||||
last_commit_message=parts[1] if len(parts) > 1 else "",
|
||||
last_commit_time=parts[2] if len(parts) > 2 else "",
|
||||
has_local_changes=bool(status_out.strip()),
|
||||
))
|
||||
return results
|
||||
|
||||
|
||||
@router.post("/git/{env}/pull", response_model=GitPullResult)
|
||||
async def git_pull(env: str):
|
||||
"""对指定环境执行 git pull。"""
|
||||
if env not in ENVIRONMENTS:
|
||||
raise HTTPException(404, f"未知环境: {env}")
|
||||
|
||||
cfg = ENVIRONMENTS[env]
|
||||
repo = cfg["repo_path"]
|
||||
if not os.path.isdir(os.path.join(repo, ".git")):
|
||||
raise HTTPException(400, f"仓库路径不存在: {repo}")
|
||||
|
||||
success, output = _run_cmd(["git", "pull", "--ff-only"], cwd=repo, timeout=60)
|
||||
return GitPullResult(env=env, success=success, output=output)
|
||||
|
||||
|
||||
@router.post("/git/{env}/sync-deps", response_model=ServiceActionResult)
|
||||
async def sync_deps(env: str):
|
||||
"""对指定环境执行 uv sync --all-packages。"""
|
||||
if env not in ENVIRONMENTS:
|
||||
raise HTTPException(404, f"未知环境: {env}")
|
||||
|
||||
cfg = ENVIRONMENTS[env]
|
||||
repo = cfg["repo_path"]
|
||||
success, output = _run_cmd(["uv", "sync", "--all-packages"], cwd=repo, timeout=120)
|
||||
return ServiceActionResult(
|
||||
env=env, action="sync-deps", success=success, message=output[:500],
|
||||
)
|
||||
|
||||
|
||||
# ---- 环境配置管理 ----
|
||||
|
||||
@router.get("/env-file/{env}")
|
||||
async def get_env_file(env: str):
|
||||
"""读取指定环境的 .env 文件(敏感值脱敏)。"""
|
||||
if env not in ENVIRONMENTS:
|
||||
raise HTTPException(404, f"未知环境: {env}")
|
||||
|
||||
env_path = Path(ENVIRONMENTS[env]["repo_path"]) / ".env"
|
||||
if not env_path.exists():
|
||||
raise HTTPException(404, f".env 文件不存在: {env_path}")
|
||||
|
||||
lines = env_path.read_text(encoding="utf-8").splitlines()
|
||||
masked_lines = []
|
||||
sensitive_keys = {"PASSWORD", "SECRET", "TOKEN", "DSN", "APP_SECRET"}
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if stripped and not stripped.startswith("#") and "=" in stripped:
|
||||
key = stripped.split("=", 1)[0].strip()
|
||||
if any(s in key.upper() for s in sensitive_keys):
|
||||
masked_lines.append(f"{key}=********")
|
||||
continue
|
||||
masked_lines.append(line)
|
||||
return {"env": env, "content": "\n".join(masked_lines)}
|
||||
@@ -4,7 +4,7 @@
|
||||
提供 4 个端点:
|
||||
- GET /api/tasks/registry — 按业务域分组的任务列表
|
||||
- GET /api/tasks/dwd-tables — 按业务域分组的 DWD 表定义
|
||||
- GET /api/tasks/flows — 7 种 Flow + 3 种处理模式
|
||||
- GET /api/tasks/flows — 7 种 Flow + 4 种处理模式
|
||||
- POST /api/tasks/validate — 验证 TaskConfig 并返回 CLI 命令预览
|
||||
|
||||
所有端点需要 JWT 认证。validate 端点从 JWT 注入 store_id。
|
||||
@@ -103,6 +103,7 @@ PROCESSING_MODE_DEFINITIONS: list[ProcessingModeDefinition] = [
|
||||
ProcessingModeDefinition(id="increment_only", name="仅增量处理", description="只处理新增和变更的数据"),
|
||||
ProcessingModeDefinition(id="verify_only", name="仅校验修复", description="校验现有数据并修复不一致"),
|
||||
ProcessingModeDefinition(id="increment_verify", name="增量 + 校验修复", description="先增量处理,再校验并修复"),
|
||||
ProcessingModeDefinition(id="full_window", name="全窗口处理", description="用 API 返回数据的实际时间范围处理全部层,无需校验"),
|
||||
]
|
||||
|
||||
|
||||
@@ -163,7 +164,7 @@ async def get_dwd_tables(
|
||||
async def get_flows(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> FlowsResponse:
|
||||
"""返回 7 种 Flow 定义和 3 种处理模式定义"""
|
||||
"""返回 7 种 Flow 定义和 4 种处理模式定义"""
|
||||
return FlowsResponse(
|
||||
flows=FLOW_DEFINITIONS,
|
||||
processing_modes=PROCESSING_MODE_DEFINITIONS,
|
||||
@@ -183,8 +184,9 @@ async def validate_task_config(
|
||||
errors: list[str] = []
|
||||
|
||||
# 验证 Flow ID
|
||||
if config.pipeline not in FLOW_LAYER_MAP:
|
||||
errors.append(f"无效的执行流程: {config.pipeline}")
|
||||
# CHANGE [2026-02-20] intent: pipeline → flow,统一命名
|
||||
if config.flow not in FLOW_LAYER_MAP:
|
||||
errors.append(f"无效的执行流程: {config.flow}")
|
||||
|
||||
# 验证任务列表非空
|
||||
if not config.tasks:
|
||||
|
||||
19
apps/backend/app/schemas/member_birthday.py
Normal file
19
apps/backend/app/schemas/member_birthday.py
Normal file
@@ -0,0 +1,19 @@
|
||||
"""
|
||||
会员生日手动补录相关 Pydantic 模型。
|
||||
|
||||
- MemberBirthdaySubmit:助教提交会员生日请求体
|
||||
"""
|
||||
|
||||
from datetime import date
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class MemberBirthdaySubmit(BaseModel):
|
||||
"""助教提交会员生日请求。"""
|
||||
|
||||
member_id: int = Field(..., gt=0, description="会员 ID")
|
||||
birthday_value: date = Field(..., description="生日日期")
|
||||
assistant_id: int = Field(..., gt=0, description="助教 ID")
|
||||
assistant_name: str = Field(..., min_length=1, max_length=50, description="助教姓名")
|
||||
site_id: int = Field(..., gt=0, description="门店 ID")
|
||||
@@ -13,8 +13,8 @@ class TaskConfigSchema(BaseModel):
|
||||
"""任务配置 — 前后端传输格式
|
||||
|
||||
字段与 CLI 参数的映射关系:
|
||||
- pipeline → --pipeline(Flow ID,7 种之一)
|
||||
- processing_mode → --processing-mode(3 种处理模式)
|
||||
- flow → --flow(Flow ID,7 种之一)
|
||||
- processing_mode → --processing-mode(4 种处理模式)
|
||||
- tasks → --tasks(逗号分隔)
|
||||
- dry_run → --dry-run(布尔标志)
|
||||
- window_mode → 决定使用 lookback 还是 custom 时间窗口(仅前端逻辑,不直接映射 CLI 参数)
|
||||
@@ -30,7 +30,8 @@ class TaskConfigSchema(BaseModel):
|
||||
"""
|
||||
|
||||
tasks: list[str]
|
||||
pipeline: str = "api_ods_dwd"
|
||||
# CHANGE [2026-02-20] intent: pipeline → flow,统一命名(消除历史别名)
|
||||
flow: str = "api_ods_dwd"
|
||||
processing_mode: str = "increment_only"
|
||||
dry_run: bool = False
|
||||
window_mode: str = "lookback"
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
支持:
|
||||
- 7 种 Flow(api_ods / api_ods_dwd / api_full / ods_dwd / dwd_dws / dwd_dws_index / dwd_index)
|
||||
- 3 种处理模式(increment_only / verify_only / increment_verify)
|
||||
- 4 种处理模式(increment_only / verify_only / increment_verify / full_window)
|
||||
- 自动注入 --store-id 参数
|
||||
"""
|
||||
|
||||
@@ -30,6 +30,7 @@ VALID_PROCESSING_MODES: set[str] = {
|
||||
"increment_only",
|
||||
"verify_only",
|
||||
"increment_verify",
|
||||
"full_window",
|
||||
}
|
||||
|
||||
# CLI 支持的 extra_args 键(值类型 + 布尔类型)
|
||||
@@ -72,7 +73,8 @@ class CLIBuilder:
|
||||
cmd: list[str] = [python_executable, "-m", "cli.main"]
|
||||
|
||||
# -- Flow(执行流程) --
|
||||
cmd.extend(["--flow", config.pipeline])
|
||||
# CHANGE [2026-02-20] intent: pipeline → flow,统一命名
|
||||
cmd.extend(["--flow", config.flow])
|
||||
|
||||
# -- 处理模式 --
|
||||
if config.processing_mode:
|
||||
|
||||
@@ -46,7 +46,7 @@ ODS_TASKS: list[TaskDefinition] = [
|
||||
TaskDefinition("ODS_ASSISTANT_LEDGER", "助教服务记录", "抽取助教服务流水", "助教", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_ASSISTANT_ABOLISH", "助教取消记录", "抽取助教取消/作废记录", "助教", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_SETTLEMENT_RECORDS", "结算记录", "抽取订单结算记录", "结算", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_SETTLEMENT_TICKET", "结账小票", "抽取结账小票明细", "结算", "ODS", is_ods=True),
|
||||
# CHANGE [2026-07-20] intent: 同步 ETL 侧移除——ODS_SETTLEMENT_TICKET 已在 Task 7.3 中彻底移除
|
||||
TaskDefinition("ODS_TABLE_USE", "台费流水", "抽取台费使用流水", "台桌", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_TABLE_FEE_DISCOUNT", "台费折扣", "抽取台费折扣记录", "台桌", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_TABLES", "台桌主数据", "抽取门店台桌信息", "台桌", "ODS", is_ods=True, requires_window=False),
|
||||
@@ -59,7 +59,7 @@ ODS_TASKS: list[TaskDefinition] = [
|
||||
TaskDefinition("ODS_RECHARGE_SETTLE", "充值结算", "抽取充值结算记录", "会员", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_GROUP_PACKAGE", "团购套餐", "抽取团购套餐定义", "团购", "ODS", is_ods=True, requires_window=False),
|
||||
TaskDefinition("ODS_GROUP_BUY_REDEMPTION", "团购核销", "抽取团购核销记录", "团购", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_INVENTORY_STOCK", "库存快照", "抽取商品库存汇总", "库存", "ODS", is_ods=True, requires_window=False),
|
||||
TaskDefinition("ODS_INVENTORY_STOCK", "库存快照", "抽取商品库存汇总", "库存", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_INVENTORY_CHANGE", "库存变动", "抽取库存出入库记录", "库存", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_GOODS_CATEGORY", "商品分类", "抽取商品分类树", "商品", "ODS", is_ods=True, requires_window=False),
|
||||
TaskDefinition("ODS_STORE_GOODS", "门店商品", "抽取门店商品主数据", "商品", "ODS", is_ods=True, requires_window=False),
|
||||
@@ -91,6 +91,10 @@ DWS_TASKS: list[TaskDefinition] = [
|
||||
TaskDefinition("DWS_FINANCE_DISCOUNT_DETAIL", "折扣明细", "汇总折扣明细", "财务", "DWS"),
|
||||
# CHANGE [2026-02-19] intent: 同步 ETL 侧合并——原 DWS_RETENTION_CLEANUP / DWS_MV_REFRESH_* 已合并为 DWS_MAINTENANCE
|
||||
TaskDefinition("DWS_MAINTENANCE", "DWS 维护", "刷新物化视图 + 清理过期留存数据", "通用", "DWS", requires_window=False, is_common=False),
|
||||
# CHANGE [2026-07-20] intent: 注册 DWS 库存汇总任务(日/周/月),依赖 DWD goods_stock_summary 加载完成(需求 12.9)
|
||||
TaskDefinition("DWS_GOODS_STOCK_DAILY", "库存日报", "按日粒度汇总商品库存数据", "库存", "DWS"),
|
||||
TaskDefinition("DWS_GOODS_STOCK_WEEKLY", "库存周报", "按周粒度汇总商品库存数据", "库存", "DWS"),
|
||||
TaskDefinition("DWS_GOODS_STOCK_MONTHLY", "库存月报", "按月粒度汇总商品库存数据", "库存", "DWS"),
|
||||
]
|
||||
|
||||
# ── INDEX 任务定义 ────────────────────────────────────────────
|
||||
@@ -99,7 +103,8 @@ INDEX_TASKS: list[TaskDefinition] = [
|
||||
TaskDefinition("DWS_WINBACK_INDEX", "回流指数 (WBI)", "计算会员回流指数", "指数", "INDEX"),
|
||||
TaskDefinition("DWS_NEWCONV_INDEX", "新客转化指数 (NCI)", "计算新客转化指数", "指数", "INDEX"),
|
||||
TaskDefinition("DWS_ML_MANUAL_IMPORT", "手动导入 (ML)", "手动导入机器学习数据", "指数", "INDEX", requires_window=False, is_common=False),
|
||||
TaskDefinition("DWS_RELATION_INDEX", "关系指数 (RS)", "计算助教-客户关系指数", "指数", "INDEX"),
|
||||
# CHANGE [2026-02-19] intent: 补充说明 RelationIndexTask 产出 RS/OS/MS/ML 四个子指数
|
||||
TaskDefinition("DWS_RELATION_INDEX", "关系指数 (RS)", "产出 RS/OS/MS/ML 四个子指数", "指数", "INDEX"),
|
||||
]
|
||||
|
||||
# ── 工具类任务定义 ────────────────────────────────────────────
|
||||
@@ -210,6 +215,9 @@ DWD_TABLES: list[DwdTableDefinition] = [
|
||||
DwdTableDefinition("dwd.dwd_payment", "支付流水", "结算", "ods.payment_transactions"),
|
||||
DwdTableDefinition("dwd.dwd_refund", "退款流水", "结算", "ods.refund_transactions"),
|
||||
DwdTableDefinition("dwd.dwd_refund_ex", "退款流水(扩展)", "结算", "ods.refund_transactions"),
|
||||
# CHANGE [2026-07-20] intent: 同步 Task 6.1/6.2 新建的 DWD 库存表
|
||||
DwdTableDefinition("dwd.dwd_goods_stock_summary", "库存汇总", "库存", "ods.goods_stock_summary"),
|
||||
DwdTableDefinition("dwd.dwd_goods_stock_movement", "库存变动", "库存", "ods.goods_stock_movements"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ dependencies = [
|
||||
"python-dotenv>=1.0",
|
||||
"python-jose[cryptography]>=3.3",
|
||||
"bcrypt>=4.0",
|
||||
"psutil>=5.9",
|
||||
]
|
||||
|
||||
[tool.uv.sources]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""CLIBuilder 单元测试
|
||||
|
||||
覆盖:7 种 Flow、3 种处理模式、时间窗口、store_id 自动注入、extra_args 等。
|
||||
覆盖:7 种 Flow、4 种处理模式、时间窗口、store_id 自动注入、extra_args 等。
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -24,11 +24,11 @@ ETL_PATH = "/fake/etl/project"
|
||||
|
||||
class TestBasicCommand:
|
||||
def test_minimal_command(self, builder: CLIBuilder):
|
||||
"""最小配置应生成 python -m cli.main --pipeline ... --processing-mode ..."""
|
||||
"""最小配置应生成 python -m cli.main --flow ... --processing-mode ..."""
|
||||
config = TaskConfigSchema(tasks=["ODS_MEMBER"])
|
||||
cmd = builder.build_command(config, ETL_PATH)
|
||||
assert cmd[:3] == ["python", "-m", "cli.main"]
|
||||
assert "--pipeline" in cmd
|
||||
assert "--flow" in cmd
|
||||
assert "--processing-mode" in cmd
|
||||
|
||||
def test_custom_python_executable(self, builder: CLIBuilder):
|
||||
@@ -56,20 +56,20 @@ class TestBasicCommand:
|
||||
class TestFlows:
|
||||
@pytest.mark.parametrize("flow_id", sorted(VALID_FLOWS))
|
||||
def test_all_flows_accepted(self, builder: CLIBuilder, flow_id: str):
|
||||
config = TaskConfigSchema(tasks=["ODS_MEMBER"], pipeline=flow_id)
|
||||
config = TaskConfigSchema(tasks=["ODS_MEMBER"], flow=flow_id)
|
||||
cmd = builder.build_command(config, ETL_PATH)
|
||||
idx = cmd.index("--pipeline")
|
||||
idx = cmd.index("--flow")
|
||||
assert cmd[idx + 1] == flow_id
|
||||
|
||||
def test_default_flow_is_api_ods_dwd(self, builder: CLIBuilder):
|
||||
config = TaskConfigSchema(tasks=["ODS_MEMBER"])
|
||||
cmd = builder.build_command(config, ETL_PATH)
|
||||
idx = cmd.index("--pipeline")
|
||||
idx = cmd.index("--flow")
|
||||
assert cmd[idx + 1] == "api_ods_dwd"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3 种处理模式
|
||||
# 4 种处理模式
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestProcessingModes:
|
||||
|
||||
@@ -36,7 +36,7 @@ _NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
# 构造测试用的 TaskConfig payload
|
||||
_VALID_CONFIG = {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
}
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ class TestRunTask:
|
||||
|
||||
def test_run_invalid_config_returns_422(self):
|
||||
"""缺少必填字段 tasks 时返回 422"""
|
||||
resp = client.post("/api/execution/run", json={"pipeline": "api_ods"})
|
||||
resp = client.post("/api/execution/run", json={"flow": "api_ods"})
|
||||
assert resp.status_code == 422
|
||||
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ _task_codes = ["ODS_MEMBER", "ODS_PAYMENT", "ODS_ORDER", "DWD_LOAD_FROM_ODS", "D
|
||||
_simple_config_st = st.builds(
|
||||
TaskConfigSchema,
|
||||
tasks=st.lists(st.sampled_from(_task_codes), min_size=1, max_size=3, unique=True),
|
||||
pipeline=st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd"]),
|
||||
flow=st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd"]),
|
||||
)
|
||||
|
||||
|
||||
@@ -254,7 +254,7 @@ def test_queue_crud_invariant(mock_get_conn, config, site_id, initial_count):
|
||||
db.rows[tid] = {
|
||||
"id": tid,
|
||||
"site_id": site_id,
|
||||
"config": {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"},
|
||||
"config": {"tasks": ["ODS_MEMBER"], "flow": "api_ods"},
|
||||
"status": "pending",
|
||||
"position": i + 1,
|
||||
}
|
||||
@@ -322,7 +322,7 @@ def test_queue_dequeue_order(mock_get_conn, site_id, num_tasks, positions):
|
||||
db.rows[tid] = {
|
||||
"id": tid,
|
||||
"site_id": site_id,
|
||||
"config": {"tasks": [_task_codes[i % len(_task_codes)]], "pipeline": "api_ods"},
|
||||
"config": {"tasks": [_task_codes[i % len(_task_codes)]], "flow": "api_ods"},
|
||||
"status": "pending",
|
||||
"position": pos,
|
||||
}
|
||||
@@ -372,7 +372,7 @@ def test_queue_reorder_consistency(mock_get_conn, site_id, num_tasks, data):
|
||||
db.rows[tid] = {
|
||||
"id": tid,
|
||||
"site_id": site_id,
|
||||
"config": {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"},
|
||||
"config": {"tasks": ["ODS_MEMBER"], "flow": "api_ods"},
|
||||
"status": "pending",
|
||||
"position": i + 1,
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ _task_codes = ["ODS_MEMBER", "ODS_PAYMENT", "ODS_ORDER", "DWD_LOAD_FROM_ODS", "D
|
||||
|
||||
_simple_task_config_st = st.fixed_dictionaries({
|
||||
"tasks": st.lists(st.sampled_from(_task_codes), min_size=1, max_size=3, unique=True),
|
||||
"pipeline": st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd", "api_full"]),
|
||||
"flow": st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd", "api_full"]),
|
||||
})
|
||||
|
||||
# 调度配置策略:覆盖 5 种调度类型
|
||||
@@ -324,8 +324,8 @@ def test_due_schedule_auto_enqueue(
|
||||
assert enqueued_config.tasks == task_config["tasks"], (
|
||||
f"入队的 tasks 应为 {task_config['tasks']},实际 {enqueued_config.tasks}"
|
||||
)
|
||||
assert enqueued_config.pipeline == task_config["pipeline"], (
|
||||
f"入队的 pipeline 应为 {task_config['pipeline']},实际 {enqueued_config.pipeline}"
|
||||
assert enqueued_config.flow == task_config["flow"], (
|
||||
f"入队的 flow 应为 {task_config['flow']},实际 {enqueued_config.flow}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -230,7 +230,7 @@ class TestCheckAndEnqueue:
|
||||
@patch("app.services.scheduler.task_queue")
|
||||
def test_enqueues_due_tasks(self, mock_tq, mock_get_conn, sched):
|
||||
"""到期任务应被入队,且更新 last_run_at / run_count / next_run_at"""
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
|
||||
schedule_config = {
|
||||
"schedule_type": "interval",
|
||||
"interval_value": 1,
|
||||
@@ -280,7 +280,7 @@ class TestCheckAndEnqueue:
|
||||
def test_skips_invalid_config(self, mock_tq, mock_get_conn, sched):
|
||||
"""配置反序列化失败的任务应被跳过"""
|
||||
# task_config 缺少必填字段 tasks
|
||||
bad_config = {"pipeline": "api_ods_dwd"}
|
||||
bad_config = {"flow": "api_ods_dwd"}
|
||||
schedule_config = {"schedule_type": "once"}
|
||||
|
||||
cur = _mock_cursor(
|
||||
@@ -300,7 +300,7 @@ class TestCheckAndEnqueue:
|
||||
@patch("app.services.scheduler.task_queue")
|
||||
def test_enqueue_failure_continues(self, mock_tq, mock_get_conn, sched):
|
||||
"""入队失败时应跳过该任务,继续处理后续任务"""
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
|
||||
schedule_config = {"schedule_type": "once"}
|
||||
|
||||
cur = _mock_cursor(
|
||||
@@ -327,7 +327,7 @@ class TestCheckAndEnqueue:
|
||||
@patch("app.services.scheduler.task_queue")
|
||||
def test_once_type_sets_next_run_none(self, mock_tq, mock_get_conn, sched):
|
||||
"""once 类型任务入队后,next_run_at 应被设为 NULL"""
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
|
||||
schedule_config = {"schedule_type": "once"}
|
||||
|
||||
select_cur = _mock_cursor(
|
||||
|
||||
@@ -40,14 +40,14 @@ _SCHEDULE_CONFIG = {
|
||||
_VALID_CREATE = {
|
||||
"name": "每日全量同步",
|
||||
"task_codes": ["ODS_MEMBER", "ODS_ORDER"],
|
||||
"task_config": {"tasks": ["ODS_MEMBER", "ODS_ORDER"], "pipeline": "api_ods"},
|
||||
"task_config": {"tasks": ["ODS_MEMBER", "ODS_ORDER"], "flow": "api_ods"},
|
||||
"schedule_config": _SCHEDULE_CONFIG,
|
||||
}
|
||||
|
||||
# 模拟数据库返回的完整行(13 列,与 _SELECT_COLS 对应)
|
||||
_DB_ROW = (
|
||||
"sched-1", 100, "每日全量同步", ["ODS_MEMBER", "ODS_ORDER"],
|
||||
json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}),
|
||||
json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"}),
|
||||
json.dumps(_SCHEDULE_CONFIG),
|
||||
True, None, _NEXT, 0, None, _NOW, _NOW,
|
||||
)
|
||||
|
||||
@@ -53,7 +53,7 @@ def _make_queue_rows(site_id: int, count: int) -> list[tuple]:
|
||||
rows.append((
|
||||
str(uuid.uuid4()), # id
|
||||
site_id, # site_id
|
||||
json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}), # config
|
||||
json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"}), # config
|
||||
"pending", # status
|
||||
i + 1, # position
|
||||
datetime(2024, 1, 1, tzinfo=timezone.utc), # created_at
|
||||
@@ -75,7 +75,7 @@ def _make_schedule_rows(site_id: int, count: int) -> list[tuple]:
|
||||
site_id, # site_id
|
||||
f"调度任务_{i}", # name
|
||||
["ODS_MEMBER"], # task_codes
|
||||
{"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}, # task_config
|
||||
{"tasks": ["ODS_MEMBER"], "flow": "api_ods"}, # task_config
|
||||
{"schedule_type": "daily", "daily_time": "04:00", # schedule_config
|
||||
"interval_value": 1, "interval_unit": "hours",
|
||||
"weekly_days": [1], "weekly_time": "04:00",
|
||||
|
||||
@@ -31,7 +31,7 @@ _tasks_st = st.lists(
|
||||
unique=True,
|
||||
)
|
||||
|
||||
_pipeline_st = st.sampled_from(sorted(VALID_FLOWS))
|
||||
_flow_st = st.sampled_from(sorted(VALID_FLOWS))
|
||||
_processing_mode_st = st.sampled_from(sorted(VALID_PROCESSING_MODES))
|
||||
_window_mode_st = st.sampled_from(["lookback", "custom"])
|
||||
|
||||
@@ -69,7 +69,7 @@ def _valid_task_config_st():
|
||||
@st.composite
|
||||
def _build(draw):
|
||||
tasks = draw(_tasks_st)
|
||||
pipeline = draw(_pipeline_st)
|
||||
flow_id = draw(_flow_st)
|
||||
processing_mode = draw(_processing_mode_st)
|
||||
dry_run = draw(st.booleans())
|
||||
window_mode = draw(_window_mode_st)
|
||||
@@ -103,7 +103,7 @@ def _valid_task_config_st():
|
||||
|
||||
return TaskConfigSchema(
|
||||
tasks=tasks,
|
||||
pipeline=pipeline,
|
||||
flow=flow_id,
|
||||
processing_mode=processing_mode,
|
||||
dry_run=dry_run,
|
||||
window_mode=window_mode,
|
||||
@@ -204,10 +204,10 @@ def test_task_config_to_cli_completeness(config: TaskConfigSchema):
|
||||
"""Property 7: CLIBuilder 生成的命令应包含 TaskConfig 中所有非空字段对应的 CLI 参数。"""
|
||||
cmd = _builder.build_command(config, _ETL_PATH)
|
||||
|
||||
# 1) --pipeline 始终存在且值正确
|
||||
assert "--pipeline" in cmd
|
||||
idx = cmd.index("--pipeline")
|
||||
assert cmd[idx + 1] == config.pipeline
|
||||
# 1) --flow 始终存在且值正确
|
||||
assert "--flow" in cmd
|
||||
idx = cmd.index("--flow")
|
||||
assert cmd[idx + 1] == config.flow
|
||||
|
||||
# 2) --processing-mode 始终存在且值正确
|
||||
assert "--processing-mode" in cmd
|
||||
|
||||
@@ -24,7 +24,7 @@ def executor() -> TaskExecutor:
|
||||
def sample_config() -> TaskConfigSchema:
|
||||
return TaskConfigSchema(
|
||||
tasks=["ODS_MEMBER", "ODS_PAYMENT"],
|
||||
pipeline="api_ods_dwd",
|
||||
flow="api_ods_dwd",
|
||||
store_id=42,
|
||||
)
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ def queue() -> TaskQueue:
|
||||
def sample_config() -> TaskConfigSchema:
|
||||
return TaskConfigSchema(
|
||||
tasks=["ODS_MEMBER", "ODS_PAYMENT"],
|
||||
pipeline="api_ods_dwd",
|
||||
flow="api_ods_dwd",
|
||||
store_id=42,
|
||||
)
|
||||
|
||||
@@ -107,7 +107,7 @@ class TestEnqueue:
|
||||
config_json_str = insert_call[0][1][2]
|
||||
parsed = json.loads(config_json_str)
|
||||
assert parsed["tasks"] == ["ODS_MEMBER", "ODS_PAYMENT"]
|
||||
assert parsed["pipeline"] == "api_ods_dwd"
|
||||
assert parsed["flow"] == "api_ods_dwd"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -129,7 +129,7 @@ class TestDequeue:
|
||||
@patch("app.services.task_queue.get_connection")
|
||||
def test_dequeue_returns_task(self, mock_get_conn, queue):
|
||||
task_id = str(uuid.uuid4())
|
||||
config_dict = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}
|
||||
config_dict = {"tasks": ["ODS_MEMBER"], "flow": "api_ods"}
|
||||
row = (
|
||||
task_id, 42, json.dumps(config_dict), "pending", 1,
|
||||
None, None, None, None, None,
|
||||
@@ -149,7 +149,7 @@ class TestDequeue:
|
||||
@patch("app.services.task_queue.get_connection")
|
||||
def test_dequeue_updates_status_to_running(self, mock_get_conn, queue):
|
||||
task_id = str(uuid.uuid4())
|
||||
config_dict = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}
|
||||
config_dict = {"tasks": ["ODS_MEMBER"], "flow": "api_ods"}
|
||||
row = (
|
||||
task_id, 42, json.dumps(config_dict), "pending", 1,
|
||||
None, None, None, None, None,
|
||||
@@ -285,7 +285,7 @@ class TestQuery:
|
||||
@patch("app.services.task_queue.get_connection")
|
||||
def test_list_pending_returns_tasks(self, mock_get_conn, queue):
|
||||
tid = str(uuid.uuid4())
|
||||
config = json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"})
|
||||
config = json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"})
|
||||
rows = [(tid, 42, config, "pending", 1, None, None, None, None, None)]
|
||||
cur = _mock_cursor(fetchall_val=rows)
|
||||
conn = _mock_conn(cur)
|
||||
@@ -353,7 +353,7 @@ class TestProcessLoop:
|
||||
task_id = str(uuid.uuid4())
|
||||
config_dict = {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "api_ods_dwd",
|
||||
"flow": "api_ods_dwd",
|
||||
"processing_mode": "increment_only",
|
||||
"dry_run": False,
|
||||
"window_mode": "lookback",
|
||||
|
||||
@@ -152,7 +152,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["ODS_MEMBER", "ODS_PAYMENT"],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
}
|
||||
})
|
||||
assert resp.status_code == 200
|
||||
@@ -169,7 +169,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["DWD_LOAD_FROM_ODS"],
|
||||
"pipeline": "ods_dwd",
|
||||
"flow": "ods_dwd",
|
||||
"store_id": 999,
|
||||
}
|
||||
})
|
||||
@@ -184,7 +184,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "nonexistent_flow",
|
||||
"flow": "nonexistent_flow",
|
||||
}
|
||||
})
|
||||
assert resp.status_code == 200
|
||||
@@ -196,7 +196,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": [],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
}
|
||||
})
|
||||
assert resp.status_code == 200
|
||||
@@ -208,7 +208,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
"window_mode": "custom",
|
||||
"window_start": "2024-01-01",
|
||||
"window_end": "2024-01-31",
|
||||
@@ -225,7 +225,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
"window_mode": "custom",
|
||||
"window_start": "2024-12-31",
|
||||
"window_end": "2024-01-01",
|
||||
@@ -237,7 +237,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
"dry_run": True,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -26,17 +26,18 @@ SCHEMA_ETL=meta
|
||||
# API 配置(上游 SaaS API)
|
||||
# ------------------------------------------------------------------------------
|
||||
API_BASE=https://pc.ficoo.vip/apiprod/admin/v1/
|
||||
API_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnQtdHlwZSI6IjQiLCJ1c2VyLXR5cGUiOiIxIiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjoiMTIiLCJyb2xlLWlkIjoiMTIiLCJ0ZW5hbnQtaWQiOiIyNzkwNjgzMTYwNzA5OTU3Iiwibmlja25hbWUiOiLnp5_miLfnrqHnkIblkZjvvJrmganmgakxIiwic2l0ZS1pZCI6IjAiLCJtb2JpbGUiOiIxMzgxMDUwMjMwNCIsInNpZCI6IjI5NTA0ODk2NTgzOTU4NDUiLCJzdGFmZi1pZCI6IjMwMDk5MTg2OTE1NTkwNDUiLCJvcmctaWQiOiIwIiwicm9sZS10eXBlIjoiMyIsInJlZnJlc2hUb2tlbiI6InoxazVzWjlDeEFKYnFkNG1pT3NwUzBsQTRMYUNGcURkQjJBdFdsQk1DbDA9IiwicmVmcmVzaEV4cGlyeVRpbWUiOiIyMDI2LzIvMjIg5LiL5Y2IMTE6NTk6MzAiLCJuZWVkQ2hlY2tUb2tlbiI6ImZhbHNlIiwiZXhwIjoxNzcxNzc1OTcwLCJpc3MiOiJ0ZXN0IiwiYXVkIjoiVXNlciJ9.27D1QgKFYGgMKR9bS5NbCSl4kIf9oFVOQLsFl_ITxdI
|
||||
API_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnQtdHlwZSI6IjQiLCJ1c2VyLXR5cGUiOiIxIiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjoiMTIiLCJyb2xlLWlkIjoiMTIiLCJ0ZW5hbnQtaWQiOiIyNzkwNjgzMTYwNzA5OTU3Iiwibmlja25hbWUiOiLnp5_miLfnrqHnkIblkZjvvJrmganmgakxIiwic2l0ZS1pZCI6IjAiLCJtb2JpbGUiOiIxMzgxMDUwMjMwNCIsInNpZCI6IjI5NTA0ODk2NTgzOTU4NDUiLCJzdGFmZi1pZCI6IjMwMDk5MTg2OTE1NTkwNDUiLCJvcmctaWQiOiIwIiwicm9sZS10eXBlIjoiMyIsInJlZnJlc2hUb2tlbiI6IjN4d3IwYjNWN01jemlvcFYyZnZibmtpMVg4MEhxNVFvOFRMcHh3RkNkQUk9IiwicmVmcmVzaEV4cGlyeVRpbWUiOiIyMDI2LzMvMSDkuIvljYgxMDo1MDozOCIsIm5lZWRDaGVja1Rva2VuIjoiZmFsc2UiLCJleHAiOjE3NzIzNzY2MzgsImlzcyI6InRlc3QiLCJhdWQiOiJVc2VyIn0.k_f4jnSGKOKPoZC22bVSrAo9A1FfRqvsNiGw-Vmc0qQ
|
||||
API_TIMEOUT=20
|
||||
API_PAGE_SIZE=200
|
||||
API_RETRY_MAX=3
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# 路径配置(已更新为 NeoZQYY 路径)
|
||||
# 路径配置
|
||||
# CHANGE 2026-02-19 | 统一迁移到 export/ETL-Connectors/feiqiu/ 下
|
||||
# ------------------------------------------------------------------------------
|
||||
EXPORT_ROOT=C:/NeoZQYY/export/ETL/JSON
|
||||
LOG_ROOT=C:/NeoZQYY/export/ETL/LOG
|
||||
FETCH_ROOT=C:/NeoZQYY/export/ETL/JSON
|
||||
EXPORT_ROOT=C:/NeoZQYY/export/ETL-Connectors/feiqiu/JSON
|
||||
LOG_ROOT=C:/NeoZQYY/export/ETL-Connectors/feiqiu/LOGS
|
||||
FETCH_ROOT=C:/NeoZQYY/export/ETL-Connectors/feiqiu/JSON
|
||||
WRITE_PRETTY_JSON=true
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
@@ -53,6 +53,7 @@ DEFAULT_LIST_KEYS: Tuple[str, ...] = (
|
||||
"goodsCategoryList",
|
||||
"orderGoodsList",
|
||||
"orderGoodsLedgers",
|
||||
"staffProfiles",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -174,7 +174,12 @@ def build_recording_client(
|
||||
tz_name = _cfg_get(cfg, "app.timezone", "Asia/Shanghai") or "Asia/Shanghai"
|
||||
tz = ZoneInfo(tz_name)
|
||||
ts = datetime.now(tz).strftime("%Y%m%d-%H%M%S")
|
||||
fetch_root = _cfg_get(cfg, "pipeline.fetch_root") or _cfg_get(cfg, "io.export_root") or "export/JSON"
|
||||
fetch_root = _cfg_get(cfg, "pipeline.fetch_root") or _cfg_get(cfg, "io.export_root")
|
||||
if not fetch_root:
|
||||
raise RuntimeError(
|
||||
"EXPORT_ROOT / FETCH_ROOT 未配置。"
|
||||
"请在根 .env 中设置,参考 .env.template"
|
||||
)
|
||||
task_upper = str(task_code).upper()
|
||||
output_dir = Path(fetch_root) / task_upper / f"{task_upper}-{run_id}-{ts}"
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
3. Layers 模式:通过 --layers 自由组合 ETL 层(ODS,DWD,DWS,INDEX)
|
||||
|
||||
注意:--flow 和 --layers 互斥,不能同时使用。
|
||||
--pipeline 是 --flow 的已弃用别名,使用时会输出 DeprecationWarning。
|
||||
|
||||
处理模式说明(--processing-mode):
|
||||
- increment_only:仅增量 - 只执行增量数据处理
|
||||
@@ -75,6 +74,7 @@ PROCESSING_MODE_CHOICES = [
|
||||
"increment_only", # 仅增量
|
||||
"verify_only", # 校验并修复(跳过增量)
|
||||
"increment_verify", # 增量 + 校验并修复
|
||||
"full_window", # 全窗口处理(用 API 返回数据的时间范围,处理所有层,无需校验)
|
||||
]
|
||||
|
||||
# 时间窗口切分选项
|
||||
@@ -150,9 +150,6 @@ def parse_args():
|
||||
|
||||
# 仅执行 DWS + INDEX 层
|
||||
python -m cli.main --layers DWS,INDEX
|
||||
|
||||
# --pipeline 仍可用(已弃用,建议迁移到 --flow)
|
||||
python -m cli.main --pipeline api_full
|
||||
""",
|
||||
)
|
||||
|
||||
@@ -167,16 +164,9 @@ def parse_args():
|
||||
choices=FLOW_CHOICES,
|
||||
help="Flow 类型(与 --layers 互斥):api_ods, api_ods_dwd, api_full, ods_dwd, dwd_dws, dwd_dws_index, dwd_index",
|
||||
)
|
||||
# --pipeline 作为已弃用别名,映射到独立 dest 以便检测使用
|
||||
parser.add_argument(
|
||||
"--pipeline",
|
||||
choices=FLOW_CHOICES,
|
||||
dest="pipeline_deprecated",
|
||||
help="[已弃用] 请使用 --flow。功能与 --flow 相同,使用时输出 DeprecationWarning",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--layers",
|
||||
help="ETL 层自由组合,逗号分隔(ODS,DWD,DWS,INDEX),与 --flow/--pipeline 互斥",
|
||||
help="ETL 层自由组合,逗号分隔(ODS,DWD,DWS,INDEX),与 --flow 互斥",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--processing-mode",
|
||||
@@ -409,6 +399,10 @@ def build_cli_overrides(args) -> dict:
|
||||
if args.allow_empty_advance:
|
||||
overrides.setdefault("run", {})["allow_empty_result_advance"] = True
|
||||
|
||||
# 处理模式(写入 config 供 ODS 任务层读取)
|
||||
if hasattr(args, "processing_mode") and args.processing_mode:
|
||||
overrides.setdefault("run", {})["processing_mode"] = args.processing_mode
|
||||
|
||||
# 强制全量更新
|
||||
if args.force_full:
|
||||
overrides.setdefault("run", {})["force_full_update"] = True
|
||||
@@ -451,19 +445,7 @@ def main():
|
||||
logger = setup_logging()
|
||||
args = parse_args()
|
||||
|
||||
# --pipeline 已弃用别名处理:合并到 args.flow(参数名保留以兼容旧调用)
|
||||
if args.pipeline_deprecated:
|
||||
import warnings
|
||||
warnings.warn(
|
||||
"--pipeline 参数已弃用,请使用 --flow",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if args.flow:
|
||||
print("错误: --pipeline 和 --flow 不能同时指定", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
args.flow = args.pipeline_deprecated
|
||||
|
||||
# CHANGE [2026-02-20] intent: 移除 --pipeline 弃用别名,统一使用 --flow
|
||||
# --layers 和 --flow 互斥校验
|
||||
if getattr(args, "layers", None) and args.flow:
|
||||
print("错误: --layers 和 --flow 互斥,请只指定其中一个", file=sys.stderr)
|
||||
@@ -552,7 +534,7 @@ def main():
|
||||
db_conn, api_client, logger,
|
||||
)
|
||||
result = runner.run(
|
||||
pipeline=args.flow,
|
||||
flow=args.flow,
|
||||
processing_mode=args.processing_mode,
|
||||
data_source=data_source,
|
||||
window_start=window_start,
|
||||
@@ -610,7 +592,7 @@ def main():
|
||||
db_conn, api_client, logger,
|
||||
)
|
||||
result = runner.run(
|
||||
pipeline=None,
|
||||
flow=None,
|
||||
layers=layers,
|
||||
processing_mode=args.processing_mode,
|
||||
data_source=data_source,
|
||||
|
||||
@@ -80,9 +80,9 @@ DEFAULTS = {
|
||||
"allow_empty_result_advance": True,
|
||||
},
|
||||
"io": {
|
||||
"export_root": "export/JSON",
|
||||
"log_root": "export/LOG",
|
||||
"fetch_root": "export/JSON",
|
||||
"export_root": "",
|
||||
"log_root": "",
|
||||
"fetch_root": "",
|
||||
"ingest_source_dir": "",
|
||||
"manifest_name": "manifest.json",
|
||||
"ingest_report_name": "ingest_report.json",
|
||||
@@ -94,7 +94,7 @@ DEFAULTS = {
|
||||
# 运行流程:FETCH_ONLY(仅在线抓取落盘)、INGEST_ONLY(本地清洗入库)、FULL(抓取 + 清洗入库)
|
||||
"flow": "FULL",
|
||||
# 在线抓取 JSON 输出根目录(按任务、run_id 与时间自动创建子目录)
|
||||
"fetch_root": "export/JSON",
|
||||
"fetch_root": "",
|
||||
# 本地清洗入库时的 JSON 输入目录(为空则默认使用本次抓取目录)
|
||||
"ingest_source_dir": "",
|
||||
},
|
||||
@@ -115,7 +115,7 @@ DEFAULTS = {
|
||||
},
|
||||
"ods": {
|
||||
# ODS 离线重建/回放相关(仅开发/运维使用)
|
||||
"json_doc_dir": "export/test-json-doc",
|
||||
"json_doc_dir": "",
|
||||
"include_files": "",
|
||||
"drop_schema_first": True,
|
||||
},
|
||||
|
||||
@@ -64,7 +64,7 @@
|
||||
- 为全部 23 张 ODS 表创建 `(业务主键, fetched_at DESC)` 复合索引
|
||||
- 使用 `CREATE INDEX CONCURRENTLY IF NOT EXISTS`,保证幂等且不锁表
|
||||
- 索引命名规范:`idx_ods_{table_name}_latest`
|
||||
- 同步更新 `db/etl_feiqiu/schemas/ods.sql` 中的索引定义
|
||||
- 同步更新 `docs/database/ddl/etl_feiqiu__ods.sql` 中的索引定义
|
||||
|
||||
### 用途
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
# dim_staff 员工档案主表
|
||||
|
||||
> 生成时间:2026-02-23
|
||||
|
||||
## 表信息
|
||||
|
||||
| 属性 | 值 |
|
||||
|------|-----|
|
||||
| Schema | dwd |
|
||||
| 表名 | dim_staff |
|
||||
| 主键 | staff_id, scd2_start_time |
|
||||
| 扩展表 | dim_staff_ex |
|
||||
| ODS 来源 | ods.staff_info_master |
|
||||
| 说明 | 员工档案维度主表(SCD2),包含核心业务字段 |
|
||||
|
||||
## 字段说明
|
||||
|
||||
| 序号 | 字段名 | 类型 | 可空 | 主键 | 说明 |
|
||||
|------|--------|------|------|------|------|
|
||||
| 1 | staff_id | BIGINT | NO | PK | 员工唯一标识(映射自 ODS id) |
|
||||
| 2 | staff_name | TEXT | YES | | 员工姓名 |
|
||||
| 3 | alias_name | TEXT | YES | | 别名 |
|
||||
| 4 | mobile | TEXT | YES | | 手机号 |
|
||||
| 5 | gender | INTEGER | YES | | 性别 |
|
||||
| 6 | job | TEXT | YES | | 职位(店长/主管/教练/收银员等) |
|
||||
| 7 | tenant_id | BIGINT | YES | | 租户 ID |
|
||||
| 8 | site_id | BIGINT | YES | | 门店 ID |
|
||||
| 9 | system_role_id | INTEGER | YES | | 系统角色 ID |
|
||||
| 10 | staff_identity | INTEGER | YES | | 员工身份类型 |
|
||||
| 11 | status | INTEGER | YES | | 账号状态 |
|
||||
| 12 | leave_status | INTEGER | YES | | 在职状态(0=在职,1=离职) |
|
||||
| 13 | entry_time | TIMESTAMPTZ | YES | | 入职时间 |
|
||||
| 14 | resign_time | TIMESTAMPTZ | YES | | 离职时间 |
|
||||
| 15 | is_delete | INTEGER | YES | | 删除标记 |
|
||||
| 16 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 |
|
||||
| 17 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 |
|
||||
| 18 | scd2_is_current | INTEGER | YES | | 当前版本标记 |
|
||||
| 19 | scd2_version | INTEGER | YES | | 版本号 |
|
||||
|
||||
## 与其他表的关系
|
||||
|
||||
- 扩展表:`dwd.dim_staff_ex`(次要/低频变更字段)
|
||||
- ODS 来源:`ods.staff_info_master`
|
||||
- 与助教维度表(`dim_assistant`)是完全独立的实体
|
||||
@@ -0,0 +1,51 @@
|
||||
# dim_staff_ex 员工档案扩展表
|
||||
|
||||
> 生成时间:2026-02-23
|
||||
|
||||
## 表信息
|
||||
|
||||
| 属性 | 值 |
|
||||
|------|-----|
|
||||
| Schema | dwd |
|
||||
| 表名 | dim_staff_ex |
|
||||
| 主键 | staff_id, scd2_start_time |
|
||||
| 主表 | dim_staff |
|
||||
| ODS 来源 | ods.staff_info_master |
|
||||
| 说明 | 员工档案维度扩展表(SCD2),包含次要/低频变更字段 |
|
||||
|
||||
## 字段说明
|
||||
|
||||
| 序号 | 字段名 | 类型 | 可空 | 主键 | 说明 |
|
||||
|------|--------|------|------|------|------|
|
||||
| 1 | staff_id | BIGINT | NO | PK | 员工唯一标识(映射自 ODS id) |
|
||||
| 2 | avatar | TEXT | YES | | 头像 URL |
|
||||
| 3 | job_num | TEXT | YES | | 工号 |
|
||||
| 4 | account_status | INTEGER | YES | | 账号启用状态 |
|
||||
| 5 | rank_id | INTEGER | YES | | 职级 ID |
|
||||
| 6 | rank_name | TEXT | YES | | 职级名称 |
|
||||
| 7 | new_rank_id | INTEGER | YES | | 新职级 ID |
|
||||
| 8 | new_staff_identity | INTEGER | YES | | 新员工身份 |
|
||||
| 9 | is_reserve | INTEGER | YES | | 预约标记 |
|
||||
| 10 | shop_name | TEXT | YES | | 门店名称 |
|
||||
| 11 | site_label | TEXT | YES | | 门店标签 |
|
||||
| 12 | tenant_org_id | BIGINT | YES | | 租户组织 ID |
|
||||
| 13 | system_user_id | BIGINT | YES | | 系统用户 ID |
|
||||
| 14 | cashier_point_id | BIGINT | YES | | 收银点 ID |
|
||||
| 15 | cashier_point_name | TEXT | YES | | 收银点名称 |
|
||||
| 16 | group_id | BIGINT | YES | | 分组 ID |
|
||||
| 17 | group_name | TEXT | YES | | 分组名称 |
|
||||
| 18 | staff_profile_id | BIGINT | YES | | 员工档案 ID |
|
||||
| 19 | auth_code | TEXT | YES | | 授权码 |
|
||||
| 20 | auth_code_create | TIMESTAMPTZ | YES | | 授权码创建时间 |
|
||||
| 21 | ding_talk_synced | INTEGER | YES | | 钉钉同步状态 |
|
||||
| 22 | salary_grant_enabled | INTEGER | YES | | 工资发放启用 |
|
||||
| 23 | entry_type | INTEGER | YES | | 入职类型 |
|
||||
| 24 | entry_sign_status | INTEGER | YES | | 入职签约状态 |
|
||||
| 25 | resign_sign_status | INTEGER | YES | | 离职签约状态 |
|
||||
| 26 | criticism_status | INTEGER | YES | | 批评状态 |
|
||||
| 27 | create_time | TIMESTAMPTZ | YES | | 创建时间 |
|
||||
| 28 | user_roles | JSONB | YES | | 用户角色列表 |
|
||||
| 29 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 |
|
||||
| 30 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 |
|
||||
| 31 | scd2_is_current | INTEGER | YES | | 当前版本标记 |
|
||||
| 32 | scd2_version | INTEGER | YES | | 版本号 |
|
||||
@@ -0,0 +1,70 @@
|
||||
# staff_info_master 员工档案主表
|
||||
|
||||
> 生成时间:2026-02-23
|
||||
|
||||
## 表信息
|
||||
|
||||
| 属性 | 值 |
|
||||
|------|-----|
|
||||
| Schema | ods |
|
||||
| 表名 | staff_info_master |
|
||||
| 主键 | id |
|
||||
| 数据来源 | SearchSystemStaffInfo API |
|
||||
| 快照模式 | FULL_TABLE |
|
||||
| 说明 | 员工档案主数据(店长、主管、教练、收银员等) |
|
||||
|
||||
## 字段说明
|
||||
|
||||
| 序号 | 字段名 | 类型 | 可空 | 说明 |
|
||||
|------|--------|------|------|------|
|
||||
| 1 | id | BIGINT | NO | 员工主键 ID |
|
||||
| 2 | tenant_id | BIGINT | YES | 租户 ID |
|
||||
| 3 | site_id | BIGINT | YES | 门店 ID |
|
||||
| 4 | tenant_org_id | BIGINT | YES | 租户组织 ID |
|
||||
| 5 | system_user_id | BIGINT | YES | 系统用户 ID |
|
||||
| 6 | staff_name | TEXT | YES | 员工姓名 |
|
||||
| 7 | alias_name | TEXT | YES | 别名 |
|
||||
| 8 | mobile | TEXT | YES | 手机号 |
|
||||
| 9 | avatar | TEXT | YES | 头像 URL |
|
||||
| 10 | gender | INTEGER | YES | 性别(3=未知) |
|
||||
| 11 | job | TEXT | YES | 职位(店长/主管/教练/收银员等) |
|
||||
| 12 | job_num | TEXT | YES | 工号 |
|
||||
| 13 | staff_identity | INTEGER | YES | 员工身份类型 |
|
||||
| 14 | status | INTEGER | YES | 账号状态 |
|
||||
| 15 | account_status | INTEGER | YES | 账号启用状态 |
|
||||
| 16 | system_role_id | INTEGER | YES | 系统角色 ID |
|
||||
| 17 | rank_id | INTEGER | YES | 职级 ID |
|
||||
| 18 | rank_name | TEXT | YES | 职级名称 |
|
||||
| 19 | new_rank_id | INTEGER | YES | 新职级 ID |
|
||||
| 20 | new_staff_identity | INTEGER | YES | 新员工身份 |
|
||||
| 21 | leave_status | INTEGER | YES | 在职状态(0=在职,1=离职) |
|
||||
| 22 | entry_time | TIMESTAMP | YES | 入职时间 |
|
||||
| 23 | resign_time | TIMESTAMP | YES | 离职时间 |
|
||||
| 24 | create_time | TIMESTAMP | YES | 创建时间 |
|
||||
| 25 | is_delete | INTEGER | YES | 删除标记 |
|
||||
| 26 | is_reserve | INTEGER | YES | 预约标记 |
|
||||
| 27 | shop_name | TEXT | YES | 门店名称 |
|
||||
| 28 | site_label | TEXT | YES | 门店标签 |
|
||||
| 29 | cashier_point_id | BIGINT | YES | 收银点 ID |
|
||||
| 30 | cashier_point_name | TEXT | YES | 收银点名称 |
|
||||
| 31 | group_id | BIGINT | YES | 分组 ID |
|
||||
| 32 | group_name | TEXT | YES | 分组名称 |
|
||||
| 33 | staff_profile_id | BIGINT | YES | 员工档案 ID |
|
||||
| 34 | auth_code | TEXT | YES | 授权码 |
|
||||
| 35 | auth_code_create | TIMESTAMP | YES | 授权码创建时间 |
|
||||
| 36 | ding_talk_synced | INTEGER | YES | 钉钉同步状态 |
|
||||
| 37 | salary_grant_enabled | INTEGER | YES | 工资发放启用 |
|
||||
| 38 | entry_type | INTEGER | YES | 入职类型 |
|
||||
| 39 | entry_sign_status | INTEGER | YES | 入职签约状态 |
|
||||
| 40 | resign_sign_status | INTEGER | YES | 离职签约状态 |
|
||||
| 41 | criticism_status | INTEGER | YES | 批评状态 |
|
||||
| 42 | user_roles | JSONB | YES | 用户角色列表 |
|
||||
| 43 | content_hash | TEXT | NO | 记录内容哈希 |
|
||||
| 44 | source_file | TEXT | YES | 来源文件路径 |
|
||||
| 45 | fetched_at | TIMESTAMPTZ | YES | 抓取时间 |
|
||||
| 46 | payload | JSONB | NO | 原始 JSON |
|
||||
|
||||
## 与其他表的关系
|
||||
|
||||
- 员工表与助教表(`assistant_accounts_master`)是完全独立的实体
|
||||
- 下游:`dwd.dim_staff`(主表)、`dwd.dim_staff_ex`(扩展表)
|
||||
@@ -0,0 +1,80 @@
|
||||
# 员工档案(SearchSystemStaffInfo) → staff_info_master 字段映射
|
||||
|
||||
> 生成时间:2026-02-23
|
||||
|
||||
## 端点信息
|
||||
|
||||
| 属性 | 值 |
|
||||
|------|-----|
|
||||
| 接口路径 | `PersonnelManagement/SearchSystemStaffInfo` |
|
||||
| 请求方法 | POST |
|
||||
| ODS 对应表 | `ods.staff_info_master` |
|
||||
| JSON 数据路径 | `data.staffProfiles` |
|
||||
| 快照模式 | FULL_TABLE(全量快照) |
|
||||
|
||||
## 请求参数
|
||||
|
||||
| 参数 | 类型 | 默认值 | 说明 |
|
||||
|------|------|--------|------|
|
||||
| workStatusEnum | int | 0 | 在职状态筛选(0=全部) |
|
||||
| dingTalkSynced | int | 0 | 钉钉同步状态(0=全部) |
|
||||
| staffIdentity | int | 0 | 员工身份筛选(0=全部) |
|
||||
| rankId | int | 0 | 职级筛选(0=全部) |
|
||||
| criticismStatus | int | 0 | 批评状态(0=全部) |
|
||||
| signStatus | int | -1 | 签约状态(-1=全部) |
|
||||
|
||||
## 字段映射
|
||||
|
||||
| JSON 字段 | ODS 列名 | 类型转换 | 说明 |
|
||||
|-----------|----------|----------|------|
|
||||
| id | id | int→BIGINT | 员工主键 ID |
|
||||
| tenant_id | tenant_id | int→BIGINT | 租户 ID |
|
||||
| site_id | site_id | int→BIGINT | 门店 ID |
|
||||
| tenant_org_id | tenant_org_id | int→BIGINT | 租户组织 ID |
|
||||
| system_user_id | system_user_id | int→BIGINT | 系统用户 ID |
|
||||
| staff_name | staff_name | string→TEXT | 员工姓名 |
|
||||
| alias_name | alias_name | string→TEXT | 别名 |
|
||||
| mobile | mobile | string→TEXT | 手机号 |
|
||||
| avatar | avatar | string→TEXT | 头像 URL |
|
||||
| gender | gender | int→INTEGER | 性别(3=未知) |
|
||||
| job | job | string→TEXT | 职位名称(店长/主管/教练等) |
|
||||
| job_num | job_num | string→TEXT | 工号 |
|
||||
| staff_identity | staff_identity | int→INTEGER | 员工身份类型 |
|
||||
| status | status | int→INTEGER | 账号状态 |
|
||||
| account_status | account_status | int→INTEGER | 账号启用状态 |
|
||||
| system_role_id | system_role_id | int→INTEGER | 系统角色 ID |
|
||||
| rank_id | rank_id | int→INTEGER | 职级 ID |
|
||||
| rankName | rank_name | string→TEXT | 职级名称(驼峰→蛇形) |
|
||||
| new_rank_id | new_rank_id | int→INTEGER | 新职级 ID |
|
||||
| new_staff_identity | new_staff_identity | int→INTEGER | 新员工身份 |
|
||||
| leave_status | leave_status | int→INTEGER | 在职状态(0=在职) |
|
||||
| entry_time | entry_time | string→TIMESTAMP | 入职时间 |
|
||||
| resign_time | resign_time | string→TIMESTAMP | 离职时间 |
|
||||
| create_time | create_time | string→TIMESTAMP | 创建时间 |
|
||||
| is_delete | is_delete | int→INTEGER | 删除标记 |
|
||||
| is_reserve | is_reserve | int→INTEGER | 预约标记 |
|
||||
| shop_name | shop_name | string→TEXT | 门店名称 |
|
||||
| site_label | site_label | string→TEXT | 门店标签 |
|
||||
| cashierPointId | cashier_point_id | int→BIGINT | 收银点 ID(驼峰→蛇形) |
|
||||
| cashierPointName | cashier_point_name | string→TEXT | 收银点名称(驼峰→蛇形) |
|
||||
| groupId | group_id | int→BIGINT | 分组 ID(驼峰→蛇形) |
|
||||
| groupName | group_name | string→TEXT | 分组名称(驼峰→蛇形) |
|
||||
| staff_profile_id | staff_profile_id | int→BIGINT | 员工档案 ID |
|
||||
| auth_code | auth_code | string→TEXT | 授权码 |
|
||||
| auth_code_create | auth_code_create | string→TIMESTAMP | 授权码创建时间 |
|
||||
| ding_talk_synced | ding_talk_synced | int→INTEGER | 钉钉同步状态 |
|
||||
| salary_grant_enabled | salary_grant_enabled | int→INTEGER | 工资发放启用 |
|
||||
| entry_type | entry_type | int→INTEGER | 入职类型 |
|
||||
| entry_sign_status | entry_sign_status | int→INTEGER | 入职签约状态 |
|
||||
| resign_sign_status | resign_sign_status | int→INTEGER | 离职签约状态 |
|
||||
| criticism_status | criticism_status | int→INTEGER | 批评状态 |
|
||||
| userRoles | user_roles | array→JSONB | 用户角色列表(驼峰→蛇形) |
|
||||
|
||||
## ETL 元数据列
|
||||
|
||||
| 列名 | 类型 | 说明 |
|
||||
|------|------|------|
|
||||
| content_hash | TEXT | 记录内容哈希(去重用) |
|
||||
| source_file | TEXT | 来源文件路径 |
|
||||
| fetched_at | TIMESTAMPTZ | 抓取时间 |
|
||||
| payload | JSONB | 原始 JSON 完整保留 |
|
||||
@@ -1,230 +1,38 @@
|
||||
# BD_Manual — 飞球 ETL 数据库手册
|
||||
# 飞球 ETL 数据库手册
|
||||
|
||||
> 本文档是 `docs/bd_manual/` 目录的导航索引,涵盖 ODS、DWD、DWS、ETL_Admin 四个数据层的表级文档、字段映射文档和变更记录。
|
||||
> 模块专属的表级文档、字段映射、扩展表说明。
|
||||
> DDL 基线见项目级 `docs/database/ddl/`,变更记录已归档至 `_archived/`。
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
docs/bd_manual/
|
||||
├── README.md ← 本文件(根索引)
|
||||
├── ddl_compare_results.md ← DDL 对比结果汇总
|
||||
├── ODS/ ← 操作数据存储层(ods schema)
|
||||
│ ├── main/ ← 表级文档
|
||||
│ ├── mappings/ ← API JSON → ODS 字段映射文档
|
||||
│ └── changes/ ← 变更记录
|
||||
├── DWD/ ← 明细数据层(dwd schema)
|
||||
│ ├── main/ ← 表级文档
|
||||
│ ├── Ex/ ← 扩展表文档(SCD2 维度扩展等)
|
||||
│ └── changes/ ← 变更记录
|
||||
├── DWS/ ← 数据服务层(dws schema)
|
||||
│ ├── main/ ← 表级文档
|
||||
│ └── changes/ ← 变更记录
|
||||
└── ETL_Admin/ ← ETL 管理层(meta schema)
|
||||
├── main/ ← 表级文档
|
||||
└── changes/ ← 变更记录
|
||||
database/
|
||||
├── ODS/
|
||||
│ ├── main/ — ODS 表级文档(BD_manual_*.md)
|
||||
│ └── mappings/ — API JSON → ODS 字段映射(mapping_*.md)
|
||||
├── DWD/
|
||||
│ ├── main/ — DWD 主表文档
|
||||
│ └── Ex/ — DWD 扩展表文档
|
||||
├── DWS/
|
||||
│ └── main/ — DWS 汇总表文档
|
||||
├── ETL_Admin/
|
||||
│ └── main/ — meta schema 表文档
|
||||
└── _archived/ — 过时的变更记录、DDL 对比报告、已删除表文档
|
||||
```
|
||||
|
||||
## 文档命名规范
|
||||
## 文档类型
|
||||
|
||||
| 文档类型 | 命名格式 | 示例 |
|
||||
|----------|----------|------|
|
||||
| 表级文档 | `BD_manual_{表名}.md` | `BD_manual_member_profiles.md` |
|
||||
| 映射文档 | `mapping_{API端点名}_{ODS表名}.md` | `mapping_GetTenantMemberList_member_profiles.md` |
|
||||
| 变更记录 | `{YYYYMMDD}_{变更简述}.md` 或 `{YYYY-MM-DD}_{变更简述}.md` | `2026-02-13_ddl_sync_ods.md` |
|
||||
| 类型 | 命名规则 | 说明 |
|
||||
|------|---------|------|
|
||||
| 表级文档 | `BD_manual_{表名}.md` | 字段说明、主键、业务含义 |
|
||||
| 扩展表文档 | `BD_manual_{表名}_ex.md` | SCD2 扩展字段、溢出字段 |
|
||||
| 字段映射 | `mapping_{API端点}_{ODS表名}.md` | API JSON 字段 → ODS 列的映射关系 |
|
||||
|
||||
## ODS 层文档清单(ods)
|
||||
## 与项目级文档的关系
|
||||
|
||||
### 表级文档(`ODS/main/`)— 共 23 份
|
||||
|
||||
| 序号 | 文件名 | 对应表 |
|
||||
|------|--------|--------|
|
||||
| 1 | `BD_manual_assistant_accounts_master.md` | assistant_accounts_master |
|
||||
| 2 | `BD_manual_assistant_cancellation_records.md` | assistant_cancellation_records |
|
||||
| 3 | `BD_manual_assistant_service_records.md` | assistant_service_records |
|
||||
| 4 | `BD_manual_goods_stock_movements.md` | goods_stock_movements |
|
||||
| 5 | `BD_manual_goods_stock_summary.md` | goods_stock_summary |
|
||||
| 6 | `BD_manual_group_buy_packages.md` | group_buy_packages |
|
||||
| 7 | `BD_manual_group_buy_redemption_records.md` | group_buy_redemption_records |
|
||||
| 8 | `BD_manual_member_balance_changes.md` | member_balance_changes |
|
||||
| 9 | `BD_manual_member_profiles.md` | member_profiles |
|
||||
| 10 | `BD_manual_member_stored_value_cards.md` | member_stored_value_cards |
|
||||
| 11 | `BD_manual_payment_transactions.md` | payment_transactions |
|
||||
| 12 | `BD_manual_platform_coupon_redemption_records.md` | platform_coupon_redemption_records |
|
||||
| 13 | `BD_manual_recharge_settlements.md` | recharge_settlements |
|
||||
| 14 | `BD_manual_refund_transactions.md` | refund_transactions |
|
||||
| 15 | `BD_manual_settlement_records.md` | settlement_records |
|
||||
| 16 | `BD_manual_settlement_ticket_details.md` | settlement_ticket_details |
|
||||
| 17 | `BD_manual_site_tables_master.md` | site_tables_master |
|
||||
| 18 | `BD_manual_stock_goods_category_tree.md` | stock_goods_category_tree |
|
||||
| 19 | `BD_manual_store_goods_master.md` | store_goods_master |
|
||||
| 20 | `BD_manual_store_goods_sales_records.md` | store_goods_sales_records |
|
||||
| 21 | `BD_manual_table_fee_discount_records.md` | table_fee_discount_records |
|
||||
| 22 | `BD_manual_table_fee_transactions.md` | table_fee_transactions |
|
||||
| 23 | `BD_manual_tenant_goods_master.md` | tenant_goods_master |
|
||||
|
||||
### API→ODS 字段映射文档(`ODS/mappings/`)— 共 23 份
|
||||
|
||||
| 序号 | 文件名 | API 端点 → ODS 表 |
|
||||
|------|--------|-------------------|
|
||||
| 1 | `mapping_GetAbolitionAssistant_assistant_cancellation_records.md` | GetAbolitionAssistant → assistant_cancellation_records |
|
||||
| 2 | `mapping_GetAllOrderSettleList_settlement_records.md` | GetAllOrderSettleList → settlement_records |
|
||||
| 3 | `mapping_GetGoodsInventoryList_store_goods_master.md` | GetGoodsInventoryList → store_goods_master |
|
||||
| 4 | `mapping_GetGoodsSalesList_store_goods_sales_records.md` | GetGoodsSalesList → store_goods_sales_records |
|
||||
| 5 | `mapping_GetGoodsStockReport_goods_stock_summary.md` | GetGoodsStockReport → goods_stock_summary |
|
||||
| 6 | `mapping_GetMemberCardBalanceChange_member_balance_changes.md` | GetMemberCardBalanceChange → member_balance_changes |
|
||||
| 7 | `mapping_GetOfflineCouponConsumePageList_platform_coupon_redemption_records.md` | GetOfflineCouponConsumePageList → platform_coupon_redemption_records |
|
||||
| 8 | `mapping_GetOrderAssistantDetails_assistant_service_records.md` | GetOrderAssistantDetails → assistant_service_records |
|
||||
| 9 | `mapping_GetOrderSettleTicketNew_settlement_ticket_details.md` | GetOrderSettleTicketNew → settlement_ticket_details |
|
||||
| 10 | `mapping_GetPayLogListPage_payment_transactions.md` | GetPayLogListPage → payment_transactions |
|
||||
| 11 | `mapping_GetRechargeSettleList_recharge_settlements.md` | GetRechargeSettleList → recharge_settlements |
|
||||
| 12 | `mapping_GetRefundPayLogList_refund_transactions.md` | GetRefundPayLogList → refund_transactions |
|
||||
| 13 | `mapping_GetSiteTableOrderDetails_table_fee_transactions.md` | GetSiteTableOrderDetails → table_fee_transactions |
|
||||
| 14 | `mapping_GetSiteTables_site_tables_master.md` | GetSiteTables → site_tables_master |
|
||||
| 15 | `mapping_GetSiteTableUseDetails_group_buy_redemption_records.md` | GetSiteTableUseDetails → group_buy_redemption_records |
|
||||
| 16 | `mapping_GetTaiFeeAdjustList_table_fee_discount_records.md` | GetTaiFeeAdjustList → table_fee_discount_records |
|
||||
| 17 | `mapping_GetTenantMemberCardList_member_stored_value_cards.md` | GetTenantMemberCardList → member_stored_value_cards |
|
||||
| 18 | `mapping_GetTenantMemberList_member_profiles.md` | GetTenantMemberList → member_profiles |
|
||||
| 19 | `mapping_QueryGoodsOutboundReceipt_goods_stock_movements.md` | QueryGoodsOutboundReceipt → goods_stock_movements |
|
||||
| 20 | `mapping_QueryPackageCouponList_group_buy_packages.md` | QueryPackageCouponList → group_buy_packages |
|
||||
| 21 | `mapping_QueryPrimarySecondaryCategory_stock_goods_category_tree.md` | QueryPrimarySecondaryCategory → stock_goods_category_tree |
|
||||
| 22 | `mapping_QueryTenantGoods_tenant_goods_master.md` | QueryTenantGoods → tenant_goods_master |
|
||||
| 23 | `mapping_SearchAssistantInfo_assistant_accounts_master.md` | SearchAssistantInfo → assistant_accounts_master |
|
||||
|
||||
### 变更记录(`ODS/changes/`)
|
||||
|
||||
| 文件名 | 说明 |
|
||||
|--------|------|
|
||||
| `2026-02-13_ddl_sync_ods.md` | DDL 对比同步 — ODS 层 |
|
||||
| `20260213_align_ods_with_api.md` | ODS 表结构与 API 对齐 |
|
||||
| `20260214_drop_ods_option_name_able_site_transfer.md` | 移除 ODS 冗余字段/表 |
|
||||
| `20260214_drop_ods_settlelist.md` | 移除 ODS settle_list 表 |
|
||||
|
||||
## DWD 层文档清单(dwd)
|
||||
|
||||
### 表级文档(`DWD/main/`)— 共 22 份
|
||||
|
||||
| 序号 | 文件名 | 对应表 |
|
||||
|------|--------|--------|
|
||||
| 1 | `BD_manual_dwd.md` | dwd(层级概览) |
|
||||
| 2 | `BD_manual_dim_assistant.md` | dim_assistant |
|
||||
| 3 | `BD_manual_dim_goods_category.md` | dim_goods_category |
|
||||
| 4 | `BD_manual_dim_groupbuy_package.md` | dim_groupbuy_package |
|
||||
| 5 | `BD_manual_dim_member.md` | dim_member |
|
||||
| 6 | `BD_manual_dim_member_card_account.md` | dim_member_card_account |
|
||||
| 7 | `BD_manual_dim_site.md` | dim_site |
|
||||
| 8 | `BD_manual_dim_store_goods.md` | dim_store_goods |
|
||||
| 9 | `BD_manual_dim_table.md` | dim_table |
|
||||
| 10 | `BD_manual_dim_tenant_goods.md` | dim_tenant_goods |
|
||||
| 11 | `BD_manual_dwd_assistant_service_log.md` | dwd_assistant_service_log |
|
||||
| 12 | `BD_manual_dwd_assistant_trash_event.md` | dwd_assistant_trash_event |
|
||||
| 13 | `BD_manual_dwd_groupbuy_redemption.md` | dwd_groupbuy_redemption |
|
||||
| 14 | `BD_manual_dwd_member_balance_change.md` | dwd_member_balance_change |
|
||||
| 15 | `BD_manual_dwd_payment.md` | dwd_payment |
|
||||
| 16 | `BD_manual_dwd_platform_coupon_redemption.md` | dwd_platform_coupon_redemption |
|
||||
| 17 | `BD_manual_dwd_recharge_order.md` | dwd_recharge_order |
|
||||
| 18 | `BD_manual_dwd_refund.md` | dwd_refund |
|
||||
| 19 | `BD_manual_dwd_settlement_head.md` | dwd_settlement_head |
|
||||
| 20 | `BD_manual_dwd_store_goods_sale.md` | dwd_store_goods_sale |
|
||||
| 21 | `BD_manual_dwd_table_fee_adjust.md` | dwd_table_fee_adjust |
|
||||
| 22 | `BD_manual_dwd_table_fee_log.md` | dwd_table_fee_log |
|
||||
|
||||
### 扩展表文档(`DWD/Ex/`)— 共 19 份
|
||||
|
||||
| 序号 | 文件名 | 对应扩展表 |
|
||||
|------|--------|------------|
|
||||
| 1 | `BD_manual_dim_assistant_ex.md` | dim_assistant_ex |
|
||||
| 2 | `BD_manual_dim_groupbuy_package_ex.md` | dim_groupbuy_package_ex |
|
||||
| 3 | `BD_manual_dim_member_card_account_ex.md` | dim_member_card_account_ex |
|
||||
| 4 | `BD_manual_dim_member_ex.md` | dim_member_ex |
|
||||
| 5 | `BD_manual_dim_site_ex.md` | dim_site_ex |
|
||||
| 6 | `BD_manual_dim_store_goods_ex.md` | dim_store_goods_ex |
|
||||
| 7 | `BD_manual_dim_table_ex.md` | dim_table_ex |
|
||||
| 8 | `BD_manual_dim_tenant_goods_ex.md` | dim_tenant_goods_ex |
|
||||
| 9 | `BD_manual_dwd_assistant_service_log_ex.md` | dwd_assistant_service_log_ex |
|
||||
| 10 | `BD_manual_dwd_assistant_trash_event_ex.md` | dwd_assistant_trash_event_ex |
|
||||
| 11 | `BD_manual_dwd_groupbuy_redemption_ex.md` | dwd_groupbuy_redemption_ex |
|
||||
| 12 | `BD_manual_dwd_member_balance_change_ex.md` | dwd_member_balance_change_ex |
|
||||
| 13 | `BD_manual_dwd_platform_coupon_redemption_ex.md` | dwd_platform_coupon_redemption_ex |
|
||||
| 14 | `BD_manual_dwd_recharge_order_ex.md` | dwd_recharge_order_ex |
|
||||
| 15 | `BD_manual_dwd_refund_ex.md` | dwd_refund_ex |
|
||||
| 16 | `BD_manual_dwd_settlement_head_ex.md` | dwd_settlement_head_ex |
|
||||
| 17 | `BD_manual_dwd_store_goods_sale_ex.md` | dwd_store_goods_sale_ex |
|
||||
| 18 | `BD_manual_dwd_table_fee_adjust_ex.md` | dwd_table_fee_adjust_ex |
|
||||
| 19 | `BD_manual_dwd_table_fee_log_ex.md` | dwd_table_fee_log_ex |
|
||||
|
||||
### 变更记录(`DWD/changes/`)
|
||||
|
||||
| 文件名 | 说明 |
|
||||
|--------|------|
|
||||
| `2026-02-13_ddl_sync_dwd.md` | DDL 对比同步 — DWD 层 |
|
||||
| `20260214_drop_dwd_settle_list.md` | 移除 DWD settle_list 表 |
|
||||
|
||||
## DWS 层文档清单(dws)
|
||||
|
||||
### 表级文档(`DWS/main/`)— 共 29 份
|
||||
|
||||
| 序号 | 文件名 | 对应表 |
|
||||
|------|--------|--------|
|
||||
| 1 | `BD_manual_cfg_area_category.md` | cfg_area_category |
|
||||
| 2 | `BD_manual_cfg_assistant_level_price.md` | cfg_assistant_level_price |
|
||||
| 3 | `BD_manual_cfg_bonus_rules.md` | cfg_bonus_rules |
|
||||
| 4 | `BD_manual_cfg_index_parameters.md` | cfg_index_parameters |
|
||||
| 5 | `BD_manual_cfg_performance_tier.md` | cfg_performance_tier |
|
||||
| 6 | `BD_manual_cfg_skill_type.md` | cfg_skill_type |
|
||||
| 7 | `BD_manual_dws_assistant_customer_stats.md` | dws_assistant_customer_stats |
|
||||
| 8 | `BD_manual_dws_assistant_daily_detail.md` | dws_assistant_daily_detail |
|
||||
| 9 | `BD_manual_dws_assistant_finance_analysis.md` | dws_assistant_finance_analysis |
|
||||
| 10 | `BD_manual_dws_assistant_monthly_summary.md` | dws_assistant_monthly_summary |
|
||||
| 11 | `BD_manual_dws_assistant_recharge_commission.md` | dws_assistant_recharge_commission |
|
||||
| 12 | `BD_manual_dws_assistant_salary_calc.md` | dws_assistant_salary_calc |
|
||||
| 13 | `BD_manual_dws_finance_daily_summary.md` | dws_finance_daily_summary |
|
||||
| 14 | `BD_manual_dws_finance_discount_detail.md` | dws_finance_discount_detail |
|
||||
| 15 | `BD_manual_dws_finance_expense_summary.md` | dws_finance_expense_summary |
|
||||
| 16 | `BD_manual_dws_finance_income_structure.md` | dws_finance_income_structure |
|
||||
| 17 | `BD_manual_dws_finance_recharge_summary.md` | dws_finance_recharge_summary |
|
||||
| 18 | `BD_manual_dws_index_percentile_history.md` | dws_index_percentile_history |
|
||||
| 19 | `BD_manual_dws_member_assistant_intimacy.md` | dws_member_assistant_intimacy |
|
||||
| 20 | `BD_manual_dws_member_assistant_relation_index.md` | dws_member_assistant_relation_index |
|
||||
| 21 | `BD_manual_dws_member_consumption_summary.md` | dws_member_consumption_summary |
|
||||
| 22 | `BD_manual_dws_member_newconv_index.md` | dws_member_newconv_index |
|
||||
| 23 | `BD_manual_dws_member_visit_detail.md` | dws_member_visit_detail |
|
||||
| 24 | `BD_manual_dws_member_winback_index.md` | dws_member_winback_index |
|
||||
| 25 | `BD_manual_dws_ml_manual_order_alloc.md` | dws_ml_manual_order_alloc |
|
||||
| 26 | `BD_manual_dws_ml_manual_order_source.md` | dws_ml_manual_order_source |
|
||||
| 27 | `BD_manual_dws_order_summary.md` | dws_order_summary |
|
||||
| 28 | `BD_manual_dws_platform_settlement.md` | dws_platform_settlement |
|
||||
| 29 | `BD_manual_v_member_recall_priority.md` | v_member_recall_priority |
|
||||
|
||||
### 变更记录(`DWS/changes/`)
|
||||
|
||||
| 文件名 | 说明 |
|
||||
|--------|------|
|
||||
| `2026-02-13_ddl_sync_dws.md` | DDL 对比同步 — DWS 层 |
|
||||
|
||||
## ETL_Admin 层文档清单(etl_admin)
|
||||
|
||||
### 表级文档(`ETL_Admin/main/`)— 共 3 份
|
||||
|
||||
| 序号 | 文件名 | 对应表 |
|
||||
|------|--------|--------|
|
||||
| 1 | `BD_manual_etl_cursor.md` | etl_cursor |
|
||||
| 2 | `BD_manual_etl_run.md` | etl_run |
|
||||
| 3 | `BD_manual_etl_task.md` | etl_task |
|
||||
|
||||
### 变更记录(`ETL_Admin/changes/`)
|
||||
|
||||
暂无变更记录。
|
||||
|
||||
## 相关资源
|
||||
|
||||
| 资源 | 路径 | 说明 |
|
||||
| 内容 | 位置 | 说明 |
|
||||
|------|------|------|
|
||||
| ODS 数据字典 | `docs/dictionary/ods_tables_dictionary.md` | ODS 层所有表的概览汇总 |
|
||||
| DDL 对比结果 | `docs/bd_manual/ddl_compare_results.md` | DDL 文件与数据库实际状态的对比报告 |
|
||||
| DDL 文件 — ODS | `database/schema_ODS_doc.sql` | ODS 层表结构定义 |
|
||||
| DDL 文件 — DWD | `database/schema_dwd_doc.sql` | DWD 层表结构定义 |
|
||||
| DDL 文件 — DWS | `database/schema_dws.sql` | DWS 层表结构定义 |
|
||||
| DDL 文件 — ETL_Admin | `database/schema_etl_admin.sql` | ETL_Admin 层表结构定义 |
|
||||
| API 端点文档 | `docs/api-reference/endpoints/` | 上游 SaaS API 端点说明 |
|
||||
| DDL 对比脚本 | `scripts/compare_ddl_db.py` | DDL 与数据库实际状态对比工具 |
|
||||
| 文档验证脚本 | `scripts/validate_bd_manual.py` | BD_Manual 文档覆盖率和格式验证 |
|
||||
| DDL 基线 | `docs/database/ddl/` | 从数据库自动导出,按 schema 分文件 |
|
||||
| ODS→DWD 字段映射 | `docs/database/BD_Manual_*.md` | 跨层映射(ODS 表 → DWD 表) |
|
||||
| 表级字段说明 | 本目录 `*/main/BD_manual_*.md` | 单表字段详情 |
|
||||
| API→ODS 字段映射 | 本目录 `ODS/mappings/` | API JSON → ODS 列映射 |
|
||||
|
||||
@@ -51,7 +51,7 @@ graph LR
|
||||
| 文档 | 说明 |
|
||||
|------|------|
|
||||
| [BaseTask 公共机制](base_task_mechanism.md) | 任务基类模板方法、TaskContext、时间窗口、注册表、Flow 执行 |
|
||||
| [ODS 层任务](ods_tasks.md) | 23 个通用 ODS 任务的架构、配置结构、API 端点、目标表 |
|
||||
| [ODS 层任务](ods_tasks.md) | 22 个通用 ODS 任务的架构、配置结构、API 端点、目标表 |
|
||||
| [DWD 层任务](dwd_tasks.md) | DWD_LOAD_FROM_ODS 核心装载、SCD2 处理、质量校验 |
|
||||
| [DWS 层任务](dws_tasks.md) | 助教业绩、会员分析、财务统计、运维任务共 13 个 DWS 任务 |
|
||||
| [INDEX 层任务](index_tasks.md) | WBI/NCI/RS 指数算法 + ML 手动台账导入 |
|
||||
@@ -88,7 +88,6 @@ graph LR
|
||||
| `ODS_TABLE_FEE_DISCOUNT` | `OdsTableDiscountTask` | `ods.table_fee_discount_records` | 台费折扣/调账 | [查看](ods_tasks.md) |
|
||||
| `ODS_STORE_GOODS_SALES` | `OdsGoodsLedgerTask` | `ods.store_goods_sales_records` | 门店商品销售流水 | [查看](ods_tasks.md) |
|
||||
| `ODS_TENANT_GOODS` | `OdsTenantGoodsTask` | `ods.tenant_goods_master` | 租户商品档案 | [查看](ods_tasks.md) |
|
||||
| `ODS_SETTLEMENT_TICKET` | `OdsSettlementTicketTask` | `ods.settlement_ticket_details` | 结账小票详情 | [查看](ods_tasks.md) |
|
||||
| `ODS_SETTLEMENT_RECORDS` | `OdsOrderSettleTask` | `ods.settlement_records` | 结账记录 | [查看](ods_tasks.md) |
|
||||
|
||||
### DWD 层(明细数据)
|
||||
@@ -280,6 +279,7 @@ Flow(执行流程)定义了多层任务的执行顺序。通过 `--flow` 参
|
||||
| `--idle-start` | str | — | 闲时窗口开始(HH:MM) |
|
||||
| `--idle-end` | str | — | 闲时窗口结束(HH:MM) |
|
||||
| `--allow-empty-advance` | flag | `false` | 允许空结果推进窗口 |
|
||||
| `--force-full` | flag | `false` | 强制全量处理:跳过 ODS hash 去重和 DWD 变更对比,无条件写入 |
|
||||
|
||||
### 已弃用参数
|
||||
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
|------|------|----------|
|
||||
| Task 1 | 新增 `SnapshotMode` 枚举(`FULL_SET` / `TIME_WINDOW` / `NONE`) | `tasks/ods/ods_tasks.py` |
|
||||
| Task 2 | content_hash 算法从 MD5 改为 SHA-256 | `tasks/ods/ods_tasks.py` |
|
||||
| Task 3 | ODS DDL 补齐 `content_hash` 列 + `is_delete` 列 + `(pk, fetched_at DESC)` 索引 | `db/etl_feiqiu/schemas/ods.sql` |
|
||||
| Task 4 | 迁移脚本:为已有 ODS 表添加 `latest_version` 索引 | `db/etl_feiqiu/migrations/2026-02-17__add_ods_latest_version_indexes.sql` |
|
||||
| Task 3 | ODS DDL 补齐 `content_hash` 列 + `is_delete` 列 + `(pk, fetched_at DESC)` 索引 | `docs/database/ddl/etl_feiqiu__ods.sql`(原 `db/etl_feiqiu/schemas/ods.sql`,已归档) |
|
||||
| Task 4 | 迁移脚本:为已有 ODS 表添加 `latest_version` 索引 | `db/_archived/ddl_baseline_2026-02-22/db/etl_feiqiu/migrations/2026-02-17__add_ods_latest_version_indexes.sql`(已归档) |
|
||||
| Task 5 | 软删除重构:`_mark_missing_as_deleted` 改为 INSERT 删除版本行 | `tasks/ods/ods_tasks.py` |
|
||||
| Task 6 | 属性测试覆盖去重和软删除逻辑 | `tests/unit/test_ods_dedup_properties.py` |
|
||||
| Task 7-9 | 文档同步、集成验证 | 多文件 |
|
||||
|
||||
@@ -1480,3 +1480,107 @@ recharge_order_flag = (consume_money = 0 AND pay_amount > 0)
|
||||
|
||||
- `refreshed`:刷新的物化视图数量
|
||||
- `cleaned`:清理的历史数据行数
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 库存汇总域
|
||||
|
||||
库存汇总域包含 3 个任务,围绕商品库存的日度、周度、月度汇总展开。数据流向为:
|
||||
|
||||
```
|
||||
dwd_goods_stock_summary ──┬──► DWS_GOODS_STOCK_DAILY(日度汇总)
|
||||
├──► DWS_GOODS_STOCK_WEEKLY(周度汇总)
|
||||
└──► DWS_GOODS_STOCK_MONTHLY(月度汇总)
|
||||
```
|
||||
|
||||
三个任务均依赖 `DWD_LOAD_FROM_ODS` 完成后的 `dwd.dwd_goods_stock_summary` 数据。
|
||||
|
||||
---
|
||||
|
||||
### DWS_GOODS_STOCK_DAILY — 库存日度汇总
|
||||
|
||||
| 属性 | 值 |
|
||||
|------|-----|
|
||||
| 任务代码 | `DWS_GOODS_STOCK_DAILY` |
|
||||
| Python 类 | `GoodsStockDailyTask`(`tasks/dws/goods_stock_daily_task.py`) |
|
||||
| 目标表 | `dws.dws_goods_stock_daily_summary` |
|
||||
| 主键 | `site_id`, `stat_date`, `site_goods_id` |
|
||||
| 粒度 | 日期 + 门店 + 商品 |
|
||||
| 更新策略 | upsert(ON CONFLICT DO UPDATE) |
|
||||
| 更新频率 | 每日更新 |
|
||||
| 依赖 | `DWD_LOAD_FROM_ODS` |
|
||||
|
||||
#### 数据来源
|
||||
|
||||
| 来源表 | Schema | 用途 |
|
||||
|--------|--------|------|
|
||||
| `dwd_goods_stock_summary` | `dwd` | 库存汇总明细(按 fetched_at 日期聚合) |
|
||||
|
||||
#### 核心业务逻辑
|
||||
|
||||
- 按 `fetched_at` 的日期部分分组,同一天同一商品可能有多条 DWD 记录
|
||||
- 数值指标(入库/出库/销售等)取 SUM 聚合
|
||||
- `range_start_stock` 取当日第一条记录的值(期初快照)
|
||||
- `range_end_stock` / `current_stock` 取当日最后一条记录的值(期末快照)
|
||||
- `stat_period = 'daily'`
|
||||
|
||||
---
|
||||
|
||||
### DWS_GOODS_STOCK_WEEKLY — 库存周度汇总
|
||||
|
||||
| 属性 | 值 |
|
||||
|------|-----|
|
||||
| 任务代码 | `DWS_GOODS_STOCK_WEEKLY` |
|
||||
| Python 类 | `GoodsStockWeeklyTask`(`tasks/dws/goods_stock_weekly_task.py`) |
|
||||
| 目标表 | `dws.dws_goods_stock_weekly_summary` |
|
||||
| 主键 | `site_id`, `stat_date`, `site_goods_id` |
|
||||
| 粒度 | ISO 周 + 门店 + 商品 |
|
||||
| 更新策略 | upsert(ON CONFLICT DO UPDATE) |
|
||||
| 更新频率 | 每周更新 |
|
||||
| 依赖 | `DWD_LOAD_FROM_ODS` |
|
||||
|
||||
#### 数据来源
|
||||
|
||||
| 来源表 | Schema | 用途 |
|
||||
|--------|--------|------|
|
||||
| `dwd_goods_stock_summary` | `dwd` | 库存汇总明细(按 ISO 周聚合) |
|
||||
|
||||
#### 核心业务逻辑
|
||||
|
||||
- 按 ISO 周分组(isocalendar),`stat_date` = 该周的周一日期
|
||||
- 同一周同一商品可能有多条 DWD 记录
|
||||
- 数值指标取 SUM 聚合
|
||||
- `range_start_stock` 取该周第一条记录的值(期初快照)
|
||||
- `range_end_stock` / `current_stock` 取该周最后一条记录的值(期末快照)
|
||||
- `stat_period = 'weekly'`
|
||||
|
||||
---
|
||||
|
||||
### DWS_GOODS_STOCK_MONTHLY — 库存月度汇总
|
||||
|
||||
| 属性 | 值 |
|
||||
|------|-----|
|
||||
| 任务代码 | `DWS_GOODS_STOCK_MONTHLY` |
|
||||
| Python 类 | `GoodsStockMonthlyTask`(`tasks/dws/goods_stock_monthly_task.py`) |
|
||||
| 目标表 | `dws.dws_goods_stock_monthly_summary` |
|
||||
| 主键 | `site_id`, `stat_date`, `site_goods_id` |
|
||||
| 粒度 | 自然月 + 门店 + 商品 |
|
||||
| 更新策略 | upsert(ON CONFLICT DO UPDATE) |
|
||||
| 更新频率 | 每日更新当月数据 |
|
||||
| 依赖 | `DWD_LOAD_FROM_ODS` |
|
||||
|
||||
#### 数据来源
|
||||
|
||||
| 来源表 | Schema | 用途 |
|
||||
|--------|--------|------|
|
||||
| `dwd_goods_stock_summary` | `dwd` | 库存汇总明细(按自然月聚合) |
|
||||
|
||||
#### 核心业务逻辑
|
||||
|
||||
- 按自然月分组,`stat_date` = 该月的第一天(如 2026-01-01 代表 2026 年 1 月)
|
||||
- 同一月同一商品可能有多条 DWD 记录
|
||||
- 数值指标取 SUM 聚合
|
||||
- `range_start_stock` 取该月第一条记录的值(期初快照)
|
||||
- `range_end_stock` / `current_stock` 取该月最后一条记录的值(期末快照)
|
||||
- `stat_period = 'monthly'`
|
||||
|
||||
@@ -751,5 +751,5 @@ ORDER BY effective_from DESC
|
||||
| `compression_mode` | 1 | 压缩模式(默认 log1p) |
|
||||
| `use_smoothing` / `ewma_alpha` | 1 / 0.2 | EWMA 平滑 |
|
||||
|
||||
> 种子数据脚本:`database/seed_index_parameters.sql`
|
||||
> DDL 定义:`database/schema_dws.sql`(第 21 节)
|
||||
> 种子数据脚本:`db/etl_feiqiu/seeds/seed_index_parameters.sql`
|
||||
> DDL 定义:`docs/database/ddl/etl_feiqiu__dws.sql`
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# ODS 任务参数矩阵
|
||||
|
||||
> 本文档列举所有 23 个 ODS 任务的关键配置参数,说明每个参数在 ETL 处理流程中的作用,
|
||||
> 本文档列举所有 22 个 ODS 任务的关键配置参数,说明每个参数在 ETL 处理流程中的作用,
|
||||
> 并以通俗语言概述飞球 ETL 的 API → ODS 现状。
|
||||
>
|
||||
> 数据来源:`tasks/ods/ods_tasks.py` 中的 `ODS_TASK_SPECS` 声明。
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
### 从哪抓
|
||||
|
||||
从上游飞球 SaaS 平台的 REST API 抓取,共对接 22 个不同的 API 端点(加上小票的特殊接口共 23 个)。覆盖台球门店的核心业务:订单结账、支付退款、会员档案与余额、助教服务与废除、商品库存、台桌、团购套餐、台费折扣等。每个端点对应一个 ODS 任务。
|
||||
从上游飞球 SaaS 平台的 REST API 抓取,共对接 22 个不同的 API 端点。覆盖台球门店的核心业务:订单结账、支付退款、会员档案与余额、助教服务与废除、商品库存、台桌、团购套餐、台费折扣等。每个端点对应一个 ODS 任务。
|
||||
|
||||
### 怎么解析
|
||||
|
||||
@@ -20,11 +20,11 @@ API 返回的 JSON 响应通过两级路径定位数据:先按 `data_path`(
|
||||
|
||||
### 写到哪
|
||||
|
||||
全部写入 PostgreSQL 的 `ods.*` schema(对应实际的 `ods` schema),23 张表一一对应 23 个任务。
|
||||
全部写入 PostgreSQL 的 `ods.*` schema(对应实际的 `ods` schema),22 张表一一对应 22 个任务。
|
||||
|
||||
### 怎么去重
|
||||
|
||||
所有 23 个任务默认开启 `skip_unchanged=True`:基于原始 payload + is_delete 计算 content_hash(SHA-256),若与数据库中该业务 ID 最新版本的 hash 相同则跳过写入,避免无意义的版本膨胀。
|
||||
所有 22 个任务默认开启 `skip_unchanged=True`:基于原始 payload + is_delete 计算 content_hash(SHA-256),若与数据库中该业务 ID 最新版本的 hash 相同则跳过写入,避免无意义的版本膨胀。
|
||||
|
||||
冲突处理模式默认是 `update`(全字段对比,有变化才更新),也支持 `backfill`(只回填 NULL 列)和 `nothing`(跳过已存在记录),通过运行时配置 `run.ods_conflict_mode` 控制。
|
||||
|
||||
@@ -34,7 +34,7 @@ API 返回的 JSON 响应通过两级路径定位数据:先按 `data_path`(
|
||||
|
||||
- 全表快照(5 个任务):API 返回全量数据,本次没返回但数据库里有的记录插入一条 `is_delete=1` 的删除版本行。适用于维度表/档案表,数据量小。
|
||||
- 窗口快照(8 个任务):只在指定时间列的窗口范围内做软删除。适用于流水表,数据量大,只能按时间段比对。
|
||||
- 无快照(10 个任务):纯增量写入,不做软删除。
|
||||
- 无快照(9 个任务):纯增量写入,不做软删除。
|
||||
|
||||
下游 DWD 层取数规约:`DISTINCT ON (id) ORDER BY id, fetched_at DESC`,再过滤 `is_delete = 0`。
|
||||
|
||||
@@ -96,12 +96,10 @@ API 返回的 JSON 响应通过两级路径定位数据:先按 `data_path`(
|
||||
| `ODS_STORE_GOODS` | ❌ | 默认 | ❌ | ✅ | ✅ | `FULL_TABLE` | — |
|
||||
| `ODS_TABLE_FEE_DISCOUNT` | ❌ | 默认 | ❌ | ✅ | ✅ | `WINDOW` | `create_time` |
|
||||
| `ODS_TENANT_GOODS` | ❌ | 默认 | ❌ | ✅ | ✅ | `FULL_TABLE` | — |
|
||||
| `ODS_SETTLEMENT_TICKET` | ❌ | — | ✅ | ✅ | ✅ | `NONE` | — |
|
||||
|
||||
> - "默认" `time_fields` 表示 `(startTime, endTime)`
|
||||
> - "—" 表示该参数为 `None` 或不适用
|
||||
> - `skip_unchanged` 所有任务均为 `True`(默认值),基于 payload + is_delete 计算 content_hash
|
||||
> - `ODS_SETTLEMENT_TICKET` 的 `time_fields` 为 `—` 是因为它不走标准分页抓取流程,而是先收集 `orderSettleId` 再逐个调用小票接口
|
||||
> - `ODS_RECHARGE_SETTLE` 是唯一手动映射 60+ 列的任务(其余任务按 DB schema 自动匹配)
|
||||
|
||||
|
||||
@@ -109,7 +107,7 @@ API 返回的 JSON 响应通过两级路径定位数据:先按 `data_path`(
|
||||
|
||||
## 去重策略
|
||||
|
||||
所有 23 个任务统一使用 `skip_unchanged=True`(默认值):
|
||||
所有 22 个任务统一使用 `skip_unchanged=True`(默认值):
|
||||
|
||||
- content_hash 基于原始 payload(`json.dumps(sort_keys=True, separators=(',',':'), ensure_ascii=False)`)+ `is_delete` 值计算 SHA-256
|
||||
- 写入时与数据库中该业务 ID 最新版本的 content_hash 比对,相同则跳过
|
||||
@@ -121,6 +119,6 @@ API 返回的 JSON 响应通过两级路径定位数据:先按 `data_path`(
|
||||
|------|:---:|----------|------|
|
||||
| 全表快照 | `FULL_TABLE` | `ODS_ASSISTANT_ACCOUNT`、`ODS_MEMBER_CARD`、`ODS_GROUP_PACKAGE`、`ODS_STORE_GOODS`、`ODS_TENANT_GOODS`(5 个) | API 返回全量,不在返回集中的记录插入一条 `is_delete=1` 的删除版本行 |
|
||||
| 窗口快照 | `WINDOW` | `ODS_TABLE_USE`、`ODS_ASSISTANT_LEDGER`、`ODS_STORE_GOODS_SALES`、`ODS_REFUND`、`ODS_PLATFORM_COUPON`、`ODS_MEMBER_BALANCE`、`ODS_GROUP_BUY_REDEMPTION`、`ODS_TABLE_FEE_DISCOUNT`(8 个) | 仅在 `snapshot_time_column` 指定的时间列窗口范围内做软删除 |
|
||||
| 无快照 | `NONE` | 其余 10 个 | 纯增量写入,不做软删除 |
|
||||
| 无快照 | `NONE` | 其余 9 个 | 纯增量写入,不做软删除 |
|
||||
|
||||
软删除语义:INSERT 一条 `is_delete=1` 的新版本行(而非 UPDATE 历史版本),保持 ODS 追加写入的一致性。若最新版本已是 `is_delete=1` 则跳过(幂等)。
|
||||
|
||||
@@ -40,7 +40,7 @@ ODS 层采用**声明式配置**驱动的通用任务模式:由 `BaseOdsTask`
|
||||
| `ODS_STORE_GOODS` | `OdsStoreGoodsTask` | `/TenantGoods/GetGoodsInventoryList` | `store_goods_master` | 门店商品档案 |
|
||||
| `ODS_TABLE_FEE_DISCOUNT` | `OdsTableDiscountTask` | `/Site/GetTaiFeeAdjustList` | `table_fee_discount_records` | 台费折扣/调账 |
|
||||
| `ODS_TENANT_GOODS` | `OdsTenantGoodsTask` | `/TenantGoods/QueryTenantGoods` | `tenant_goods_master` | 租户商品档案 |
|
||||
| `ODS_SETTLEMENT_TICKET` | `OdsSettlementTicketTask` | `/Order/GetOrderSettleTicketNew` | `settlement_ticket_details` | 结账小票详情 |
|
||||
| `ODS_STAFF_INFO` | `OdsStaffInfoTask` | `/PersonnelManagement/SearchSystemStaffInfo` | `staff_info_master` | 员工档案(含在职/离职) |
|
||||
|
||||
> 所有目标表均位于 `ods` schema 下。
|
||||
|
||||
@@ -228,7 +228,7 @@ execute(cursor_data)
|
||||
|
||||
### content_hash 去重机制
|
||||
|
||||
`content_hash` 是通用 ODS 任务的核心去重手段,所有 23 个任务默认开启(`skip_unchanged=True`)。
|
||||
`content_hash` 是通用 ODS 任务的核心去重手段,所有 22 个任务默认开启(`skip_unchanged=True`)。
|
||||
|
||||
#### 计算方式
|
||||
|
||||
@@ -295,8 +295,6 @@ ORDER BY id, fetched_at DESC;
|
||||
| `ODS_STORE_GOODS` | 否 | `FULL_TABLE` | — | 门店商品档案 |
|
||||
| `ODS_TABLE_FEE_DISCOUNT` | 否 | `WINDOW` | `create_time` | 台费折扣/调账 |
|
||||
| `ODS_TENANT_GOODS` | 否 | `FULL_TABLE` | — | 租户商品档案 |
|
||||
| `ODS_SETTLEMENT_TICKET` | 否 | `NONE` | — | 结账小票详情(专用实现,见下文) |
|
||||
| `ODS_STAFF_INFO` | 否 | `FULL_TABLE` | — | 员工档案,全量快照 |
|
||||
|
||||
> 所有 23 个任务默认 `skip_unchanged=True`(去重开启)。
|
||||
|
||||
> **特殊任务**:`ODS_SETTLEMENT_TICKET` 虽然在 `ODS_TASK_SPECS` 中声明,但其 `ODS_TASK_CLASSES` 条目被 `OdsSettlementTicketTask` 专用实现覆盖。该任务不走标准分页抓取流程,而是先从 `payment_transactions` 表或支付 API 收集 `orderSettleId`,再逐个调用小票接口获取详情。
|
||||
|
||||
@@ -52,8 +52,11 @@ load()
|
||||
|
||||
| DDL 文件 | 创建的 Schema | 主要内容 |
|
||||
|----------|--------------|----------|
|
||||
| `database/schema_etl_admin.sql` | `meta` | `etl_task`(任务注册表)、`etl_cursor`(游标表)、`etl_run`(运行记录表) |
|
||||
| `database/schema_ODS_doc.sql` | `ods` | 20+ 张 ODS 原始表(member_profiles、settlement_records、payment_transactions 等) |
|
||||
| meta DDL(代码默认 `database/schema_etl_admin.sql`) | `meta` | `etl_task`(任务注册表)、`etl_cursor`(游标表)、`etl_run`(运行记录表) |
|
||||
| ODS DDL(代码默认 `database/schema_ODS_doc.sql`) | `ods` | 20+ 张 ODS 原始表(member_profiles、settlement_records、payment_transactions 等) |
|
||||
|
||||
> 注:模块内 `database/` 目录下的旧 DDL 文件已不存在。INIT 任务通过配置参数 `schema.ods_file` / `schema.etl_admin_file` 指定 DDL 路径。
|
||||
> 当前 DDL 基线统一维护在 `docs/database/ddl/`(如 `etl_feiqiu__ods.sql`、`etl_feiqiu__meta.sql`),使用时需通过配置参数指向正确路径。
|
||||
|
||||
### ODS DDL 清洗逻辑
|
||||
|
||||
@@ -75,8 +78,8 @@ ODS DDL 文件可能包含头部说明文本和 `COMMENT ON` 语句(CamelCase
|
||||
|
||||
| 参数 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `schema.ods_file` | `database/schema_ODS_doc.sql` | ODS DDL 文件路径 |
|
||||
| `schema.etl_admin_file` | `database/schema_etl_admin.sql` | meta DDL 文件路径 |
|
||||
| `schema.ods_file` | `database/schema_ODS_doc.sql`(⚠️ 旧路径,需配置为实际 DDL 位置) | ODS DDL 文件路径 |
|
||||
| `schema.etl_admin_file` | `database/schema_etl_admin.sql`(⚠️ 旧路径) | meta DDL 文件路径 |
|
||||
| `io.log_root` | — | 日志目录 |
|
||||
| `io.export_root` | — | 导出目录 |
|
||||
| `pipeline.fetch_root` | — | 抓取数据目录 |
|
||||
@@ -115,7 +118,9 @@ load()
|
||||
|
||||
| DDL 文件 | 创建的 Schema | 主要内容 |
|
||||
|----------|--------------|----------|
|
||||
| `database/schema_dwd_doc.sql` | `dwd` | 维度表(dim_*,含 SCD2 约束)、事实表(dwd_*、fact_*)、扩展表(*_ex) |
|
||||
| DWD DDL(代码默认 `database/schema_dwd_doc.sql`) | `dwd` | 维度表(dim_*,含 SCD2 约束)、事实表(dwd_*、fact_*)、扩展表(*_ex) |
|
||||
|
||||
> 注:DDL 基线见 `docs/database/ddl/etl_feiqiu__dwd.sql`,使用时需通过 `schema.dwd_file` 配置参数指向正确路径。
|
||||
|
||||
DWD DDL 的特殊处理:
|
||||
- 自动为含 `scd2_start_time` 列的表设置 SCD2 默认值(`scd2_start_time=now()`、`scd2_end_time='9999-12-31'`、`scd2_is_current=1`、`scd2_version=1`)
|
||||
@@ -126,7 +131,7 @@ DWD DDL 的特殊处理:
|
||||
|
||||
| 参数 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `schema.dwd_file` | `database/schema_dwd_doc.sql` | DWD DDL 文件路径 |
|
||||
| `schema.dwd_file` | `database/schema_dwd_doc.sql`(⚠️ 旧路径,需配置为实际 DDL 位置) | DWD DDL 文件路径 |
|
||||
| `dwd.drop_schema_first` | `False` | 是否先 DROP 再重建(危险操作,会丢失所有 DWD 数据) |
|
||||
|
||||
### CLI 示例
|
||||
@@ -166,7 +171,9 @@ load()
|
||||
|
||||
| DDL 文件 | 创建的 Schema | 主要内容 |
|
||||
|----------|--------------|----------|
|
||||
| `database/schema_dws.sql` | `dws` | 配置表(5 张 cfg_*)、助教域(5 张)、会员域(2 张)、财务域(7 张)、订单汇总(1 张) |
|
||||
| DWS DDL(代码默认 `database/schema_dws.sql`) | `dws` | 配置表(5 张 cfg_*)、助教域(5 张)、会员域(2 张)、财务域(7 张)、订单汇总(1 张) |
|
||||
|
||||
> 注:DDL 基线见 `docs/database/ddl/etl_feiqiu__dws.sql`,使用时需通过 `schema.dws_file` 配置参数指向正确路径。
|
||||
|
||||
DWS Schema 包含的配置表:
|
||||
|
||||
@@ -182,7 +189,7 @@ DWS Schema 包含的配置表:
|
||||
|
||||
| 参数 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `schema.dws_file` | `database/schema_dws.sql` | DWS DDL 文件路径 |
|
||||
| `schema.dws_file` | `database/schema_dws.sql`(⚠️ 旧路径,需配置为实际 DDL 位置) | DWS DDL 文件路径 |
|
||||
| `dws.drop_schema_first` | `False` | 是否先 DROP 再重建(危险操作) |
|
||||
|
||||
### CLI 示例
|
||||
@@ -231,7 +238,7 @@ load()
|
||||
|
||||
| 参数 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `schema.seed_dws_file` | `database/seed_dws_config.sql` | 种子数据 SQL 文件路径 |
|
||||
| `schema.seed_dws_file` | `database/seed_dws_config.sql`(⚠️ 旧路径,种子数据现位于 `db/etl_feiqiu/seeds/seed_dws_config.sql`) | 种子数据 SQL 文件路径 |
|
||||
|
||||
### CLI 示例
|
||||
|
||||
@@ -290,7 +297,6 @@ execute()
|
||||
| `platform_coupon_redemption_records` | `ods.platform_coupon_redemption_records` |
|
||||
| `group_buy_redemption_records` | `ods.group_buy_redemption_records` |
|
||||
| `group_buy_packages` | `ods.group_buy_packages` |
|
||||
| `settlement_ticket_details` | `ods.settlement_ticket_details` |
|
||||
| `store_goods_master` | `ods.store_goods_master` |
|
||||
| `tenant_goods_master` | `ods.tenant_goods_master` |
|
||||
| `store_goods_sales_records` | `ods.store_goods_sales_records` |
|
||||
|
||||
@@ -51,15 +51,18 @@ uv sync --group test
|
||||
在项目根目录创建 `.env` 文件(禁止提交到版本控制):
|
||||
|
||||
```dotenv
|
||||
# 数据库连接
|
||||
# 数据库连接(推荐使用 DSN 模式)
|
||||
PG_DSN=postgresql://用户名:密码@主机:端口/数据库名
|
||||
# 或分别指定
|
||||
# 或分离式配置(不使用 DSN 时启用)
|
||||
PG_HOST=localhost
|
||||
PG_PORT=5432
|
||||
PG_NAME=billiards
|
||||
PG_NAME=your_database
|
||||
PG_USER=your_user
|
||||
PG_PASSWORD=your_password
|
||||
|
||||
# 业务库 DSN(后端 / 跨模块脚本使用)
|
||||
APP_DB_DSN=postgresql://用户名:密码@主机:端口/数据库名
|
||||
|
||||
# 门店与 API
|
||||
STORE_ID=1
|
||||
API_TOKEN=your_bearer_token
|
||||
@@ -99,19 +102,22 @@ config/defaults.py → .env / 环境变量 → CLI 参数
|
||||
初始化步骤:
|
||||
|
||||
```bash
|
||||
# 1. 执行 DDL 创建表结构(DDL 文件位于 db/etl_feiqiu/schemas/)
|
||||
psql "$PG_DSN" -f db/etl_feiqiu/schemas/schema_etl_admin.sql
|
||||
psql "$PG_DSN" -f db/etl_feiqiu/schemas/schema_ODS_doc.sql
|
||||
psql "$PG_DSN" -f db/etl_feiqiu/schemas/schema_dwd_doc.sql
|
||||
psql "$PG_DSN" -f db/etl_feiqiu/schemas/schema_dws.sql
|
||||
# 推荐使用 CLI 工具任务初始化(DDL 基线见 docs/database/ddl/)
|
||||
python -m cli.main --tasks INIT_ODS_SCHEMA,INIT_DWD_SCHEMA,INIT_DWS_SCHEMA,SEED_DWS_CONFIG --pg-dsn "$PG_DSN"
|
||||
|
||||
# 2. 执行种子数据(如有)
|
||||
# 或手动执行 DDL(从 docs/database/ddl/ 获取最新基线)
|
||||
psql "$PG_DSN" -f docs/database/ddl/etl_feiqiu__meta.sql
|
||||
psql "$PG_DSN" -f docs/database/ddl/etl_feiqiu__ods.sql
|
||||
psql "$PG_DSN" -f docs/database/ddl/etl_feiqiu__dwd.sql
|
||||
psql "$PG_DSN" -f docs/database/ddl/etl_feiqiu__dws.sql
|
||||
|
||||
# 执行种子数据
|
||||
psql "$PG_DSN" -f db/etl_feiqiu/seeds/seed_*.sql
|
||||
|
||||
# 3. 执行迁移脚本(按日期前缀顺序)
|
||||
ls db/etl_feiqiu/migrations/*.sql | sort | xargs -I {} psql "$PG_DSN" -f {}
|
||||
```
|
||||
|
||||
> 注:旧的 `db/etl_feiqiu/schemas/` 和 `db/etl_feiqiu/migrations/` 已归档至 `db/_archived/`。
|
||||
> DDL 基线现由 `docs/database/ddl/` 统一管理,可通过 `python scripts/ops/gen_consolidated_ddl.py` 重新生成。
|
||||
|
||||
或使用 CLI 工具任务初始化:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
参考路径:
|
||||
- DWD schema 文档:`docs/database/`
|
||||
- DWS DDL:`db/etl_feiqiu/schemas/`
|
||||
- DWS DDL:`docs/database/ddl/etl_feiqiu__dws.sql`
|
||||
- DWS 种子数据:`db/etl_feiqiu/seeds/`
|
||||
|
||||
---
|
||||
@@ -170,7 +170,7 @@ Top3 销冠奖(2026-03-01 起):第 1 名 1000 元,第 2 名 600 元,
|
||||
需求中明确 7/10/15/30/60/90 天窗口,在 `dws_assistant_customer_stats`、`dws_member_consumption_summary` 中直接落多窗口字段。
|
||||
|
||||
### 6.2 DDL 完整性
|
||||
所有方案中列出的表需在 `db/etl_feiqiu/schemas/` 中落全 DDL。
|
||||
所有方案中列出的表需在 `docs/database/ddl/etl_feiqiu__dws.sql` 中落全 DDL。
|
||||
|
||||
### 6.3 时间分层实现
|
||||
UI 需要"最近半年不含本月、上季度"等时间维度,并满足上个周期的环比。DWS 分层仅到 3 个月,可能需要额外聚合层。
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
{
|
||||
"task_code": "ODS_ASSISTANT_ABOLISH",
|
||||
"run_id": 8727,
|
||||
"endpoint": "/AssistantPerformance/GetAbolitionAssistant",
|
||||
"params": {
|
||||
"siteId": 2790685415443269,
|
||||
"startTime": "2026-02-09 22:00:00",
|
||||
"endTime": "2026-02-16 02:00:00"
|
||||
},
|
||||
"endpoint_routing": {
|
||||
"calls": [
|
||||
{
|
||||
"endpoint": "/AssistantPerformance/GetAbolitionAssistant",
|
||||
"params": {
|
||||
"siteId": 2790685415443269,
|
||||
"startTime": "2026-02-09 22:00:00",
|
||||
"endTime": "2026-02-16 02:00:00"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"page_size": 200,
|
||||
"pages": [
|
||||
{
|
||||
"page": 1,
|
||||
"request": {
|
||||
"siteId": 2790685415443269,
|
||||
"startTime": "2026-02-09 22:00:00",
|
||||
"endTime": "2026-02-16 02:00:00",
|
||||
"page": 1,
|
||||
"limit": 200
|
||||
},
|
||||
"response": {
|
||||
"data": {
|
||||
"total": 0,
|
||||
"abolitionAssistants": []
|
||||
},
|
||||
"code": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"total_records": 0,
|
||||
"dumped_at": "2026-02-15T18:05:45.781620Z"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,386 +0,0 @@
|
||||
{
|
||||
"task_code": "ODS_GOODS_CATEGORY",
|
||||
"run_id": 8730,
|
||||
"endpoint": "/TenantGoodsCategory/QueryPrimarySecondaryCategory",
|
||||
"params": {
|
||||
"siteId": 2790685415443269
|
||||
},
|
||||
"endpoint_routing": {
|
||||
"calls": [
|
||||
{
|
||||
"endpoint": "/TenantGoodsCategory/QueryPrimarySecondaryCategory",
|
||||
"params": {
|
||||
"siteId": 2790685415443269
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"page_size": 200,
|
||||
"pages": [
|
||||
{
|
||||
"page": 1,
|
||||
"request": {
|
||||
"siteId": 2790685415443269,
|
||||
"page": 1,
|
||||
"limit": 200
|
||||
},
|
||||
"response": {
|
||||
"data": {
|
||||
"total": 0,
|
||||
"goodsCategoryList": [
|
||||
{
|
||||
"id": 2790683528350533,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "槟榔",
|
||||
"alias_name": "",
|
||||
"pid": 0,
|
||||
"business_name": "槟榔",
|
||||
"tenant_goods_business_id": 2790683528317766,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [
|
||||
{
|
||||
"id": 2790683528350534,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "槟榔",
|
||||
"alias_name": "",
|
||||
"pid": 2790683528350533,
|
||||
"business_name": "槟榔",
|
||||
"tenant_goods_business_id": 2790683528317766,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
}
|
||||
],
|
||||
"sort": 1,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2790683528350535,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "器材",
|
||||
"alias_name": "",
|
||||
"pid": 0,
|
||||
"business_name": "器材",
|
||||
"tenant_goods_business_id": 2790683528317767,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [
|
||||
{
|
||||
"id": 2790683528350536,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "皮头",
|
||||
"alias_name": "",
|
||||
"pid": 2790683528350535,
|
||||
"business_name": "器材",
|
||||
"tenant_goods_business_id": 2790683528317767,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2790683528350537,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "球杆",
|
||||
"alias_name": "",
|
||||
"pid": 2790683528350535,
|
||||
"business_name": "器材",
|
||||
"tenant_goods_business_id": 2790683528317767,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2790683528350538,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "其他",
|
||||
"alias_name": "",
|
||||
"pid": 2790683528350535,
|
||||
"business_name": "器材",
|
||||
"tenant_goods_business_id": 2790683528317767,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
}
|
||||
],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2790683528350539,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "酒水",
|
||||
"alias_name": "",
|
||||
"pid": 0,
|
||||
"business_name": "酒水",
|
||||
"tenant_goods_business_id": 2790683528317768,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [
|
||||
{
|
||||
"id": 2790683528350540,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "饮料",
|
||||
"alias_name": "",
|
||||
"pid": 2790683528350539,
|
||||
"business_name": "酒水",
|
||||
"tenant_goods_business_id": 2790683528317768,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2790683528350541,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "酒水",
|
||||
"alias_name": "",
|
||||
"pid": 2790683528350539,
|
||||
"business_name": "酒水",
|
||||
"tenant_goods_business_id": 2790683528317768,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2790683528350542,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "茶水",
|
||||
"alias_name": "",
|
||||
"pid": 2790683528350539,
|
||||
"business_name": "酒水",
|
||||
"tenant_goods_business_id": 2790683528317768,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2790683528350543,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "咖啡",
|
||||
"alias_name": "",
|
||||
"pid": 2790683528350539,
|
||||
"business_name": "酒水",
|
||||
"tenant_goods_business_id": 2790683528317768,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2790683528350544,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "加料",
|
||||
"alias_name": "",
|
||||
"pid": 2790683528350539,
|
||||
"business_name": "酒水",
|
||||
"tenant_goods_business_id": 2790683528317768,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2793221553489733,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "洋酒",
|
||||
"alias_name": "",
|
||||
"pid": 2790683528350539,
|
||||
"business_name": "酒水",
|
||||
"tenant_goods_business_id": 2790683528317768,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
}
|
||||
],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2790683528350545,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "果盘",
|
||||
"alias_name": "",
|
||||
"pid": 0,
|
||||
"business_name": "水果",
|
||||
"tenant_goods_business_id": 2790683528317769,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [
|
||||
{
|
||||
"id": 2792050275864453,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "果盘",
|
||||
"alias_name": "",
|
||||
"pid": 2790683528350545,
|
||||
"business_name": "水果",
|
||||
"tenant_goods_business_id": 2790683528317769,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
}
|
||||
],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2791941988405125,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "零食",
|
||||
"alias_name": "",
|
||||
"pid": 0,
|
||||
"business_name": "零食",
|
||||
"tenant_goods_business_id": 2791932037238661,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [
|
||||
{
|
||||
"id": 2791948300259205,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "零食",
|
||||
"alias_name": "",
|
||||
"pid": 2791941988405125,
|
||||
"business_name": "零食",
|
||||
"tenant_goods_business_id": 2791932037238661,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2793236829620037,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "面",
|
||||
"alias_name": "",
|
||||
"pid": 2791941988405125,
|
||||
"business_name": "零食",
|
||||
"tenant_goods_business_id": 2791932037238661,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
}
|
||||
],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2791942087561093,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "雪糕",
|
||||
"alias_name": "",
|
||||
"pid": 0,
|
||||
"business_name": "雪糕",
|
||||
"tenant_goods_business_id": 2791931866402693,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [
|
||||
{
|
||||
"id": 2792035069284229,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "雪糕",
|
||||
"alias_name": "",
|
||||
"pid": 2791942087561093,
|
||||
"business_name": "雪糕",
|
||||
"tenant_goods_business_id": 2791931866402693,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
}
|
||||
],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2792062778003333,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "香烟",
|
||||
"alias_name": "",
|
||||
"pid": 0,
|
||||
"business_name": "香烟",
|
||||
"tenant_goods_business_id": 2790683528317765,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [
|
||||
{
|
||||
"id": 2792063209623429,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "香烟",
|
||||
"alias_name": "",
|
||||
"pid": 2792062778003333,
|
||||
"business_name": "香烟",
|
||||
"tenant_goods_business_id": 2790683528317765,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 1,
|
||||
"is_warehousing": 1
|
||||
}
|
||||
],
|
||||
"sort": 1,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2793217944864581,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "其他",
|
||||
"alias_name": "",
|
||||
"pid": 0,
|
||||
"business_name": "其他",
|
||||
"tenant_goods_business_id": 2793217599407941,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [
|
||||
{
|
||||
"id": 2793218343257925,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "其他2",
|
||||
"alias_name": "",
|
||||
"pid": 2793217944864581,
|
||||
"business_name": "其他",
|
||||
"tenant_goods_business_id": 2793217599407941,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
}
|
||||
],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
},
|
||||
{
|
||||
"id": 2793220945250117,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "小吃",
|
||||
"alias_name": "",
|
||||
"pid": 0,
|
||||
"business_name": "小吃",
|
||||
"tenant_goods_business_id": 2793220268902213,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [
|
||||
{
|
||||
"id": 2793221283104581,
|
||||
"tenant_id": 2790683160709957,
|
||||
"category_name": "小吃",
|
||||
"alias_name": "",
|
||||
"pid": 2793220945250117,
|
||||
"business_name": "小吃",
|
||||
"tenant_goods_business_id": 2793220268902213,
|
||||
"open_salesman": 2,
|
||||
"categoryBoxes": [],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
}
|
||||
],
|
||||
"sort": 0,
|
||||
"is_warehousing": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"code": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"total_records": 9,
|
||||
"dumped_at": "2026-02-15T18:06:02.836562Z"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,867 +0,0 @@
|
||||
{
|
||||
"task_code": "ODS_GROUP_PACKAGE",
|
||||
"run_id": 8732,
|
||||
"endpoint": "/PackageCoupon/QueryPackageCouponList",
|
||||
"params": {
|
||||
"siteId": 2790685415443269
|
||||
},
|
||||
"endpoint_routing": {
|
||||
"calls": [
|
||||
{
|
||||
"endpoint": "/PackageCoupon/QueryPackageCouponList",
|
||||
"params": {
|
||||
"siteId": 2790685415443269
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"page_size": 200,
|
||||
"pages": [
|
||||
{
|
||||
"page": 1,
|
||||
"request": {
|
||||
"siteId": 2790685415443269,
|
||||
"page": 1,
|
||||
"limit": 200
|
||||
},
|
||||
"response": {
|
||||
"data": {
|
||||
"total": 18,
|
||||
"packageCouponList": [
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791960001957765
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"A区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3048468851870085,
|
||||
"add_end_clock": "1.02:00:00",
|
||||
"add_start_clock": "23:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2026-01-12 22:42:55",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 3600,
|
||||
"end_clock": "1.02:00:00",
|
||||
"end_time": "2027-01-13 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1134850618,
|
||||
"package_name": "午夜场9.9",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "23:00:00",
|
||||
"start_time": "2026-01-12 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791961347968901
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"斯诺克区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3035353129373509,
|
||||
"add_end_clock": "23:59:59",
|
||||
"add_start_clock": "10:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2026-01-03 16:20:55",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "0",
|
||||
"date_type": 1,
|
||||
"duration": 7200,
|
||||
"end_clock": "23:59:59",
|
||||
"end_time": "2027-01-04 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1130443985,
|
||||
"package_name": "斯诺克两小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "10:00:00",
|
||||
"start_time": "2026-01-03 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 3,
|
||||
"tenantTableAreaIdList": [
|
||||
2791961347968901
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"斯诺克区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3035353102028549,
|
||||
"add_end_clock": "23:59:59",
|
||||
"add_start_clock": "10:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2026-01-03 16:20:53",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "0",
|
||||
"date_type": 1,
|
||||
"duration": 7200,
|
||||
"end_clock": "23:59:59",
|
||||
"end_time": "2027-01-04 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 2,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1130443985,
|
||||
"package_name": "斯诺克两小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "10:00:00",
|
||||
"start_time": "2026-01-03 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 3,
|
||||
"tenantTableAreaIdList": [
|
||||
2791961347968901
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"斯诺克区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3035346503502213,
|
||||
"add_end_clock": "23:59:59",
|
||||
"add_start_clock": "10:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2026-01-03 16:14:10",
|
||||
"creator_name": "管理员:郑丽珊",
|
||||
"date_info": "0",
|
||||
"date_type": 1,
|
||||
"duration": 7200,
|
||||
"end_clock": "23:59:59",
|
||||
"end_time": "2026-01-03 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 2,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1130443985,
|
||||
"package_name": "斯诺克两小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "10:00:00",
|
||||
"start_time": "2026-01-03 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791960001957765
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"A区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3031000036133445,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-31 14:32:43",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 3600,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2027-01-01 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1203035334,
|
||||
"package_name": "新人特惠一小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-12-31 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 3,
|
||||
"tenantTableAreaIdList": [
|
||||
2791961060364165
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"VIP包厢"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3030876421573701,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-31 12:26:58",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 7200,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2027-01-01 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 2,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1812429097416714,
|
||||
"package_name": "中八、斯诺克包厢两小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-07-21 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 2,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 3,
|
||||
"tenantTableAreaIdList": [
|
||||
2791960001957765
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"A区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3030875431701445,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-31 12:25:58",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 3600,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2027-01-01 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 2,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1814707240811572,
|
||||
"package_name": "新人特惠A区中八一小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-07-21 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 2,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791961709907845
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"K包"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3030874907937861,
|
||||
"add_end_clock": "18:00:00",
|
||||
"add_start_clock": "10:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-31 12:25:26",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 14400,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2027-01-01 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1137882866,
|
||||
"package_name": "KTV欢唱四小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-07-21 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791960001957765
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"A区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3030874716834757,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-31 12:25:14",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 7200,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2027-09-01 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1130465371,
|
||||
"package_name": "全天A区中八两小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-07-21 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791960521691013
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"B区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3030874133269445,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-31 12:24:38",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 7200,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2027-01-01 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1137872168,
|
||||
"package_name": "全天B区中八两小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-07-21 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 3,
|
||||
"tenantTableAreaIdList": [
|
||||
2791961347968901
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"斯诺克区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3030873639701573,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-31 12:24:08",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 7200,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2027-01-01 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 2,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1814983609169019,
|
||||
"package_name": "斯诺克两小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-07-21 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 2,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791960001957765
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"A区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3030873437310021,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-31 12:23:56",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 7200,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2027-01-01 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1173128804,
|
||||
"package_name": "助理教练竞技教学两小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-07-21 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 3,
|
||||
"tenantTableAreaIdList": [
|
||||
2791961060364165
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"VIP包厢"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3030873263688773,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-31 12:23:45",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 7200,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2027-01-01 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 2,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1812429097416714,
|
||||
"package_name": "中八、斯诺克包厢两小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-07-21 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 2,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791961060364165
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"VIP包厢"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3030872859429829,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-31 12:23:21",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 7200,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2027-01-01 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1126976372,
|
||||
"package_name": "中八、斯诺克包厢两小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-07-22 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791960001957765
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"A区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3030872476945477,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-31 12:22:57",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 3600,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2027-01-01 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1128411555,
|
||||
"package_name": "全天A区中八一小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-07-21 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791961347968901
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"斯诺克区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3030025905916997,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-30 22:01:47",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 3600,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2026-12-31 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1147633733,
|
||||
"package_name": "全天斯诺克一小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-12-30 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791962314215301
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"麻将房"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 3029784419027909,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-12-30 17:56:07",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 14400,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2026-12-31 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1134269810,
|
||||
"package_name": "麻将包厢4小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-12-30 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
},
|
||||
{
|
||||
"site_name": "朗朗桌球",
|
||||
"effective_status": 1,
|
||||
"tenantTableAreaIdList": [
|
||||
2791960521691013
|
||||
],
|
||||
"tableAreaNameList": [
|
||||
"B区"
|
||||
],
|
||||
"tenantCouponSaleOrderItemId": 0,
|
||||
"id": 2861343275830405,
|
||||
"add_end_clock": "1.00:00:00",
|
||||
"add_start_clock": "00:00:00",
|
||||
"area_tag_type": 1,
|
||||
"card_type_ids": "0",
|
||||
"coupon_money": 0.0,
|
||||
"create_time": "2025-09-02 18:08:56",
|
||||
"creator_name": "店长:郑丽珊",
|
||||
"date_info": "",
|
||||
"date_type": 1,
|
||||
"duration": 3600,
|
||||
"end_clock": "1.00:00:00",
|
||||
"end_time": "2026-09-03 00:00:00",
|
||||
"group_type": 1,
|
||||
"is_delete": 0,
|
||||
"is_enabled": 1,
|
||||
"is_first_limit": 1,
|
||||
"max_selectable_categories": 0,
|
||||
"package_id": 1370841337,
|
||||
"package_name": "B区桌球一小时",
|
||||
"selling_price": 0.0,
|
||||
"site_id": 2790685415443269,
|
||||
"sort": 100,
|
||||
"start_clock": "00:00:00",
|
||||
"start_time": "2025-09-02 00:00:00",
|
||||
"system_group_type": 1,
|
||||
"table_area_id": "0",
|
||||
"table_area_id_list": "",
|
||||
"table_area_name": "B区",
|
||||
"tenant_id": 2790683160709957,
|
||||
"tenant_table_area_id": "0",
|
||||
"tenant_table_area_id_list": "2791960521691013",
|
||||
"type": 1,
|
||||
"usable_count": 0,
|
||||
"usable_range": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
"code": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"total_records": 18,
|
||||
"dumped_at": "2026-02-15T18:08:54.360629Z"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,44 +0,0 @@
|
||||
{
|
||||
"task_code": "ODS_RECHARGE_SETTLE",
|
||||
"run_id": 8740,
|
||||
"endpoint": "/Site/GetRechargeSettleList",
|
||||
"params": {
|
||||
"siteId": 2790685415443269,
|
||||
"rangeStartTime": "2026-02-09 22:00:00",
|
||||
"rangeEndTime": "2026-02-16 02:00:00"
|
||||
},
|
||||
"endpoint_routing": {
|
||||
"calls": [
|
||||
{
|
||||
"endpoint": "/Site/GetRechargeSettleList",
|
||||
"params": {
|
||||
"siteId": 2790685415443269,
|
||||
"rangeStartTime": "2026-02-09 22:00:00",
|
||||
"rangeEndTime": "2026-02-16 02:00:00"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"page_size": 200,
|
||||
"pages": [
|
||||
{
|
||||
"page": 1,
|
||||
"request": {
|
||||
"siteId": 2790685415443269,
|
||||
"rangeStartTime": "2026-02-09 22:00:00",
|
||||
"rangeEndTime": "2026-02-16 02:00:00",
|
||||
"page": 1,
|
||||
"limit": 200
|
||||
},
|
||||
"response": {
|
||||
"data": {
|
||||
"total": 0,
|
||||
"settleList": []
|
||||
},
|
||||
"code": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"total_records": 0,
|
||||
"dumped_at": "2026-02-15T18:17:37.586917Z"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,38 +0,0 @@
|
||||
{
|
||||
"task_code": "ODS_STORE_GOODS_SALES",
|
||||
"run_id": 8745,
|
||||
"endpoint": "/TenantGoods/GetGoodsSalesList",
|
||||
"params": {
|
||||
"siteId": 2790685415443269
|
||||
},
|
||||
"endpoint_routing": {
|
||||
"calls": [
|
||||
{
|
||||
"endpoint": "/TenantGoods/GetGoodsSalesList",
|
||||
"params": {
|
||||
"siteId": 2790685415443269
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"page_size": 200,
|
||||
"pages": [
|
||||
{
|
||||
"page": 1,
|
||||
"request": {
|
||||
"siteId": 2790685415443269,
|
||||
"page": 1,
|
||||
"limit": 200
|
||||
},
|
||||
"response": {
|
||||
"data": {
|
||||
"total": 0,
|
||||
"orderGoodsLedgers": []
|
||||
},
|
||||
"code": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"total_records": 0,
|
||||
"dumped_at": "2026-02-15T18:18:24.817712Z"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -4,18 +4,21 @@
|
||||
从原 ETLScheduler 中提取 Flow 编排逻辑,委托 TaskExecutor 执行具体任务。
|
||||
所有依赖通过构造函数注入,不自行创建资源。
|
||||
|
||||
术语说明:代码中保留 pipeline 参数名以兼容调用方,概念上统一使用 Flow。
|
||||
术语说明:统一使用 Flow 概念,pipeline 参数已移除。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from tasks.verification import filter_verify_tables
|
||||
from orchestration.topological_sort import topological_sort
|
||||
from utils.timer import EtlTimer
|
||||
|
||||
|
||||
class FlowRunner:
|
||||
@@ -49,9 +52,10 @@ class FlowRunner:
|
||||
self.logger = logger
|
||||
self.tz = ZoneInfo(config.get("app.timezone", "Asia/Shanghai"))
|
||||
|
||||
# CHANGE [2026-02-20] intent: 移除 pipeline 参数,统一使用 flow(消除历史别名)
|
||||
def run(
|
||||
self,
|
||||
pipeline: str | None = None,
|
||||
flow: str | None = None,
|
||||
layers: list[str] | None = None,
|
||||
processing_mode: str = "increment_only",
|
||||
data_source: str = "hybrid",
|
||||
@@ -65,9 +69,9 @@ class FlowRunner:
|
||||
"""执行 Flow,返回汇总结果。
|
||||
|
||||
Args:
|
||||
pipeline: Flow 名称 (api_ods, api_ods_dwd, api_full, ...),与 layers 二选一(参数名保留以兼容调用方)
|
||||
flow: Flow 名称 (api_ods, api_ods_dwd, api_full, ...),与 layers 二选一
|
||||
layers: 直接指定层列表 (["ODS", "DWD"] 等),与 flow 名称二选一
|
||||
processing_mode: 处理模式 (increment_only / verify_only / increment_verify)
|
||||
processing_mode: 处理模式 (increment_only / verify_only / increment_verify / full_window)
|
||||
data_source: 数据源模式 (online / offline / hybrid)
|
||||
window_start: 时间窗口开始
|
||||
window_end: 时间窗口结束
|
||||
@@ -77,16 +81,16 @@ class FlowRunner:
|
||||
verify_tables: 指定校验的表名列表(可用于单表验证)
|
||||
|
||||
Returns:
|
||||
执行结果字典,包含 status / pipeline / layers / results / verification_summary
|
||||
执行结果字典,包含 status / flow / layers / results / verification_summary
|
||||
"""
|
||||
from utils.task_logger import TaskLogger
|
||||
|
||||
# 解析层列表:Flow 名称查找 或 直接使用 layers 参数
|
||||
if pipeline is not None:
|
||||
if pipeline not in self.FLOW_LAYERS:
|
||||
raise ValueError(f"无效的 Flow 名称: {pipeline}")
|
||||
resolved_layers = self.FLOW_LAYERS[pipeline]
|
||||
run_label = pipeline
|
||||
if flow is not None:
|
||||
if flow not in self.FLOW_LAYERS:
|
||||
raise ValueError(f"无效的 Flow 名称: {flow}")
|
||||
resolved_layers = self.FLOW_LAYERS[flow]
|
||||
run_label = flow
|
||||
elif layers is not None:
|
||||
resolved_layers = layers
|
||||
run_label = f"layers({','.join(layers)})"
|
||||
@@ -97,6 +101,12 @@ class FlowRunner:
|
||||
flow_logger = TaskLogger(f"FLOW_{run_label.upper()}", self.logger)
|
||||
flow_logger.start(f"开始执行 Flow: {run_label}")
|
||||
|
||||
# CHANGE [2026-02-20] intent: 集成 EtlTimer 到 Flow 执行流程,记录每个任务的耗时(需求 15.1, 15.2, 15.3)
|
||||
# assumptions: EtlTimer 已通过单元测试,输出路径由 ETL_REPORT_ROOT 环境变量控制
|
||||
# prompt: "将 EtlTimer 集成到 FlowRunner.run()"
|
||||
timer = EtlTimer(tz=self.tz)
|
||||
timer.start()
|
||||
|
||||
layers = resolved_layers
|
||||
results: list[dict[str, Any]] = []
|
||||
verification_summary: dict[str, Any] | None = None
|
||||
@@ -119,12 +129,16 @@ class FlowRunner:
|
||||
ods_tasks = [t for t in task_codes if t.startswith("ODS_")]
|
||||
if ods_tasks:
|
||||
self.logger.info("从 API 获取数据: %s", ods_tasks)
|
||||
timer.start_step("FETCH_BEFORE_VERIFY")
|
||||
results = self.task_executor.run_tasks(ods_tasks, data_source=data_source)
|
||||
timer.stop_step("FETCH_BEFORE_VERIFY")
|
||||
else:
|
||||
auto_tasks = self._resolve_tasks(["ODS"])
|
||||
if auto_tasks:
|
||||
self.logger.info("从 API 获取数据: %s", auto_tasks)
|
||||
timer.start_step("FETCH_BEFORE_VERIFY")
|
||||
results = self.task_executor.run_tasks(auto_tasks, data_source=data_source)
|
||||
timer.stop_step("FETCH_BEFORE_VERIFY")
|
||||
|
||||
ods_dump_dirs = {
|
||||
r.get("task_code"): r.get("dump_dir")
|
||||
@@ -150,15 +164,18 @@ class FlowRunner:
|
||||
# 增量 ETL(increment_only 或 increment_verify)
|
||||
self.logger.info("Flow %s: 执行增量 ETL,层=%s", run_label, layers)
|
||||
|
||||
timer.start_step("INCREMENT_ETL")
|
||||
if task_codes:
|
||||
results = self.task_executor.run_tasks(task_codes, data_source=data_source)
|
||||
else:
|
||||
auto_tasks = self._resolve_tasks(layers)
|
||||
results = self.task_executor.run_tasks(auto_tasks, data_source=data_source)
|
||||
timer.stop_step("INCREMENT_ETL")
|
||||
|
||||
# increment_verify 模式:增量后执行校验
|
||||
if processing_mode == "increment_verify":
|
||||
self.logger.info("Flow %s: 开始校验并修复", run_label)
|
||||
timer.start_step("VERIFICATION")
|
||||
verification_summary = self._run_verification(
|
||||
layers=layers,
|
||||
window_start=window_start,
|
||||
@@ -168,14 +185,36 @@ class FlowRunner:
|
||||
use_local_json=use_local_json,
|
||||
verify_tables=verify_tables,
|
||||
)
|
||||
timer.stop_step("VERIFICATION")
|
||||
flow_logger.set_verification_result(verification_summary)
|
||||
|
||||
# 汇总计数
|
||||
# CHANGE [2026-02-20] intent: 集成 ConsistencyChecker 到 Flow 执行流程,ETL 完成后自动运行一致性检查(需求 16.1, 16.4)
|
||||
# assumptions: ConsistencyChecker 已通过单元测试,报告输出到 ETL_REPORT_ROOT
|
||||
# prompt: "将 ConsistencyChecker 集成到 FlowRunner.run()"
|
||||
consistency_report_path = self._run_post_consistency_check(timer)
|
||||
|
||||
# 输出计时报告
|
||||
try:
|
||||
timing_report = timer.finish(write_report=True)
|
||||
self.logger.info("计时报告已生成")
|
||||
except KeyError as ke:
|
||||
self.logger.warning("计时报告输出跳过(环境变量缺失): %s", ke)
|
||||
timing_report = timer.finish(write_report=False)
|
||||
|
||||
# 汇总计数 — CHANGE 2026-02-21: BUG 11 fix — errors 可能是 list,需类型安全处理
|
||||
def _safe_int(val) -> int:
|
||||
"""将 int/list/None 统一转为 int 计数。"""
|
||||
if isinstance(val, int):
|
||||
return val
|
||||
if isinstance(val, list):
|
||||
return len(val)
|
||||
return 0
|
||||
|
||||
flow_logger.set_counts(
|
||||
fetched=sum(r.get("counts", {}).get("fetched", 0) for r in results),
|
||||
inserted=sum(r.get("counts", {}).get("inserted", 0) for r in results),
|
||||
updated=sum(r.get("counts", {}).get("updated", 0) for r in results),
|
||||
errors=sum(r.get("counts", {}).get("errors", 0) for r in results),
|
||||
fetched=sum(_safe_int(r.get("counts", {}).get("fetched", 0)) for r in results),
|
||||
inserted=sum(_safe_int(r.get("counts", {}).get("inserted", 0)) for r in results),
|
||||
updated=sum(_safe_int(r.get("counts", {}).get("updated", 0)) for r in results),
|
||||
errors=sum(_safe_int(r.get("counts", {}).get("errors", 0)) for r in results),
|
||||
)
|
||||
|
||||
summary_text = flow_logger.end(status="成功")
|
||||
@@ -183,17 +222,66 @@ class FlowRunner:
|
||||
|
||||
return {
|
||||
"status": "SUCCESS",
|
||||
"pipeline": run_label,
|
||||
"flow": run_label,
|
||||
"layers": layers,
|
||||
"results": results,
|
||||
"verification_summary": verification_summary,
|
||||
"consistency_report_path": consistency_report_path,
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
# 异常时也尝试输出计时报告(便于排查耗时瓶颈)
|
||||
try:
|
||||
timer.finish(write_report=True)
|
||||
except Exception:
|
||||
pass
|
||||
summary_text = flow_logger.end(status="失败", error_message=str(exc))
|
||||
self.logger.error("\n%s", summary_text)
|
||||
raise
|
||||
|
||||
def _run_post_consistency_check(self, timer: EtlTimer) -> str | None:
|
||||
"""ETL 完成后运行数据一致性检查,输出黑盒测试报告。
|
||||
|
||||
返回报告文件路径,失败时返回 None(不阻断主流程)。
|
||||
"""
|
||||
try:
|
||||
from quality.consistency_checker import (
|
||||
run_consistency_check,
|
||||
write_consistency_report,
|
||||
)
|
||||
except ImportError:
|
||||
self.logger.warning("一致性检查模块未安装,跳过")
|
||||
return None
|
||||
|
||||
timer.start_step("CONSISTENCY_CHECK")
|
||||
try:
|
||||
api_sample_dir_str = os.environ.get("API_SAMPLE_CACHE_ROOT")
|
||||
api_sample_dir = Path(api_sample_dir_str) if api_sample_dir_str else None
|
||||
|
||||
report = run_consistency_check(
|
||||
self.db_conn,
|
||||
api_sample_dir=api_sample_dir,
|
||||
include_api_vs_ods=bool(api_sample_dir),
|
||||
include_ods_vs_dwd=True,
|
||||
tz=self.tz,
|
||||
)
|
||||
|
||||
report_path = write_consistency_report(report)
|
||||
self.logger.info("一致性检查报告已生成: %s", report_path)
|
||||
return report_path
|
||||
|
||||
except KeyError as ke:
|
||||
self.logger.warning("一致性检查报告输出跳过(环境变量缺失): %s", ke)
|
||||
return None
|
||||
except Exception as exc:
|
||||
self.logger.warning("一致性检查失败(不阻断主流程): %s", exc, exc_info=True)
|
||||
return None
|
||||
finally:
|
||||
try:
|
||||
timer.stop_step("CONSISTENCY_CHECK")
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def _resolve_tasks(self, layers: list[str]) -> list[str]:
|
||||
"""根据层列表解析任务代码。
|
||||
|
||||
|
||||
@@ -20,8 +20,8 @@ from orchestration.task_executor import TaskExecutor
|
||||
from orchestration.flow_runner import FlowRunner
|
||||
|
||||
|
||||
# 保留模块级常量以兼容外部引用
|
||||
PIPELINE_LAYERS = FlowRunner.FLOW_LAYERS
|
||||
# CHANGE [2026-02-20] intent: 移除 PIPELINE_LAYERS 别名,统一使用 FlowRunner.FLOW_LAYERS
|
||||
FLOW_LAYERS = FlowRunner.FLOW_LAYERS
|
||||
|
||||
|
||||
class ETLScheduler:
|
||||
|
||||
@@ -40,11 +40,16 @@ from tasks.dws import (
|
||||
FinanceRechargeTask,
|
||||
FinanceIncomeStructureTask,
|
||||
FinanceDiscountDetailTask,
|
||||
# 库存汇总任务
|
||||
GoodsStockDailyTask,
|
||||
GoodsStockWeeklyTask,
|
||||
GoodsStockMonthlyTask,
|
||||
# 指数算法任务
|
||||
WinbackIndexTask,
|
||||
NewconvIndexTask,
|
||||
MlManualImportTask,
|
||||
RelationIndexTask,
|
||||
SpendingPowerIndexTask,
|
||||
)
|
||||
# CHANGE [2026-07-14] intent: 合并 MV 刷新 + 数据清理为 DWS_MAINTENANCE
|
||||
from tasks.dws.maintenance_task import DwsMaintenanceTask
|
||||
@@ -153,6 +158,10 @@ default_registry.register("DWS_FINANCE_DAILY", FinanceDailyTask, layer="DWS")
|
||||
default_registry.register("DWS_FINANCE_RECHARGE", FinanceRechargeTask, layer="DWS")
|
||||
default_registry.register("DWS_FINANCE_INCOME_STRUCTURE", FinanceIncomeStructureTask, layer="DWS")
|
||||
default_registry.register("DWS_FINANCE_DISCOUNT_DETAIL", FinanceDiscountDetailTask, layer="DWS")
|
||||
# CHANGE [2026-07-20] intent: 注册 DWS 库存汇总任务(日/周/月),依赖 DWD 装载完成(需求 12.9)
|
||||
default_registry.register("DWS_GOODS_STOCK_DAILY", GoodsStockDailyTask, layer="DWS", depends_on=["DWD_LOAD_FROM_ODS"])
|
||||
default_registry.register("DWS_GOODS_STOCK_WEEKLY", GoodsStockWeeklyTask, layer="DWS", depends_on=["DWD_LOAD_FROM_ODS"])
|
||||
default_registry.register("DWS_GOODS_STOCK_MONTHLY", GoodsStockMonthlyTask, layer="DWS", depends_on=["DWD_LOAD_FROM_ODS"])
|
||||
# CHANGE [2026-07-14] intent: 移除 DWS_RETENTION_CLEANUP / DWS_MV_REFRESH_FINANCE_DAILY / DWS_MV_REFRESH_ASSISTANT_DAILY,
|
||||
# 替换为统一维护任务 DWS_MAINTENANCE(需求 4.5)
|
||||
# depends_on: 所有其他 DWS 任务——MV 刷新和清理应在数据写入后执行
|
||||
@@ -163,6 +172,7 @@ default_registry.register("DWS_MAINTENANCE", DwsMaintenanceTask, layer="DWS", de
|
||||
"DWS_FINANCE_DAILY", "DWS_FINANCE_RECHARGE",
|
||||
"DWS_FINANCE_INCOME_STRUCTURE", "DWS_FINANCE_DISCOUNT_DETAIL",
|
||||
"DWS_BUILD_ORDER_SUMMARY",
|
||||
"DWS_GOODS_STOCK_DAILY", "DWS_GOODS_STOCK_WEEKLY", "DWS_GOODS_STOCK_MONTHLY",
|
||||
])
|
||||
|
||||
# ── INDEX 层:指数算法任务 ────────────────────────────────────
|
||||
@@ -171,3 +181,4 @@ default_registry.register("DWS_WINBACK_INDEX", WinbackIndexTask, requires_db_con
|
||||
default_registry.register("DWS_NEWCONV_INDEX", NewconvIndexTask, requires_db_config=False, layer="INDEX", depends_on=["DWS_MEMBER_VISIT", "DWS_MEMBER_CONSUMPTION"])
|
||||
default_registry.register("DWS_ML_MANUAL_IMPORT", MlManualImportTask, requires_db_config=False, layer="INDEX")
|
||||
default_registry.register("DWS_RELATION_INDEX", RelationIndexTask, requires_db_config=False, layer="INDEX", depends_on=["DWS_ASSISTANT_DAILY"])
|
||||
default_registry.register("DWS_SPENDING_POWER_INDEX", SpendingPowerIndexTask, requires_db_config=False, layer="INDEX", depends_on=["DWS_MEMBER_CONSUMPTION"])
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""数据质量检查模块"""
|
||||
|
||||
758
apps/etl/connectors/feiqiu/quality/consistency_checker.py
Normal file
758
apps/etl/connectors/feiqiu/quality/consistency_checker.py
Normal file
@@ -0,0 +1,758 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
数据一致性检查器(黑盒测试)
|
||||
|
||||
以黑盒测试者角度,检查 ETL 数据流各层之间的一致性:
|
||||
1. API 源数据 vs ODS 落库数据 — 字段完整性对比
|
||||
2. ODS 数据 vs DWD 落库数据 — 映射正确性对比
|
||||
|
||||
输出 Markdown 格式的黑盒测试报告。
|
||||
|
||||
Requirements: 16.1, 16.2, 16.3, 16.4
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Sequence, Tuple
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from tasks.dwd.dwd_load_task import DwdLoadTask
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 数据结构
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class FieldCheckResult:
|
||||
"""单个字段的检查结果"""
|
||||
field_name: str
|
||||
status: str # "pass" | "missing" | "mismatch" | "type_diff" | "skip"
|
||||
detail: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class TableCheckResult:
|
||||
"""单张表的检查结果"""
|
||||
table_name: str
|
||||
check_type: str # "api_vs_ods" | "ods_vs_dwd"
|
||||
passed: bool = True
|
||||
total_fields: int = 0
|
||||
passed_fields: int = 0
|
||||
missing_fields: int = 0
|
||||
mismatch_fields: int = 0
|
||||
field_results: List[FieldCheckResult] = field(default_factory=list)
|
||||
row_count_source: int = 0
|
||||
row_count_target: int = 0
|
||||
sample_mismatches: List[Dict[str, Any]] = field(default_factory=list)
|
||||
error: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConsistencyReport:
|
||||
"""完整的一致性检查报告"""
|
||||
generated_at: str = ""
|
||||
api_vs_ods_results: List[TableCheckResult] = field(default_factory=list)
|
||||
ods_vs_dwd_results: List[TableCheckResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def all_passed(self) -> bool:
|
||||
return all(r.passed for r in self.api_vs_ods_results + self.ods_vs_dwd_results)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ODS 表名 → API JSON 文件名映射(与 json_store.ENDPOINT_FILENAME_MAP 对齐)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ODS_TABLE_TO_JSON_FILE: Dict[str, str] = {
|
||||
"assistant_accounts_master": "assistant_accounts_master.json",
|
||||
"assistant_service_records": "assistant_service_records.json",
|
||||
"assistant_cancellation_records": "assistant_cancellation_records.json",
|
||||
"member_profiles": "member_profiles.json",
|
||||
"member_stored_value_cards": "member_stored_value_cards.json",
|
||||
"member_balance_changes": "member_balance_changes.json",
|
||||
"recharge_settlements": "recharge_settlements.json",
|
||||
"settlement_records": "settlement_records.json",
|
||||
"table_fee_transactions": "table_fee_transactions.json",
|
||||
"table_fee_discount_records": "table_fee_discount_records.json",
|
||||
"store_goods_sales_records": "store_goods_sales_records.json",
|
||||
"store_goods_master": "store_goods_master.json",
|
||||
"tenant_goods_master": "tenant_goods_master.json",
|
||||
"site_tables_master": "site_tables_master.json",
|
||||
"group_buy_packages": "group_buy_packages.json",
|
||||
"group_buy_redemption_records": "group_buy_redemption_records.json",
|
||||
"platform_coupon_redemption_records": "platform_coupon_redemption_records.json",
|
||||
"payment_transactions": "payment_transactions.json",
|
||||
"refund_transactions": "refund_transactions.json",
|
||||
"goods_stock_summary": "goods_stock_summary.json",
|
||||
"goods_stock_movements": "goods_stock_movements.json",
|
||||
"stock_goods_category_tree": "stock_goods_category_tree.json",
|
||||
}
|
||||
|
||||
# ODS 元数据列——不来自 API,由 ETL 框架自动填充
|
||||
ODS_META_COLUMNS = frozenset({
|
||||
"payload", "source_file", "source_endpoint",
|
||||
"fetched_at", "content_hash", "record_index",
|
||||
"site_id", "tenant_id", "siteprofile", "site_profile",
|
||||
})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 核心检查逻辑(纯函数,不依赖数据库连接)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def extract_api_fields_from_json(json_path: Path) -> set[str] | None:
|
||||
"""从 API JSON 缓存文件中提取第一条记录的所有字段名"""
|
||||
if not json_path.exists():
|
||||
return None
|
||||
try:
|
||||
with json_path.open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
|
||||
# 支持多种 JSON 结构:直接列表 / {"data": [...]} / {"data": {"xxxList": [...]}}
|
||||
records = _extract_records(data)
|
||||
if not records:
|
||||
return None
|
||||
|
||||
# 合并前 N 条记录的字段(避免单条记录字段不全)
|
||||
all_fields: set[str] = set()
|
||||
for rec in records[:10]:
|
||||
if isinstance(rec, dict):
|
||||
all_fields.update(rec.keys())
|
||||
return all_fields
|
||||
|
||||
|
||||
def _extract_records(data: Any) -> list[dict]:
|
||||
"""从 API 响应中提取记录列表"""
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
if isinstance(data, dict):
|
||||
# 尝试 data 键
|
||||
inner = data.get("data")
|
||||
if isinstance(inner, list):
|
||||
return inner
|
||||
if isinstance(inner, dict):
|
||||
# 尝试各种 list_key
|
||||
for key, val in inner.items():
|
||||
if isinstance(val, list) and val:
|
||||
return val
|
||||
return []
|
||||
|
||||
|
||||
def check_api_vs_ods_fields(
|
||||
api_fields: set[str],
|
||||
ods_columns: set[str],
|
||||
) -> TableCheckResult:
|
||||
"""
|
||||
对比 API JSON 字段与 ODS 表列的覆盖率。
|
||||
|
||||
检查逻辑:API 中的每个字段是否在 ODS 表中有对应列(小写匹配)。
|
||||
ODS 元数据列(payload, fetched_at 等)不参与对比。
|
||||
"""
|
||||
result = TableCheckResult(table_name="", check_type="api_vs_ods")
|
||||
|
||||
# 标准化为小写
|
||||
api_lower = {f.lower() for f in api_fields}
|
||||
ods_lower = ods_columns - ODS_META_COLUMNS
|
||||
|
||||
result.total_fields = len(api_lower)
|
||||
|
||||
for api_field in sorted(api_lower):
|
||||
if api_field in ods_lower:
|
||||
result.field_results.append(
|
||||
FieldCheckResult(api_field, "pass", "ODS 中存在同名列")
|
||||
)
|
||||
result.passed_fields += 1
|
||||
else:
|
||||
# 嵌套对象字段(含大写字母的驼峰命名)可能被展开或存入 payload
|
||||
result.field_results.append(
|
||||
FieldCheckResult(api_field, "missing", "ODS 中无对应列")
|
||||
)
|
||||
result.missing_fields += 1
|
||||
|
||||
result.passed = result.missing_fields == 0
|
||||
return result
|
||||
|
||||
|
||||
def check_ods_vs_dwd_mappings(
|
||||
dwd_table: str,
|
||||
ods_table: str,
|
||||
dwd_columns: set[str],
|
||||
ods_columns: set[str],
|
||||
fact_mappings: list[tuple[str, str, str | None]] | None,
|
||||
) -> TableCheckResult:
|
||||
"""
|
||||
对比 ODS 数据与 DWD 落库数据的映射正确性。
|
||||
|
||||
检查逻辑:
|
||||
1. DWD 表的每个非 SCD2 列,是否在 FACT_MAPPINGS 中有显式映射或在 ODS 中有同名列(自动映射)
|
||||
2. FACT_MAPPINGS 中引用的 ODS 列/表达式是否合法
|
||||
"""
|
||||
# CHANGE [2026-02-20] intent: 上游 API 不提供的字段,标记为已知无源而非报错
|
||||
KNOWN_NO_SOURCE: Dict[str, set[str]] = {
|
||||
"dwd.dim_member": {"update_time"},
|
||||
}
|
||||
|
||||
result = TableCheckResult(
|
||||
table_name=dwd_table,
|
||||
check_type="ods_vs_dwd",
|
||||
)
|
||||
|
||||
scd_cols = {c.lower() for c in DwdLoadTask.SCD_COLS}
|
||||
ods_lower = {c.lower() for c in ods_columns}
|
||||
|
||||
# 构建显式映射字典:dwd_col -> (ods_expr, cast_type)
|
||||
explicit_map: Dict[str, Tuple[str, str | None]] = {}
|
||||
if fact_mappings:
|
||||
for dwd_col, ods_expr, cast_type in fact_mappings:
|
||||
explicit_map[dwd_col.lower()] = (ods_expr, cast_type)
|
||||
|
||||
check_cols = sorted(c for c in dwd_columns if c.lower() not in scd_cols)
|
||||
result.total_fields = len(check_cols)
|
||||
|
||||
for col in check_cols:
|
||||
col_lower = col.lower()
|
||||
|
||||
if col_lower in explicit_map:
|
||||
ods_expr, cast_type = explicit_map[col_lower]
|
||||
# 验证 ODS 表达式引用的列是否存在
|
||||
expr_valid = _validate_ods_expression(ods_expr, ods_lower)
|
||||
if expr_valid:
|
||||
result.field_results.append(
|
||||
FieldCheckResult(col, "pass", f"显式映射: {ods_expr}")
|
||||
)
|
||||
result.passed_fields += 1
|
||||
else:
|
||||
result.field_results.append(
|
||||
FieldCheckResult(
|
||||
col, "mismatch",
|
||||
f"显式映射引用的 ODS 列/表达式无法验证: {ods_expr}"
|
||||
)
|
||||
)
|
||||
result.mismatch_fields += 1
|
||||
|
||||
elif col_lower in ods_lower:
|
||||
# 自动映射:ODS 和 DWD 同名
|
||||
result.field_results.append(
|
||||
FieldCheckResult(col, "pass", "自动映射(同名列)")
|
||||
)
|
||||
result.passed_fields += 1
|
||||
|
||||
elif col_lower == "fetched_at" and "fetched_at" in ods_lower:
|
||||
result.field_results.append(
|
||||
FieldCheckResult(col, "pass", "ETL 元数据列")
|
||||
)
|
||||
result.passed_fields += 1
|
||||
|
||||
else:
|
||||
# CHANGE [2026-02-20] intent: 白名单跳过已知无源字段,避免误报
|
||||
no_source = KNOWN_NO_SOURCE.get(dwd_table, set())
|
||||
if col_lower in no_source:
|
||||
result.field_results.append(
|
||||
FieldCheckResult(col, "pass", "已知无源字段(上游 API 不提供)")
|
||||
)
|
||||
result.passed_fields += 1
|
||||
else:
|
||||
result.field_results.append(
|
||||
FieldCheckResult(
|
||||
col, "missing",
|
||||
"DWD 列无 ODS 映射源(无显式映射且无同名 ODS 列)"
|
||||
)
|
||||
)
|
||||
result.missing_fields += 1
|
||||
|
||||
result.passed = result.missing_fields == 0 and result.mismatch_fields == 0
|
||||
return result
|
||||
|
||||
|
||||
def _validate_ods_expression(expr: str, ods_columns: set[str]) -> bool:
|
||||
"""
|
||||
验证 FACT_MAPPINGS 中的 ODS 表达式是否合法。
|
||||
|
||||
简单列名:检查是否在 ODS 列集合中
|
||||
JSON 表达式(含 ->>、#>>):检查基础列名
|
||||
SQL 表达式(含 CASE、COALESCE 等):视为合法(无法静态验证)
|
||||
NULL 字面量:合法
|
||||
"""
|
||||
if expr.upper() == "NULL":
|
||||
return True
|
||||
|
||||
# 带引号的列名(如 "siteGoodsId")
|
||||
stripped = expr.strip('"')
|
||||
|
||||
# JSON 路径表达式
|
||||
if "->>" in expr or "#>>" in expr:
|
||||
base_col = expr.split("->>")[0].split("#>>")[0].strip().strip('"').lower()
|
||||
return base_col in ods_columns
|
||||
|
||||
# SQL 表达式(CASE WHEN, COALESCE, 函数调用等)
|
||||
sql_keywords = {"case", "when", "coalesce", "nullif", "cast", "concat"}
|
||||
if any(kw in expr.lower() for kw in sql_keywords):
|
||||
return True
|
||||
|
||||
# 简单列名
|
||||
return stripped.lower() in ods_columns
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 数据库交互层
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _fetch_table_columns(cur, schema: str, table: str) -> set[str]:
|
||||
"""从 information_schema 获取表的列名集合(小写)"""
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = %s AND table_name = %s
|
||||
ORDER BY ordinal_position
|
||||
""",
|
||||
(schema, table),
|
||||
)
|
||||
return {row[0].lower() for row in cur.fetchall()}
|
||||
|
||||
|
||||
def _fetch_row_count(cur, schema: str, table: str) -> int:
|
||||
"""获取表的行数"""
|
||||
cur.execute(f'SELECT COUNT(1) FROM "{schema}"."{table}"')
|
||||
row = cur.fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
|
||||
def _split_table(name: str, default_schema: str) -> Tuple[str, str]:
|
||||
if "." in name:
|
||||
schema, table = name.split(".", 1)
|
||||
return schema, table
|
||||
return default_schema, name
|
||||
|
||||
|
||||
def _sample_value_mismatches(
|
||||
cur,
|
||||
dwd_schema: str,
|
||||
dwd_table: str,
|
||||
ods_schema: str,
|
||||
ods_table: str,
|
||||
dwd_pk_cols: list[str],
|
||||
ods_pk_cols: list[str],
|
||||
explicit_map: Dict[str, Tuple[str, str | None]],
|
||||
dwd_columns: set[str],
|
||||
ods_columns: set[str],
|
||||
limit: int = 5,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
采样对比 ODS 与 DWD 的实际数据值差异。
|
||||
|
||||
通过主键 JOIN 两表,对比映射列的值是否一致。
|
||||
返回不一致的采样记录。
|
||||
"""
|
||||
scd_cols = {c.lower() for c in DwdLoadTask.SCD_COLS}
|
||||
ods_meta = {"payload", "source_file", "source_endpoint",
|
||||
"fetched_at", "content_hash", "record_index"}
|
||||
ods_lower = {c.lower() for c in ods_columns}
|
||||
|
||||
# 确定 JOIN 键
|
||||
# DWD 主键中第一个在 ODS 中也存在的列(或通过映射对应的列)
|
||||
join_pairs: list[Tuple[str, str]] = [] # (dwd_col, ods_expr)
|
||||
for pk in dwd_pk_cols:
|
||||
pk_lower = pk.lower()
|
||||
if pk_lower in scd_cols:
|
||||
continue
|
||||
if pk_lower in explicit_map:
|
||||
ods_expr, cast_type = explicit_map[pk_lower]
|
||||
# 简单列名才能 JOIN
|
||||
stripped = ods_expr.strip('"')
|
||||
if stripped.lower() in ods_lower and stripped.isidentifier():
|
||||
join_pairs.append((pk_lower, ods_expr))
|
||||
elif pk_lower in ods_lower:
|
||||
join_pairs.append((pk_lower, f'"{pk_lower}"'))
|
||||
elif "id" in ods_lower and pk_lower.endswith("_id"):
|
||||
join_pairs.append((pk_lower, '"id"'))
|
||||
|
||||
if not join_pairs:
|
||||
return []
|
||||
|
||||
# 确定对比列
|
||||
compare_cols: list[Tuple[str, str]] = [] # (dwd_col, ods_expr)
|
||||
for col in sorted(dwd_columns):
|
||||
col_lower = col.lower()
|
||||
if col_lower in scd_cols or col_lower in ods_meta:
|
||||
continue
|
||||
if col_lower in {jp[0] for jp in join_pairs}:
|
||||
continue
|
||||
if col_lower in explicit_map:
|
||||
ods_expr, _ = explicit_map[col_lower]
|
||||
# 跳过复杂 SQL 表达式
|
||||
if any(kw in ods_expr.lower() for kw in ("case", "coalesce", "nullif")):
|
||||
continue
|
||||
compare_cols.append((col_lower, ods_expr))
|
||||
elif col_lower in ods_lower:
|
||||
compare_cols.append((col_lower, f'"{col_lower}"'))
|
||||
|
||||
if not compare_cols:
|
||||
return []
|
||||
|
||||
# 构建 SQL
|
||||
join_cond = " AND ".join(
|
||||
f'd."{dwd_col}" = o.{ods_expr}' if not ods_expr.startswith('"') or ods_expr.strip('"').isidentifier()
|
||||
else f'd."{dwd_col}" = o.{ods_expr}'
|
||||
for dwd_col, ods_expr in join_pairs
|
||||
)
|
||||
|
||||
# 构建 WHERE 条件:任一对比列不一致
|
||||
diff_conditions = []
|
||||
for dwd_col, ods_expr in compare_cols:
|
||||
# 使用 IS DISTINCT FROM 处理 NULL
|
||||
diff_conditions.append(
|
||||
f'd."{dwd_col}"::text IS DISTINCT FROM o.{ods_expr}::text'
|
||||
)
|
||||
|
||||
if not diff_conditions:
|
||||
return []
|
||||
|
||||
where_diff = " OR ".join(diff_conditions)
|
||||
|
||||
# ODS 快照去重(取最新 fetched_at)
|
||||
ods_has_content_hash = "content_hash" in ods_lower
|
||||
if ods_has_content_hash and ods_pk_cols:
|
||||
ods_biz_pks = [c for c in ods_pk_cols if c.lower() != "content_hash"]
|
||||
if ods_biz_pks:
|
||||
distinct_on = ", ".join(f'"{c}"' for c in ods_biz_pks)
|
||||
ods_subquery = (
|
||||
f'(SELECT DISTINCT ON ({distinct_on}) * '
|
||||
f'FROM "{ods_schema}"."{ods_table}" '
|
||||
f'ORDER BY {distinct_on}, "fetched_at" DESC NULLS LAST) o'
|
||||
)
|
||||
else:
|
||||
ods_subquery = f'"{ods_schema}"."{ods_table}" o'
|
||||
else:
|
||||
ods_subquery = f'"{ods_schema}"."{ods_table}" o'
|
||||
|
||||
# DWD SCD2 过滤
|
||||
dwd_where = ""
|
||||
if any(c.lower() == "scd2_is_current" for c in dwd_columns):
|
||||
dwd_where = "WHERE COALESCE(scd2_is_current, 1) = 1"
|
||||
|
||||
select_parts = []
|
||||
for dwd_col, _ in join_pairs:
|
||||
select_parts.append(f'd."{dwd_col}"')
|
||||
for dwd_col, ods_expr in compare_cols[:10]: # 限制对比列数
|
||||
select_parts.append(f'd."{dwd_col}" AS "dwd_{dwd_col}"')
|
||||
select_parts.append(f'o.{ods_expr}::text AS "ods_{dwd_col}"')
|
||||
|
||||
select_sql = ", ".join(select_parts)
|
||||
|
||||
sql = (
|
||||
f"SELECT {select_sql} "
|
||||
f'FROM (SELECT * FROM "{dwd_schema}"."{dwd_table}" {dwd_where}) d '
|
||||
f"JOIN {ods_subquery} ON {join_cond} "
|
||||
f"WHERE {where_diff} "
|
||||
f"LIMIT %s"
|
||||
)
|
||||
|
||||
try:
|
||||
cur.execute(sql, (limit,))
|
||||
rows = cur.fetchall()
|
||||
if not rows:
|
||||
return []
|
||||
columns = [desc[0] for desc in (cur.description or [])]
|
||||
return [dict(zip(columns, r)) for r in rows]
|
||||
except Exception:
|
||||
# 复杂表达式可能导致 SQL 错误,静默跳过
|
||||
return []
|
||||
|
||||
|
||||
def _fetch_pk_columns(cur, schema: str, table: str) -> list[str]:
|
||||
"""获取表的主键列"""
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT kcu.column_name
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.constraint_name = kcu.constraint_name
|
||||
AND tc.table_schema = kcu.table_schema
|
||||
WHERE tc.constraint_type = 'PRIMARY KEY'
|
||||
AND tc.table_schema = %s
|
||||
AND tc.table_name = %s
|
||||
ORDER BY kcu.ordinal_position
|
||||
""",
|
||||
(schema, table),
|
||||
)
|
||||
return [r[0] for r in cur.fetchall()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 主入口:运行完整一致性检查
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def run_consistency_check(
|
||||
db_conn,
|
||||
*,
|
||||
api_sample_dir: Path | None = None,
|
||||
include_api_vs_ods: bool = True,
|
||||
include_ods_vs_dwd: bool = True,
|
||||
sample_limit: int = 5,
|
||||
tz: ZoneInfo | None = None,
|
||||
) -> ConsistencyReport:
|
||||
"""
|
||||
执行完整的数据一致性检查。
|
||||
|
||||
参数:
|
||||
db_conn: 数据库连接对象(需有 .conn 属性返回 psycopg2 connection)
|
||||
api_sample_dir: API JSON 缓存目录(用于 API vs ODS 检查)
|
||||
include_api_vs_ods: 是否执行 API vs ODS 检查
|
||||
include_ods_vs_dwd: 是否执行 ODS vs DWD 检查
|
||||
sample_limit: 值不一致时的采样行数
|
||||
tz: 时区
|
||||
"""
|
||||
if tz is None:
|
||||
tz = ZoneInfo("Asia/Shanghai")
|
||||
|
||||
report = ConsistencyReport(
|
||||
generated_at=datetime.now(tz).isoformat(),
|
||||
)
|
||||
|
||||
with db_conn.conn.cursor() as cur:
|
||||
# --- 1. API vs ODS 字段完整性检查 ---
|
||||
if include_api_vs_ods and api_sample_dir:
|
||||
for ods_table, json_file in sorted(ODS_TABLE_TO_JSON_FILE.items()):
|
||||
json_path = api_sample_dir / json_file
|
||||
api_fields = extract_api_fields_from_json(json_path)
|
||||
if api_fields is None:
|
||||
result = TableCheckResult(
|
||||
table_name=f"ods.{ods_table}",
|
||||
check_type="api_vs_ods",
|
||||
passed=True, # 无 JSON 缓存时跳过,不算失败
|
||||
error=f"API JSON 缓存不存在: {json_file}",
|
||||
)
|
||||
report.api_vs_ods_results.append(result)
|
||||
continue
|
||||
|
||||
ods_columns = _fetch_table_columns(cur, "ods", ods_table)
|
||||
if not ods_columns:
|
||||
result = TableCheckResult(
|
||||
table_name=f"ods.{ods_table}",
|
||||
check_type="api_vs_ods",
|
||||
passed=False,
|
||||
error="ODS 表不存在或无列",
|
||||
)
|
||||
report.api_vs_ods_results.append(result)
|
||||
continue
|
||||
|
||||
result = check_api_vs_ods_fields(api_fields, ods_columns)
|
||||
result.table_name = f"ods.{ods_table}"
|
||||
report.api_vs_ods_results.append(result)
|
||||
|
||||
# --- 2. ODS vs DWD 映射正确性检查 ---
|
||||
if include_ods_vs_dwd:
|
||||
table_map = DwdLoadTask.TABLE_MAP
|
||||
fact_mappings = DwdLoadTask.FACT_MAPPINGS
|
||||
|
||||
for dwd_full, ods_full in sorted(table_map.items()):
|
||||
dwd_schema, dwd_table = _split_table(dwd_full, "dwd")
|
||||
ods_schema, ods_table = _split_table(ods_full, "ods")
|
||||
|
||||
try:
|
||||
dwd_columns = _fetch_table_columns(cur, dwd_schema, dwd_table)
|
||||
ods_columns = _fetch_table_columns(cur, ods_schema, ods_table)
|
||||
|
||||
if not dwd_columns:
|
||||
result = TableCheckResult(
|
||||
table_name=dwd_full,
|
||||
check_type="ods_vs_dwd",
|
||||
passed=False,
|
||||
error=f"DWD 表 {dwd_full} 不存在或无列",
|
||||
)
|
||||
report.ods_vs_dwd_results.append(result)
|
||||
continue
|
||||
|
||||
mappings = fact_mappings.get(dwd_full)
|
||||
result = check_ods_vs_dwd_mappings(
|
||||
dwd_full, ods_full,
|
||||
dwd_columns, ods_columns,
|
||||
mappings,
|
||||
)
|
||||
|
||||
# 补充行数统计
|
||||
result.row_count_source = _fetch_row_count(cur, ods_schema, ods_table)
|
||||
result.row_count_target = _fetch_row_count(cur, dwd_schema, dwd_table)
|
||||
|
||||
# 采样值对比
|
||||
if sample_limit > 0:
|
||||
explicit_map: Dict[str, Tuple[str, str | None]] = {}
|
||||
if mappings:
|
||||
for dwd_col, ods_expr, cast_type in mappings:
|
||||
explicit_map[dwd_col.lower()] = (ods_expr, cast_type)
|
||||
|
||||
dwd_pk = _fetch_pk_columns(cur, dwd_schema, dwd_table)
|
||||
ods_pk = _fetch_pk_columns(cur, ods_schema, ods_table)
|
||||
|
||||
samples = _sample_value_mismatches(
|
||||
cur,
|
||||
dwd_schema, dwd_table,
|
||||
ods_schema, ods_table,
|
||||
dwd_pk, ods_pk,
|
||||
explicit_map,
|
||||
dwd_columns, ods_columns,
|
||||
limit=sample_limit,
|
||||
)
|
||||
result.sample_mismatches = samples
|
||||
|
||||
report.ods_vs_dwd_results.append(result)
|
||||
|
||||
except Exception as exc:
|
||||
result = TableCheckResult(
|
||||
table_name=dwd_full,
|
||||
check_type="ods_vs_dwd",
|
||||
passed=False,
|
||||
error=f"{type(exc).__name__}: {exc}",
|
||||
)
|
||||
report.ods_vs_dwd_results.append(result)
|
||||
|
||||
return report
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 报告生成
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def generate_markdown_report(report: ConsistencyReport) -> str:
|
||||
"""将 ConsistencyReport 转换为 Markdown 格式的黑盒测试报告"""
|
||||
lines: list[str] = []
|
||||
lines.append("# 数据一致性黑盒测试报告")
|
||||
lines.append("")
|
||||
lines.append(f"生成时间: {report.generated_at}")
|
||||
lines.append(f"总体结果: **{'✅ 全部通过' if report.all_passed else '❌ 存在异常'}**")
|
||||
lines.append("")
|
||||
|
||||
# --- 汇总表格 ---
|
||||
lines.append("## 汇总")
|
||||
lines.append("")
|
||||
|
||||
if report.api_vs_ods_results:
|
||||
api_pass = sum(1 for r in report.api_vs_ods_results if r.passed)
|
||||
api_total = len(report.api_vs_ods_results)
|
||||
lines.append(f"- API vs ODS 字段完整性: {api_pass}/{api_total} 张表通过")
|
||||
|
||||
if report.ods_vs_dwd_results:
|
||||
dwd_pass = sum(1 for r in report.ods_vs_dwd_results if r.passed)
|
||||
dwd_total = len(report.ods_vs_dwd_results)
|
||||
lines.append(f"- ODS vs DWD 映射正确性: {dwd_pass}/{dwd_total} 张表通过")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# --- API vs ODS 详细结果 ---
|
||||
if report.api_vs_ods_results:
|
||||
lines.append("## API vs ODS 字段完整性检查")
|
||||
lines.append("")
|
||||
lines.append("| 表名 | 状态 | 总字段 | 通过 | 缺失 | 备注 |")
|
||||
lines.append("|------|------|--------|------|------|------|")
|
||||
|
||||
for r in report.api_vs_ods_results:
|
||||
status = "✅" if r.passed else "❌"
|
||||
note = r.error or ""
|
||||
lines.append(
|
||||
f"| {r.table_name} | {status} | {r.total_fields} "
|
||||
f"| {r.passed_fields} | {r.missing_fields} | {note} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
|
||||
# 失败表的详细字段列表
|
||||
failed = [r for r in report.api_vs_ods_results if not r.passed and r.field_results]
|
||||
for r in failed:
|
||||
lines.append(f"### {r.table_name} — 缺失字段明细")
|
||||
lines.append("")
|
||||
missing = [f for f in r.field_results if f.status == "missing"]
|
||||
for f in missing:
|
||||
lines.append(f"- `{f.field_name}`: {f.detail}")
|
||||
lines.append("")
|
||||
|
||||
# --- ODS vs DWD 详细结果 ---
|
||||
if report.ods_vs_dwd_results:
|
||||
lines.append("## ODS vs DWD 映射正确性检查")
|
||||
lines.append("")
|
||||
lines.append("| DWD 表 | 状态 | 总字段 | 通过 | 缺失 | 不一致 | ODS 行数 | DWD 行数 | 备注 |")
|
||||
lines.append("|--------|------|--------|------|------|--------|----------|----------|------|")
|
||||
|
||||
for r in report.ods_vs_dwd_results:
|
||||
status = "✅" if r.passed else "❌"
|
||||
note = r.error or ""
|
||||
lines.append(
|
||||
f"| {r.table_name} | {status} | {r.total_fields} "
|
||||
f"| {r.passed_fields} | {r.missing_fields} | {r.mismatch_fields} "
|
||||
f"| {r.row_count_source} | {r.row_count_target} | {note} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
|
||||
# 失败表的详细字段列表
|
||||
failed_dwd = [r for r in report.ods_vs_dwd_results if not r.passed and r.field_results]
|
||||
for r in failed_dwd:
|
||||
lines.append(f"### {r.table_name} — 映射异常明细")
|
||||
lines.append("")
|
||||
issues = [f for f in r.field_results if f.status in ("missing", "mismatch")]
|
||||
for f in issues:
|
||||
lines.append(f"- `{f.field_name}` [{f.status}]: {f.detail}")
|
||||
lines.append("")
|
||||
|
||||
if r.sample_mismatches:
|
||||
lines.append(f"#### 值不一致采样(前 {len(r.sample_mismatches)} 条)")
|
||||
lines.append("")
|
||||
lines.append("```json")
|
||||
# 序列化时处理不可序列化的类型
|
||||
safe_samples = _safe_serialize(r.sample_mismatches)
|
||||
lines.append(json.dumps(safe_samples, ensure_ascii=False, indent=2))
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _safe_serialize(obj: Any) -> Any:
|
||||
"""将不可 JSON 序列化的类型转为字符串"""
|
||||
if isinstance(obj, list):
|
||||
return [_safe_serialize(item) for item in obj]
|
||||
if isinstance(obj, dict):
|
||||
return {k: _safe_serialize(v) for k, v in obj.items()}
|
||||
if isinstance(obj, (datetime,)):
|
||||
return obj.isoformat()
|
||||
if isinstance(obj, bytes):
|
||||
return obj.hex()
|
||||
try:
|
||||
json.dumps(obj)
|
||||
return obj
|
||||
except (TypeError, ValueError):
|
||||
return str(obj)
|
||||
|
||||
|
||||
def write_consistency_report(
|
||||
report: ConsistencyReport,
|
||||
*,
|
||||
report_path: Path | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
将一致性检查报告写入文件。
|
||||
|
||||
输出路径通过 ETL_REPORT_ROOT 环境变量控制。
|
||||
"""
|
||||
if report_path is None:
|
||||
env_root = os.environ.get("ETL_REPORT_ROOT")
|
||||
if not env_root:
|
||||
raise KeyError(
|
||||
"环境变量 ETL_REPORT_ROOT 未定义。"
|
||||
"请在根 .env 中配置,参考 docs/deployment/EXPORT-PATHS.md"
|
||||
)
|
||||
root = Path(env_root)
|
||||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
report_path = root / f"consistency_report_{stamp}.md"
|
||||
|
||||
md_content = generate_markdown_report(report)
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(md_content, encoding="utf-8")
|
||||
return str(report_path)
|
||||
@@ -592,9 +592,15 @@ def run_dwd_vs_ods_check(
|
||||
|
||||
|
||||
def _default_report_path(prefix: str) -> Path:
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
env_root = os.environ.get("ETL_REPORT_ROOT")
|
||||
if not env_root:
|
||||
raise KeyError(
|
||||
"环境变量 ETL_REPORT_ROOT 未定义。"
|
||||
"请在根 .env 中配置,参考 docs/deployment/EXPORT-PATHS.md"
|
||||
)
|
||||
root = Path(env_root)
|
||||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
return root / "reports" / f"{prefix}_{stamp}.json"
|
||||
return root / f"{prefix}_{stamp}.json"
|
||||
|
||||
|
||||
def run_integrity_window(
|
||||
|
||||
@@ -9,6 +9,7 @@ from typing import Any, Iterable, Tuple
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from quality.integrity_checker import IntegrityWindow, compute_last_etl_end, run_integrity_history, run_integrity_window
|
||||
from scripts.repair.backfill_missing_data import run_backfill
|
||||
@@ -249,9 +250,16 @@ def run_history_flow(
|
||||
|
||||
def write_report(report: dict, *, prefix: str, tz: ZoneInfo, report_path: Path | None = None) -> str:
|
||||
if report_path is None:
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
# 从 .env 读取 ETL_REPORT_ROOT(必须配置)
|
||||
env_root = os.environ.get("ETL_REPORT_ROOT")
|
||||
if not env_root:
|
||||
raise KeyError(
|
||||
"环境变量 ETL_REPORT_ROOT 未定义。"
|
||||
"请在根 .env 中配置,参考 .env.template 和 docs/deployment/EXPORT-PATHS.md"
|
||||
)
|
||||
root_dir = Path(env_root)
|
||||
stamp = datetime.now(tz).strftime("%Y%m%d_%H%M%S")
|
||||
report_path = root / "reports" / f"{prefix}_{stamp}.json"
|
||||
report_path = root_dir / f"{prefix}_{stamp}.json"
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return str(report_path)
|
||||
|
||||
@@ -1,783 +0,0 @@
|
||||
{
|
||||
"generated_at": "2026-02-18T13:54:42.674118",
|
||||
"tables": [
|
||||
{
|
||||
"dwd_table": "dwd.dim_site",
|
||||
"ods_table": "ods.table_fee_transactions",
|
||||
"count": {
|
||||
"dwd": 54,
|
||||
"ods": 38585,
|
||||
"diff": -38531
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_site_ex",
|
||||
"ods_table": "ods.table_fee_transactions",
|
||||
"count": {
|
||||
"dwd": 54,
|
||||
"ods": 38585,
|
||||
"diff": -38531
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_table",
|
||||
"ods_table": "ods.site_tables_master",
|
||||
"count": {
|
||||
"dwd": 2881,
|
||||
"ods": 1074,
|
||||
"diff": 1807
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_table_ex",
|
||||
"ods_table": "ods.site_tables_master",
|
||||
"count": {
|
||||
"dwd": 3035,
|
||||
"ods": 1074,
|
||||
"diff": 1961
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_assistant",
|
||||
"ods_table": "ods.assistant_accounts_master",
|
||||
"count": {
|
||||
"dwd": 1266,
|
||||
"ods": 295,
|
||||
"diff": 971
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_assistant_ex",
|
||||
"ods_table": "ods.assistant_accounts_master",
|
||||
"count": {
|
||||
"dwd": 1251,
|
||||
"ods": 295,
|
||||
"diff": 956
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_member",
|
||||
"ods_table": "ods.member_profiles",
|
||||
"count": {
|
||||
"dwd": 10801,
|
||||
"ods": 1771,
|
||||
"diff": 9030
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "pay_money_sum",
|
||||
"dwd_sum": -44578911.7,
|
||||
"ods_sum": -12031004.21,
|
||||
"diff": -32547907.490000002
|
||||
},
|
||||
{
|
||||
"column": "recharge_money_sum",
|
||||
"dwd_sum": 51067011.97,
|
||||
"ods_sum": 13293169.41,
|
||||
"diff": 37773842.56
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_member_ex",
|
||||
"ods_table": "ods.member_profiles",
|
||||
"count": {
|
||||
"dwd": 10770,
|
||||
"ods": 1771,
|
||||
"diff": 8999
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_member_card_account",
|
||||
"ods_table": "ods.member_stored_value_cards",
|
||||
"count": {
|
||||
"dwd": 18370,
|
||||
"ods": 2973,
|
||||
"diff": 15397
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "balance",
|
||||
"dwd_sum": 7834178.2,
|
||||
"ods_sum": 1369125.58,
|
||||
"diff": 6465052.62
|
||||
},
|
||||
{
|
||||
"column": "principal_balance",
|
||||
"dwd_sum": 4659292.04,
|
||||
"ods_sum": 921317.68,
|
||||
"diff": 3737974.36
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_member_card_account_ex",
|
||||
"ods_table": "ods.member_stored_value_cards",
|
||||
"count": {
|
||||
"dwd": 18172,
|
||||
"ods": 2973,
|
||||
"diff": 15199
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "deliveryfeededuct",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_tenant_goods",
|
||||
"ods_table": "ods.tenant_goods_master",
|
||||
"count": {
|
||||
"dwd": 2018,
|
||||
"ods": 353,
|
||||
"diff": 1665
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_tenant_goods_ex",
|
||||
"ods_table": "ods.tenant_goods_master",
|
||||
"count": {
|
||||
"dwd": 2016,
|
||||
"ods": 353,
|
||||
"diff": 1663
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_store_goods",
|
||||
"ods_table": "ods.store_goods_master",
|
||||
"count": {
|
||||
"dwd": 5444,
|
||||
"ods": 1744,
|
||||
"diff": 3700
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_store_goods_ex",
|
||||
"ods_table": "ods.store_goods_master",
|
||||
"count": {
|
||||
"dwd": 5402,
|
||||
"ods": 1744,
|
||||
"diff": 3658
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_goods_category",
|
||||
"ods_table": "ods.stock_goods_category_tree",
|
||||
"count": {
|
||||
"dwd": 125,
|
||||
"ods": 18,
|
||||
"diff": 107
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_groupbuy_package",
|
||||
"ods_table": "ods.group_buy_packages",
|
||||
"count": {
|
||||
"dwd": 373,
|
||||
"ods": 70,
|
||||
"diff": 303
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dim_groupbuy_package_ex",
|
||||
"ods_table": "ods.group_buy_packages",
|
||||
"count": {
|
||||
"dwd": 373,
|
||||
"ods": 70,
|
||||
"diff": 303
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_settlement_head",
|
||||
"ods_table": "ods.settlement_records",
|
||||
"count": {
|
||||
"dwd": 26115,
|
||||
"ods": 55457,
|
||||
"diff": -29342
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_settlement_head_ex",
|
||||
"ods_table": "ods.settlement_records",
|
||||
"count": {
|
||||
"dwd": 26115,
|
||||
"ods": 55457,
|
||||
"diff": -29342
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_table_fee_log",
|
||||
"ods_table": "ods.table_fee_transactions",
|
||||
"count": {
|
||||
"dwd": 20132,
|
||||
"ods": 38585,
|
||||
"diff": -18453
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "activity_discount_amount",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "adjust_amount",
|
||||
"dwd_sum": 360935.76,
|
||||
"ods_sum": 748219.39,
|
||||
"diff": -387283.63
|
||||
},
|
||||
{
|
||||
"column": "coupon_promotion_amount",
|
||||
"dwd_sum": 1043924.79,
|
||||
"ods_sum": 1960731.94,
|
||||
"diff": -916807.1499999999
|
||||
},
|
||||
{
|
||||
"column": "ledger_amount",
|
||||
"dwd_sum": 2329813.19,
|
||||
"ods_sum": 4552084.15,
|
||||
"diff": -2222270.9600000004
|
||||
},
|
||||
{
|
||||
"column": "member_discount_amount",
|
||||
"dwd_sum": 200164.42,
|
||||
"ods_sum": 363032.98,
|
||||
"diff": -162868.55999999997
|
||||
},
|
||||
{
|
||||
"column": "real_service_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "real_table_charge_money",
|
||||
"dwd_sum": 831450.6,
|
||||
"ods_sum": 1676836.7,
|
||||
"diff": -845386.1
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_table_fee_log_ex",
|
||||
"ods_table": "ods.table_fee_transactions",
|
||||
"count": {
|
||||
"dwd": 20132,
|
||||
"ods": 38585,
|
||||
"diff": -18453
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "fee_total",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "mgmt_fee",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "service_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "used_card_amount",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_table_fee_adjust",
|
||||
"ods_table": "ods.table_fee_discount_records",
|
||||
"count": {
|
||||
"dwd": 3124,
|
||||
"ods": 4818,
|
||||
"diff": -1694
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "ledger_amount",
|
||||
"dwd_sum": 370340.42,
|
||||
"ods_sum": 580703.61,
|
||||
"diff": -210363.19
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_table_fee_adjust_ex",
|
||||
"ods_table": "ods.table_fee_discount_records",
|
||||
"count": {
|
||||
"dwd": 3124,
|
||||
"ods": 4818,
|
||||
"diff": -1694
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_store_goods_sale",
|
||||
"ods_table": "ods.store_goods_sales_records",
|
||||
"count": {
|
||||
"dwd": 17563,
|
||||
"ods": 17563,
|
||||
"diff": 0
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "cost_money",
|
||||
"dwd_sum": 3116.75,
|
||||
"ods_sum": 3116.75,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "coupon_share_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "ledger_amount",
|
||||
"dwd_sum": 373588.1,
|
||||
"ods_sum": 373588.1,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "real_goods_money",
|
||||
"dwd_sum": 351914.9,
|
||||
"ods_sum": 351914.9,
|
||||
"diff": 0.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_store_goods_sale_ex",
|
||||
"ods_table": "ods.store_goods_sales_records",
|
||||
"count": {
|
||||
"dwd": 17563,
|
||||
"ods": 17563,
|
||||
"diff": 0
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "coupon_deduct_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "discount_money",
|
||||
"dwd_sum": 21673.2,
|
||||
"ods_sum": 21673.2,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "member_discount_amount",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "option_coupon_deduct_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "option_member_discount_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "point_discount_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "point_discount_money_cost",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "push_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_assistant_service_log",
|
||||
"ods_table": "ods.assistant_service_records",
|
||||
"count": {
|
||||
"dwd": 5281,
|
||||
"ods": 10219,
|
||||
"diff": -4938
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "coupon_deduct_money",
|
||||
"dwd_sum": 11071.07,
|
||||
"ods_sum": 21950.93,
|
||||
"diff": -10879.86
|
||||
},
|
||||
{
|
||||
"column": "ledger_amount",
|
||||
"dwd_sum": 1653011.03,
|
||||
"ods_sum": 3196243.99,
|
||||
"diff": -1543232.9600000002
|
||||
},
|
||||
{
|
||||
"column": "real_service_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_assistant_service_log_ex",
|
||||
"ods_table": "ods.assistant_service_records",
|
||||
"count": {
|
||||
"dwd": 5281,
|
||||
"ods": 10219,
|
||||
"diff": -4938
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "manual_discount_amount",
|
||||
"dwd_sum": 414.17,
|
||||
"ods_sum": 828.34,
|
||||
"diff": -414.17
|
||||
},
|
||||
{
|
||||
"column": "member_discount_amount",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "service_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_assistant_trash_event",
|
||||
"ods_table": "ods.assistant_cancellation_records",
|
||||
"count": {
|
||||
"dwd": 100,
|
||||
"ods": 100,
|
||||
"diff": 0
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_assistant_trash_event_ex",
|
||||
"ods_table": "ods.assistant_cancellation_records",
|
||||
"count": {
|
||||
"dwd": 100,
|
||||
"ods": 100,
|
||||
"diff": 0
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_member_balance_change",
|
||||
"ods_table": "ods.member_balance_changes",
|
||||
"count": {
|
||||
"dwd": 5081,
|
||||
"ods": 9684,
|
||||
"diff": -4603
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_member_balance_change_ex",
|
||||
"ods_table": "ods.member_balance_changes",
|
||||
"count": {
|
||||
"dwd": 5081,
|
||||
"ods": 9684,
|
||||
"diff": -4603
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "refund_amount",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_groupbuy_redemption",
|
||||
"ods_table": "ods.group_buy_redemption_records",
|
||||
"count": {
|
||||
"dwd": 12768,
|
||||
"ods": 27284,
|
||||
"diff": -14516
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "coupon_money",
|
||||
"dwd_sum": 822390.0,
|
||||
"ods_sum": 1748666.0,
|
||||
"diff": -926276.0
|
||||
},
|
||||
{
|
||||
"column": "ledger_amount",
|
||||
"dwd_sum": 788754.37,
|
||||
"ods_sum": 1675417.16,
|
||||
"diff": -886662.7899999999
|
||||
},
|
||||
{
|
||||
"column": "member_discount_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_groupbuy_redemption_ex",
|
||||
"ods_table": "ods.group_buy_redemption_records",
|
||||
"count": {
|
||||
"dwd": 12768,
|
||||
"ods": 27284,
|
||||
"diff": -14516
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "assistant_promotion_money",
|
||||
"dwd_sum": 7544.8,
|
||||
"ods_sum": 16557.98,
|
||||
"diff": -9013.18
|
||||
},
|
||||
{
|
||||
"column": "assistant_service_promotion_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "assistant_service_share_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "assistant_share_money",
|
||||
"dwd_sum": 2285.57,
|
||||
"ods_sum": 6697.18,
|
||||
"diff": -4411.610000000001
|
||||
},
|
||||
{
|
||||
"column": "good_service_share_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "goods_promotion_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "goods_share_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "recharge_promotion_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "recharge_share_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "reward_promotion_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "table_service_promotion_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "table_service_share_money",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "table_share_money",
|
||||
"dwd_sum": 382004.98,
|
||||
"ods_sum": 846145.6,
|
||||
"diff": -464140.62
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_platform_coupon_redemption",
|
||||
"ods_table": "ods.platform_coupon_redemption_records",
|
||||
"count": {
|
||||
"dwd": 18311,
|
||||
"ods": 36678,
|
||||
"diff": -18367
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "coupon_money",
|
||||
"dwd_sum": 1193916.0,
|
||||
"ods_sum": 2391858.0,
|
||||
"diff": -1197942.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_platform_coupon_redemption_ex",
|
||||
"ods_table": "ods.platform_coupon_redemption_records",
|
||||
"count": {
|
||||
"dwd": 18311,
|
||||
"ods": 36678,
|
||||
"diff": -18367
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_recharge_order",
|
||||
"ods_table": "ods.recharge_settlements",
|
||||
"count": {
|
||||
"dwd": 485,
|
||||
"ods": 3333,
|
||||
"diff": -2848
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_recharge_order_ex",
|
||||
"ods_table": "ods.recharge_settlements",
|
||||
"count": {
|
||||
"dwd": 485,
|
||||
"ods": 3333,
|
||||
"diff": -2848
|
||||
},
|
||||
"amounts": []
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_payment",
|
||||
"ods_table": "ods.payment_transactions",
|
||||
"count": {
|
||||
"dwd": 24922,
|
||||
"ods": 36006,
|
||||
"diff": -11084
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "pay_amount",
|
||||
"dwd_sum": 2424522.0,
|
||||
"ods_sum": 3652765.0,
|
||||
"diff": -1228243.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_refund",
|
||||
"ods_table": "ods.refund_transactions",
|
||||
"count": {
|
||||
"dwd": 51,
|
||||
"ods": 83,
|
||||
"diff": -32
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "channel_fee",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "pay_amount",
|
||||
"dwd_sum": -79176.5,
|
||||
"ods_sum": -94892.0,
|
||||
"diff": 15715.5
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dwd_table": "dwd.dwd_refund_ex",
|
||||
"ods_table": "ods.refund_transactions",
|
||||
"count": {
|
||||
"dwd": 51,
|
||||
"ods": 83,
|
||||
"diff": -32
|
||||
},
|
||||
"amounts": [
|
||||
{
|
||||
"column": "balance_frozen_amount",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "card_frozen_amount",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "refund_amount",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
},
|
||||
{
|
||||
"column": "round_amount",
|
||||
"dwd_sum": 0.0,
|
||||
"ods_sum": 0.0,
|
||||
"diff": 0.0
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"note": "行数/金额核对,金额字段基于列名包含 amount/money/fee/balance 的数值列自动扫描。"
|
||||
}
|
||||
@@ -105,7 +105,13 @@ def _iter_rows(
|
||||
def _build_report_path(out_arg: str | None) -> Path:
|
||||
if out_arg:
|
||||
return Path(out_arg)
|
||||
reports_dir = PROJECT_ROOT / "reports"
|
||||
env_root = os.environ.get("ETL_REPORT_ROOT")
|
||||
if not env_root:
|
||||
raise KeyError(
|
||||
"环境变量 ETL_REPORT_ROOT 未定义。"
|
||||
"请在根 .env 中配置,参考 docs/deployment/EXPORT-PATHS.md"
|
||||
)
|
||||
reports_dir = Path(env_root)
|
||||
reports_dir.mkdir(parents=True, exist_ok=True)
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
return reports_dir / f"ods_content_hash_check_{ts}.json"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user