root/Neo-ZQYY
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

微信小程序页面迁移校验之前 P5任务处理之前

Browse Source
This commit is contained in:
Neo
2026-03-09 01:19:21 +08:00
parent 263bf96035
commit 6e20987d2f
1112 changed files with 153824 additions and 219694 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
apps/admin-web/README.md
View File

@@ -29,6 +29,7 @@ apps/admin-web/
│ │ ├── EnvConfig.tsx # 环境配置管理
│ │ └── OpsPanel.tsx # 运维面板（服务启停 + Git + 系统监控）
│ ├── components/ # 可复用组件
│ │ ├── BusinessDayHint.tsx # 营业日提示组件
│ │ ├── DwdTableSelector.tsx # DWD 表选择器
│ │ ├── ErrorBoundary.tsx # 错误边界
│ │ ├── LogStream.tsx # 实时日志流组件
@@ -36,6 +37,7 @@ apps/admin-web/
│ │ └── TaskSelector.tsx # 任务选择器
│ ├── api/ # API 调用层
│ │ ├── client.ts # Axios 实例（baseURL + JWT 拦截器）
│ │ ├── businessDay.ts # 营业日配置 API
│ │ ├── tasks.ts # 任务配置 API
│ │ ├── execution.ts # 任务执行 API
│ │ ├── schedules.ts # 调度管理 API
@@ -44,7 +46,8 @@ apps/admin-web/
│ │ ├── envConfig.ts # 环境配置 API
│ │ └── opsPanel.ts # 运维面板 API
│ ├── store/
│ │ └── authStore.ts # Zustand 认证状态（JWT 持久化 + hydrate）
│ │ ├── authStore.ts # Zustand 认证状态（JWT 持久化 + hydrate）
│ │ └── businessDayStore.ts # 营业日状态管理
│ └── types/ # TypeScript 类型定义
├── index.html # HTML 入口
├── vite.config.ts # Vite 配置
@@ -72,13 +75,13 @@ ETL 任务的核心配置界面：
- 选择执行流程（7 种 Flow）
- 勾选要执行的任务（按业务域分组）
- 设置处理模式（增量/校验/全窗口）
- 配置时间窗口参数
- 配置时间窗口参数（含营业日提示）
- 实时预览生成的 CLI 命令
- 一键执行或加入队列
### 任务管理 (`/task-manager`)
- 查看执行队列（拖拽排序、删除、取消）
- 执行历史列表（状态、耗时、退出码）
- 执行历史列表（状态、耗时、退出码、终止操作）
- 实时日志流（WebSocket 推送）
### ETL 状态 (`/etl-status`)
@@ -121,11 +124,17 @@ ETL 任务的核心配置界面：
## 状态管理
使用 Zustand 管理全局认证状态：
使用 Zustand 管理全局状态：
**认证状态** (`authStore`)：
- `isAuthenticated`：是否已登录
- `token` / `refreshToken`：JWT 令牌
- `login()` / `logout()` / `hydrate()`：状态操作
**营业日状态** (`businessDayStore`)：
- `businessDayStartHour`：营业日分割点
- `init()` / `refresh()`：配置获取与刷新
## 与后端的关系
管理后台通过 REST API 与 `apps/backend/` 通信：

6
apps/admin-web/src/App.tsx
View File

@@ -21,6 +21,7 @@ import {
} from "@ant-design/icons";
import type { MenuProps } from "antd";
import { useAuthStore } from "./store/authStore";
import { useBusinessDayStore } from "./store/businessDayStore";
import { fetchQueue } from "./api/execution";
import type { QueuedTask } from "./types";
import Login from "./pages/Login";
@@ -179,12 +180,15 @@ const AppLayout: React.FC = () => {
const App: React.FC = () => {
const hydrate = useAuthStore((s) => s.hydrate);
const initBusinessDay = useBusinessDayStore((s) => s.init);
const [hydrated, setHydrated] = useState(false);
useEffect(() => {
hydrate();
setHydrated(true);
}, [hydrate]);
// 启动时请求一次营业日配置，降级策略在 store 内部处理
initBusinessDay();
}, [hydrate, initBusinessDay]);
/* hydrate 完成前不渲染路由，避免 PrivateRoute 误判跳转到 /login */
if (!hydrated) return <Spin style={{ display: "flex", justifyContent: "center", marginTop: 120 }} />;

170
apps/admin-web/src/__tests__/taskLogParser.test.ts Normal file
View File

@@ -0,0 +1,170 @@
/**
* 任务日志解析与分组测试
*
* **Validates: Requirements 10.2, 10.5, 10.6**
*
* 验证 parseLogLine / groupLogsByTask / filterTaskGroups 的正确性。
*/
import { describe, it, expect } from "vitest";
import {
parseLogLine,
groupLogsByTask,
filterTaskGroups,
} from "../utils/taskLogParser";
/* ------------------------------------------------------------------ */
/* parseLogLine */
/* ------------------------------------------------------------------ */
describe("parseLogLine — 单行日志解析", () => {
it("解析标准格式日志行", () => {
const line = "[ODS_MEMBER] 2024-06-01 12:00:00 INFO 开始拉取会员数据";
const result = parseLogLine(line);
expect(result).not.toBeNull();
expect(result!.taskCode).toBe("ODS_MEMBER");
expect(result!.timestamp).toBe("2024-06-01 12:00:00");
expect(result!.level).toBe("INFO");
expect(result!.message).toBe("开始拉取会员数据");
});
it("解析带毫秒的时间戳", () => {
const line = "[ODS_ORDER] 2024-06-01 12:00:00,123 ERROR 请求失败";
const result = parseLogLine(line);
expect(result).not.toBeNull();
expect(result!.taskCode).toBe("ODS_ORDER");
expect(result!.timestamp).toBe("2024-06-01 12:00:00,123");
expect(result!.level).toBe("ERROR");
});
it("解析 WARNING 级别", () => {
const line = "[DWD_LOAD_FROM_ODS] 2024-06-01 12:00:00 WARNING 队列积压";
const result = parseLogLine(line);
expect(result).not.toBeNull();
expect(result!.level).toBe("WARNING");
});
it("解析 DEBUG 级别", () => {
const line = "[ODS_MEMBER] 2024-06-01 12:00:00 DEBUG 调试信息";
const result = parseLogLine(line);
expect(result).not.toBeNull();
expect(result!.level).toBe("DEBUG");
});
it("无法解析的行返回 null", () => {
expect(parseLogLine("这是一行普通文本")).toBeNull();
expect(parseLogLine("")).toBeNull();
expect(parseLogLine("[lowercase] 2024-06-01 12:00:00 INFO msg")).toBeNull();
});
});
/* ------------------------------------------------------------------ */
/* groupLogsByTask */
/* ------------------------------------------------------------------ */
describe("groupLogsByTask — 按任务分组", () => {
const sampleLines = [
"[ODS_MEMBER] 2024-06-01 12:00:00 INFO 开始拉取",
"[ODS_MEMBER] 2024-06-01 12:00:05 INFO 拉取完成",
"[ODS_ORDER] 2024-06-01 12:00:10 INFO 开始拉取订单",
"[ODS_ORDER] 2024-06-01 12:00:20 ERROR 请求超时",
"[ODS_MEMBER] 2024-06-01 12:00:25 INFO 写入完成 success",
];
it("正确分组到不同任务", () => {
const groups = groupLogsByTask(sampleLines);
expect(groups).toHaveLength(2);
const memberGroup = groups.find((g) => g.taskCode === "ODS_MEMBER");
const orderGroup = groups.find((g) => g.taskCode === "ODS_ORDER");
expect(memberGroup).toBeDefined();
expect(orderGroup).toBeDefined();
expect(memberGroup!.entries).toHaveLength(3);
expect(orderGroup!.entries).toHaveLength(2);
});
it("提取正确的开始/结束时间", () => {
const groups = groupLogsByTask(sampleLines);
const memberGroup = groups.find((g) => g.taskCode === "ODS_MEMBER")!;
expect(memberGroup.startTime).toBe("2024-06-01 12:00:00");
expect(memberGroup.endTime).toBe("2024-06-01 12:00:25");
});
it("有 ERROR 的任务状态为 failed", () => {
const groups = groupLogsByTask(sampleLines);
const orderGroup = groups.find((g) => g.taskCode === "ODS_ORDER")!;
expect(orderGroup.status).toBe("failed");
expect(orderGroup.counts.error).toBe(1);
});
it("最后一条含 success 的任务状态为 success", () => {
const groups = groupLogsByTask(sampleLines);
const memberGroup = groups.find((g) => g.taskCode === "ODS_MEMBER")!;
expect(memberGroup.status).toBe("success");
});
it("空行被忽略", () => {
const groups = groupLogsByTask(["", " ", "[ODS_A] 2024-01-01 00:00:00 INFO ok"]);
expect(groups).toHaveLength(1);
expect(groups[0].taskCode).toBe("ODS_A");
});
it("无法解析的行归入 _UNKNOWN", () => {
const groups = groupLogsByTask(["普通文本行", "另一行"]);
expect(groups).toHaveLength(1);
expect(groups[0].taskCode).toBe("_UNKNOWN");
expect(groups[0].entries).toHaveLength(2);
});
it("空数组返回空分组", () => {
expect(groupLogsByTask([])).toEqual([]);
});
it("日志计数正确", () => {
const lines = [
"[T1] 2024-01-01 00:00:00 INFO a",
"[T1] 2024-01-01 00:00:01 WARNING b",
"[T1] 2024-01-01 00:00:02 ERROR c",
"[T1] 2024-01-01 00:00:03 DEBUG d",
"[T1] 2024-01-01 00:00:04 INFO e",
];
const groups = groupLogsByTask(lines);
expect(groups[0].counts).toEqual({ info: 2, warning: 1, error: 1, debug: 1 });
});
});
/* ------------------------------------------------------------------ */
/* filterTaskGroups */
/* ------------------------------------------------------------------ */
describe("filterTaskGroups — 按任务代码过滤", () => {
const lines = [
"[ODS_MEMBER] 2024-06-01 12:00:00 INFO a",
"[ODS_ORDER] 2024-06-01 12:00:01 INFO b",
"[DWD_LOAD_FROM_ODS] 2024-06-01 12:00:02 INFO c",
];
it("空关键词返回全部", () => {
const groups = groupLogsByTask(lines);
expect(filterTaskGroups(groups, "")).toHaveLength(3);
expect(filterTaskGroups(groups, " ")).toHaveLength(3);
});
it("精确匹配任务代码", () => {
const groups = groupLogsByTask(lines);
const filtered = filterTaskGroups(groups, "ODS_MEMBER");
expect(filtered).toHaveLength(1);
expect(filtered[0].taskCode).toBe("ODS_MEMBER");
});
it("部分匹配（大小写不敏感）", () => {
const groups = groupLogsByTask(lines);
const filtered = filterTaskGroups(groups, "ods");
// "ODS_MEMBER", "ODS_ORDER", "DWD_LOAD_FROM_ODS" 都包含 "ods"
expect(filtered).toHaveLength(3);
});
it("无匹配返回空数组", () => {
const groups = groupLogsByTask(lines);
expect(filterTaskGroups(groups, "NONEXISTENT")).toEqual([]);
});
});

17
apps/admin-web/src/api/businessDay.ts Normal file
View File

@@ -0,0 +1,17 @@
/**
* 营业日配置 API。
*
* 从后端 /api/config/business-day 获取营业日分割点小时值。
*/
import { apiClient } from "./client";
export interface BusinessDayConfig {
business_day_start_hour: number;
}
/** 获取营业日分割点配置 */
export async function fetchBusinessDayConfig(): Promise<BusinessDayConfig> {
const { data } = await apiClient.get<BusinessDayConfig>("/config/business-day");
return data;
}

21
apps/admin-web/src/api/schedules.ts
View File

@@ -3,7 +3,7 @@
*/
import { apiClient } from './client';
import type { ScheduledTask, ScheduleConfig, TaskConfig } from '../types';
import type { ScheduledTask, ScheduleConfig, TaskConfig, ExecutionLog } from '../types';
/** 获取调度任务列表 */
export async function fetchSchedules(): Promise<ScheduledTask[]> {
@@ -17,6 +17,7 @@ export async function createSchedule(payload: {
task_codes: string[];
task_config: TaskConfig;
schedule_config: ScheduleConfig;
run_immediately?: boolean;
}): Promise<ScheduledTask> {
const { data } = await apiClient.post<ScheduledTask>('/schedules', payload);
return data;
@@ -46,3 +47,21 @@ export async function toggleSchedule(id: string): Promise<ScheduledTask> {
const { data } = await apiClient.patch<ScheduledTask>(`/schedules/${id}/toggle`);
return data;
}
/** 手动执行调度任务一次（不更新调度间隔） */
export async function runScheduleNow(id: string): Promise<{ message: string; task_id: string }> {
const { data } = await apiClient.post<{ message: string; task_id: string }>(`/schedules/${id}/run`);
return data;
}
/** 获取调度任务的执行历史 */
export async function fetchScheduleHistory(
id: string,
page = 1,
pageSize = 50,
): Promise<ExecutionLog[]> {
const { data } = await apiClient.get<ExecutionLog[]>(`/schedules/${id}/history`, {
params: { page, page_size: pageSize },
});
return data;
}

29
apps/admin-web/src/components/BusinessDayHint.tsx Normal file
View File

@@ -0,0 +1,29 @@
/**
* 营业日口径提示组件。
*
* 在日期选择器旁显示 Tooltip + 文字标注，说明当前营业日分割点。
* 例如：「营业日：08:00 起」
*/
import React from "react";
import { Tooltip, Typography } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
import { useBusinessDayStore } from "../store/businessDayStore";
const { Text } = Typography;
const BusinessDayHint: React.FC = () => {
const startHour = useBusinessDayStore((s) => s.startHour);
const hh = String(startHour).padStart(2, "0");
return (
<Tooltip title={`统计日期按营业日口径划分：每天 ${hh}:00 至次日 ${hh}:00`}>
<Text type="secondary" style={{ fontSize: 12, marginLeft: 4 }}>
<InfoCircleOutlined style={{ marginRight: 2 }} />
营业日：{hh}:00 起
</Text>
</Tooltip>
);
};
export default BusinessDayHint;

15
apps/admin-web/src/components/LogStream.tsx
View File

@@ -64,11 +64,16 @@ const LogStream: React.FC<LogStreamProps> = ({ lines }) => {
{lines.length === 0 ? (
<div style={{ color: "#888" }}>暂无日志</div>
) : (
lines.map((line, i) => (
<div key={i} style={{ whiteSpace: "pre-wrap", wordBreak: "break-all" }}>
{line}
</div>
))
lines.map((line, i) => {
let color = "#d4d4d4";
if (/\bERROR\b/i.test(line)) color = "#f56c6c";
else if (/\bWARN(?:ING)?\b/i.test(line)) color = "#e6a23c";
return (
<div key={i} style={{ whiteSpace: "pre-wrap", wordBreak: "break-all", color }}>
{line}
</div>
);
})
)}
<div ref={bottomRef} />
</div>

226
apps/admin-web/src/components/ScheduleHistoryDrawer.tsx Normal file
View File

@@ -0,0 +1,226 @@
/**
* 调度任务执行历史抽屉组件。
*
* - 表格展示执行记录（ID、状态、开始时间、耗时、退出码），50 条/页分页
* - 点击行打开详情（复用 LogStream 展示日志，running 任务走 WebSocket）
*/
import React, { useEffect, useState, useCallback, useRef } from 'react';
import {
Drawer, Table, Tag, Typography, Descriptions, Spin, Space, message,
} from 'antd';
import { FileTextOutlined } from '@ant-design/icons';
import type { ColumnsType } from 'antd/es/table';
import type { ExecutionLog } from '../types';
import { fetchScheduleHistory } from '../api/schedules';
import { apiClient } from '../api/client';
import LogStream from './LogStream';
const { Text } = Typography;
const STATUS_COLOR: Record<string, string> = {
pending: 'default',
running: 'processing',
success: 'success',
failed: 'error',
cancelled: 'warning',
};
function fmtTime(iso: string | null | undefined): string {
if (!iso) return '—';
return new Date(iso).toLocaleString('zh-CN');
}
function fmtDuration(ms: number | null | undefined): string {
if (ms == null) return '—';
if (ms < 1000) return `${ms}ms`;
const sec = ms / 1000;
if (sec < 60) return `${sec.toFixed(1)}s`;
const min = Math.floor(sec / 60);
const remainSec = Math.round(sec % 60);
return `${min}m${remainSec}s`;
}
interface Props {
open: boolean;
scheduleId: string | null;
scheduleName: string;
onClose: () => void;
}
const ScheduleHistoryDrawer: React.FC<Props> = ({ open, scheduleId, scheduleName, onClose }) => {
const [data, setData] = useState<ExecutionLog[]>([]);
const [loading, setLoading] = useState(false);
const [page, setPage] = useState(1);
const [detail, setDetail] = useState<ExecutionLog | null>(null);
const [logLines, setLogLines] = useState<string[]>([]);
const [logLoading, setLogLoading] = useState(false);
const [wsConnected, setWsConnected] = useState(false);
const wsRef = useRef<WebSocket | null>(null);
const closeWs = useCallback(() => {
wsRef.current?.close();
wsRef.current = null;
setWsConnected(false);
}, []);
const load = useCallback(async (p: number) => {
if (!scheduleId) return;
setLoading(true);
try {
setData(await fetchScheduleHistory(scheduleId, p, 50));
} catch {
message.error('加载执行历史失败');
} finally {
setLoading(false);
}
}, [scheduleId]);
useEffect(() => {
if (open && scheduleId) {
setPage(1);
setDetail(null);
load(1);
}
return () => { closeWs(); };
}, [open, scheduleId, load, closeWs]);
const handleRowClick = useCallback(async (record: ExecutionLog) => {
setDetail(record);
setLogLines([]);
setLogLoading(true);
closeWs();
if (record.status === 'running') {
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
const host = window.location.host;
const ws = new WebSocket(`${protocol}//${host}/ws/logs/${record.id}`);
wsRef.current = ws;
ws.onopen = () => { setWsConnected(true); setLogLoading(false); };
ws.onmessage = (event) => { setLogLines((prev) => [...prev, event.data]); };
ws.onclose = () => { setWsConnected(false); };
ws.onerror = () => {
setWsConnected(false);
apiClient.get<{ output_log: string | null; error_log: string | null }>(
`/execution/${record.id}/logs`,
).then(({ data: logData }) => {
const parts: string[] = [];
if (logData.output_log) parts.push(logData.output_log);
if (logData.error_log) parts.push(logData.error_log);
setLogLines(parts.join('\n').split('\n').filter(Boolean));
}).catch(() => {}).finally(() => setLogLoading(false));
};
} else {
try {
const { data: logData } = await apiClient.get<{
output_log: string | null; error_log: string | null;
}>(`/execution/${record.id}/logs`);
const parts: string[] = [];
if (logData.output_log) parts.push(logData.output_log);
if (logData.error_log) parts.push(logData.error_log);
setLogLines(parts.join('\n').split('\n').filter(Boolean));
} catch { /* 静默 */ }
finally { setLogLoading(false); }
}
}, [closeWs]);
const columns: ColumnsType<ExecutionLog> = [
{
title: '执行 ID', dataIndex: 'id', key: 'id', width: 120,
render: (id: string) => (
<Text copyable={{ text: id }} style={{ fontSize: 11 }}>
{id.slice(0, 8)}…
</Text>
),
},
{
title: '状态', dataIndex: 'status', key: 'status', width: 90,
render: (s: string) => <Tag color={STATUS_COLOR[s] ?? 'default'}>{s}</Tag>,
},
{ title: '开始时间', dataIndex: 'started_at', key: 'started_at', width: 170, render: fmtTime },
{ title: '时长', dataIndex: 'duration_ms', key: 'duration_ms', width: 90, render: fmtDuration },
{
title: '退出码', dataIndex: 'exit_code', key: 'exit_code', width: 70, align: 'center',
render: (v: number | null) => v != null
? <Tag color={v === 0 ? 'success' : 'error'}>{v}</Tag>
: '—',
},
];
return (
<Drawer
title={`执行历史 — ${scheduleName}`}
open={open}
onClose={() => { closeWs(); setDetail(null); onClose(); }}
width={800}
styles={{ body: { padding: 12 } }}
>
<Table<ExecutionLog>
rowKey="id"
columns={columns}
dataSource={data}
loading={loading}
size="small"
pagination={{
current: page,
pageSize: 50,
onChange: (p) => { setPage(p); load(p); },
showTotal: (t) => `共 ${t} 条`,
}}
onRow={(record) => ({
onClick: () => handleRowClick(record),
style: { cursor: 'pointer' },
})}
/>
{/* 执行详情 */}
<Drawer
title={
<Space>
<span>执行详情</span>
{detail?.status === 'running' && (
wsConnected ? <Tag color="processing">实时连接中</Tag> : <Tag>未连接</Tag>
)}
</Space>
}
open={!!detail}
onClose={() => { closeWs(); setDetail(null); }}
width={700}
styles={{ body: { padding: 12 } }}
>
{detail && (
<>
<Descriptions column={1} bordered size="small" style={{ marginBottom: 16 }}>
<Descriptions.Item label="执行 ID">{detail.id}</Descriptions.Item>
<Descriptions.Item label="任务">{detail.task_codes?.join(', ')}</Descriptions.Item>
<Descriptions.Item label="状态">
<Tag color={STATUS_COLOR[detail.status] ?? 'default'}>{detail.status}</Tag>
</Descriptions.Item>
<Descriptions.Item label="开始时间">{fmtTime(detail.started_at)}</Descriptions.Item>
<Descriptions.Item label="结束时间">{fmtTime(detail.finished_at)}</Descriptions.Item>
<Descriptions.Item label="时长">{fmtDuration(detail.duration_ms)}</Descriptions.Item>
<Descriptions.Item label="退出码">
{detail.exit_code != null
? <Tag color={detail.exit_code === 0 ? 'success' : 'error'}>{detail.exit_code}</Tag>
: '—'}
</Descriptions.Item>
<Descriptions.Item label="命令">
<code style={{ wordBreak: 'break-all', fontSize: 12 }}>{detail.command || '—'}</code>
</Descriptions.Item>
</Descriptions>
<div style={{ marginBottom: 8, display: 'flex', alignItems: 'center', gap: 8 }}>
<FileTextOutlined />
<Text strong>执行日志</Text>
{logLoading && <Spin size="small" />}
</div>
<div style={{ height: 400 }}>
<LogStream executionId={detail.id} lines={logLines} />
</div>
</>
)}
</Drawer>
</Drawer>
);
};
export default ScheduleHistoryDrawer;

144
apps/admin-web/src/components/ScheduleTab.tsx
View File

@@ -10,19 +10,22 @@
import React, { useEffect, useState, useCallback } from 'react';
import {
Table, Tag, Button, Switch, Popconfirm, Space, Modal, Form,
Input, Select, InputNumber, TimePicker, Checkbox, message,
Input, Select, InputNumber, TimePicker, Checkbox, message, Typography,
} from 'antd';
import { PlusOutlined, ReloadOutlined, EditOutlined, DeleteOutlined } from '@ant-design/icons';
import { ReloadOutlined, EditOutlined, DeleteOutlined, HistoryOutlined, PlayCircleOutlined } from '@ant-design/icons';
import type { ColumnsType } from 'antd/es/table';
import dayjs from 'dayjs';
import type { ScheduledTask, ScheduleConfig } from '../types';
import {
fetchSchedules,
createSchedule,
updateSchedule,
deleteSchedule,
toggleSchedule,
runScheduleNow,
} from '../api/schedules';
import ScheduleHistoryDrawer from './ScheduleHistoryDrawer';
const { Text } = Typography;
/* ------------------------------------------------------------------ */
/* 常量 & 工具 */
@@ -150,6 +153,11 @@ const ScheduleTab: React.FC = () => {
const [scheduleType, setScheduleType] = useState<string>('daily');
const [form] = Form.useForm();
/* 执行历史抽屉状态 */
const [historyOpen, setHistoryOpen] = useState(false);
const [historyScheduleId, setHistoryScheduleId] = useState<string | null>(null);
const [historyScheduleName, setHistoryScheduleName] = useState('');
/* 加载列表 */
const load = useCallback(async () => {
setLoading(true);
@@ -164,25 +172,6 @@ const ScheduleTab: React.FC = () => {
useEffect(() => { load(); }, [load]);
/* 打开创建 Modal */
const openCreate = () => {
setEditing(null);
form.resetFields();
form.setFieldsValue({
schedule_config: {
schedule_type: 'daily',
interval_value: 1,
interval_unit: 'hours',
daily_time: dayjs('04:00', 'HH:mm'),
weekly_days: [1],
weekly_time: dayjs('04:00', 'HH:mm'),
cron_expression: '0 4 * * *',
},
});
setScheduleType('daily');
setModalOpen(true);
};
/* 打开编辑 Modal */
const openEdit = (record: ScheduledTask) => {
setEditing(record);
@@ -199,13 +188,20 @@ const ScheduleTab: React.FC = () => {
setModalOpen(true);
};
/* 提交创建/编辑 */
/* 打开执行历史 */
const openHistory = (record: ScheduledTask) => {
setHistoryScheduleId(record.id);
setHistoryScheduleName(record.name);
setHistoryOpen(true);
};
/* 提交编辑 */
const handleSubmit = async () => {
if (!editing) return;
try {
const values = await form.validateFields();
setSubmitting(true);
// 将 dayjs 对象转为字符串
const cfg = { ...values.schedule_config };
if (cfg.daily_time && typeof cfg.daily_time !== 'string') {
cfg.daily_time = cfg.daily_time.format('HH:mm');
@@ -227,47 +223,16 @@ const ScheduleTab: React.FC = () => {
end_date: null,
};
if (editing) {
await updateSchedule(editing.id, {
name: values.name,
schedule_config: scheduleConfig,
});
message.success('调度任务已更新');
} else {
// 创建时使用默认 task_config（简化实现）
await createSchedule({
name: values.name,
task_codes: [],
task_config: {
tasks: [],
flow: 'api_full',
processing_mode: 'increment_only',
pipeline_flow: 'FULL',
dry_run: false,
window_mode: 'lookback',
window_start: null,
window_end: null,
window_split: null,
window_split_days: null,
lookback_hours: 24,
overlap_seconds: 600,
fetch_before_verify: false,
skip_ods_when_fetch_before_verify: false,
ods_use_local_json: false,
store_id: null,
dwd_only_tables: null,
force_full: false,
extra_args: {},
},
schedule_config: scheduleConfig,
});
message.success('调度任务已创建');
}
await updateSchedule(editing.id, {
name: values.name,
schedule_config: scheduleConfig,
});
message.success('调度任务已更新');
setModalOpen(false);
load();
} catch {
// 表单验证失败，不做额外处理
// 表单验证失败
} finally {
setSubmitting(false);
}
@@ -294,8 +259,29 @@ const ScheduleTab: React.FC = () => {
}
};
/* 手动执行一次（不更新调度间隔） */
const handleRunNow = async (id: string) => {
try {
await runScheduleNow(id);
message.success('已提交到执行队列');
} catch {
message.error('执行失败');
}
};
/* 表格列定义 */
const columns: ColumnsType<ScheduledTask> = [
{
title: '调度 ID',
dataIndex: 'id',
key: 'id',
width: 120,
render: (id: string) => (
<Text copyable={{ text: id }} style={{ fontSize: 11 }}>
{id.slice(0, 8)}…
</Text>
),
},
{
title: '名称',
dataIndex: 'name',
@@ -338,9 +324,17 @@ const ScheduleTab: React.FC = () => {
{
title: '操作',
key: 'action',
width: 140,
width: 300,
render: (_: unknown, record: ScheduledTask) => (
<Space size="small">
<Popconfirm title="确认立即执行一次？（不影响调度间隔）" onConfirm={() => handleRunNow(record.id)}>
<Button type="link" icon={<PlayCircleOutlined />} size="small">
立即执行
</Button>
</Popconfirm>
<Button type="link" icon={<HistoryOutlined />} size="small" onClick={() => openHistory(record)}>
执行历史
</Button>
<Button type="link" icon={<EditOutlined />} size="small" onClick={() => openEdit(record)}>
编辑
</Button>
@@ -356,15 +350,11 @@ const ScheduleTab: React.FC = () => {
return (
<>
<div style={{ marginBottom: 12 }}>
<Space>
<Button type="primary" icon={<PlusOutlined />} onClick={openCreate}>
新建调度
</Button>
<Button icon={<ReloadOutlined />} onClick={load} loading={loading}>
刷新
</Button>
</Space>
<div style={{ marginBottom: 12, display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
<Text type="secondary">共 {data.length} 个调度任务</Text>
<Button icon={<ReloadOutlined />} onClick={load} loading={loading}>
刷新
</Button>
</div>
<Table<ScheduledTask>
@@ -376,9 +366,9 @@ const ScheduleTab: React.FC = () => {
size="middle"
/>
{/* 创建/编辑 Modal */}
{/* 编辑 Modal */}
<Modal
title={editing ? '编辑调度任务' : '新建调度任务'}
title="编辑调度任务"
open={modalOpen}
onOk={handleSubmit}
onCancel={() => setModalOpen(false)}
@@ -400,6 +390,14 @@ const ScheduleTab: React.FC = () => {
<ScheduleConfigFields scheduleType={scheduleType} />
</Form>
</Modal>
{/* 执行历史抽屉 */}
<ScheduleHistoryDrawer
open={historyOpen}
scheduleId={historyScheduleId}
scheduleName={historyScheduleName}
onClose={() => setHistoryOpen(false)}
/>
</>
);
};

313
apps/admin-web/src/components/TaskLogViewer.tsx Normal file
View File

@@ -0,0 +1,313 @@
/**
* 按任务分组的日志展示组件。
*
* - 顶部：任务执行时间线概览（可点击跳转）
* - 中部：按任务代码过滤搜索框
* - 主体：Collapse 折叠面板，每个任务一个区块
* - 展开后显示该任务的完整日志（时间戳、级别、消息）
*
* 需求: 10.2, 10.5, 10.6
*/
import React, { useMemo, useRef, useCallback, useState } from "react";
import {
Collapse,
Timeline,
Tag,
Input,
Empty,
Badge,
Space,
Typography,
} from "antd";
import {
CheckCircleOutlined,
CloseCircleOutlined,
ClockCircleOutlined,
QuestionCircleOutlined,
SearchOutlined,
} from "@ant-design/icons";
import {
groupLogsByTask,
filterTaskGroups,
type TaskLogGroup,
type ParsedLogEntry,
} from "../utils/taskLogParser";
const { Text } = Typography;
/* ------------------------------------------------------------------ */
/* 状态 → 颜色/图标映射 */
/* ------------------------------------------------------------------ */
const STATUS_CONFIG: Record<
TaskLogGroup["status"],
{ color: string; icon: React.ReactNode; label: string }
> = {
success: { color: "green", icon: <CheckCircleOutlined />, label: "成功" },
failed: { color: "red", icon: <CloseCircleOutlined />, label: "失败" },
running: { color: "blue", icon: <ClockCircleOutlined />, label: "运行中" },
unknown: { color: "default", icon: <QuestionCircleOutlined />, label: "未知" },
};
const LEVEL_COLOR: Record<string, string> = {
ERROR: "#ff4d4f",
CRITICAL: "#ff4d4f",
WARNING: "#faad14",
INFO: "#52c41a",
DEBUG: "#8c8c8c",
};
/* ------------------------------------------------------------------ */
/* 日志级别标签 */
/* ------------------------------------------------------------------ */
const LevelTag: React.FC<{ level: string }> = ({ level }) => (
<span
style={{
color: LEVEL_COLOR[level] ?? "#8c8c8c",
fontWeight: level === "ERROR" || level === "CRITICAL" ? 600 : 400,
fontFamily: "monospace",
fontSize: 12,
minWidth: 56,
display: "inline-block",
}}
>
{level}
</span>
);
/* ------------------------------------------------------------------ */
/* 单条日志行 */
/* ------------------------------------------------------------------ */
const LogLine: React.FC<{ entry: ParsedLogEntry }> = ({ entry }) => (
<div
style={{
fontFamily: "'Cascadia Code', 'Fira Code', Consolas, monospace",
fontSize: 12,
lineHeight: 1.7,
padding: "1px 0",
whiteSpace: "pre-wrap",
wordBreak: "break-all",
}}
>
{entry.timestamp && (
<Text type="secondary" style={{ fontSize: 12, marginRight: 8, fontFamily: "monospace" }}>
{entry.timestamp}
</Text>
)}
<LevelTag level={entry.level} />
<span style={{ marginLeft: 8 }}>{entry.message}</span>
</div>
);
/* ------------------------------------------------------------------ */
/* 任务面板头部 */
/* ------------------------------------------------------------------ */
const TaskPanelHeader: React.FC<{ group: TaskLogGroup }> = ({ group }) => {
const cfg = STATUS_CONFIG[group.status];
return (
<Space size={12}>
<Text strong style={{ fontFamily: "monospace" }}>{group.taskCode}</Text>
<Tag color={cfg.color} icon={cfg.icon}>{cfg.label}</Tag>
<Text type="secondary" style={{ fontSize: 12 }}>
{group.entries.length} 条日志
</Text>
{group.counts.error > 0 && (
<Badge count={group.counts.error} size="small" title="错误数" />
)}
{group.counts.warning > 0 && (
<Badge
count={group.counts.warning}
size="small"
color="#faad14"
title="警告数"
/>
)}
{group.startTime && (
<Text type="secondary" style={{ fontSize: 11 }}>
{group.startTime}
{group.endTime && group.endTime !== group.startTime
? ` → ${group.endTime}`
: ""}
</Text>
)}
</Space>
);
};
/* ------------------------------------------------------------------ */
/* 时间线概览 */
/* ------------------------------------------------------------------ */
const TaskTimeline: React.FC<{
groups: TaskLogGroup[];
onClickTask: (taskCode: string) => void;
}> = ({ groups, onClickTask }) => {
if (groups.length === 0) return null;
const items = groups.map((g) => {
const cfg = STATUS_CONFIG[g.status];
return {
color: cfg.color === "default" ? "gray" : cfg.color,
children: (
<div
style={{ cursor: "pointer" }}
onClick={() => onClickTask(g.taskCode)}
role="button"
tabIndex={0}
onKeyDown={(e) => {
if (e.key === "Enter") onClickTask(g.taskCode);
}}
>
<Space size={8}>
<Text strong style={{ fontFamily: "monospace", fontSize: 13 }}>
{g.taskCode}
</Text>
<Tag color={cfg.color} style={{ fontSize: 11 }}>
{cfg.label}
</Tag>
{g.startTime && (
<Text type="secondary" style={{ fontSize: 11 }}>
{g.startTime}
{g.endTime && g.endTime !== g.startTime
? ` → ${g.endTime}`
: ""}
</Text>
)}
<Text type="secondary" style={{ fontSize: 11 }}>
({g.entries.length} 条)
</Text>
</Space>
</div>
),
};
});
return (
<div style={{ marginBottom: 12, padding: "8px 12px", background: "#fafafa", borderRadius: 4 }}>
<Text type="secondary" style={{ fontSize: 12, marginBottom: 8, display: "block" }}>
任务执行时间线（点击跳转）
</Text>
<Timeline items={items} style={{ marginBottom: 0, paddingTop: 8 }} />
</div>
);
};
/* ------------------------------------------------------------------ */
/* 主组件 */
/* ------------------------------------------------------------------ */
export interface TaskLogViewerProps {
/** 原始日志行数组 */
lines: string[];
}
const TaskLogViewer: React.FC<TaskLogViewerProps> = ({ lines }) => {
const [taskFilter, setTaskFilter] = useState("");
const [activeKeys, setActiveKeys] = useState<string[]>([]);
const panelRefs = useRef<Record<string, HTMLDivElement | null>>({});
// 解析并分组
const allGroups = useMemo(() => groupLogsByTask(lines), [lines]);
const filteredGroups = useMemo(
() => filterTaskGroups(allGroups, taskFilter),
[allGroups, taskFilter],
);
// 点击时间线跳转到对应任务面板
const handleTimelineClick = useCallback((taskCode: string) => {
// 展开目标面板
setActiveKeys((prev) =>
prev.includes(taskCode) ? prev : [...prev, taskCode],
);
// 滚动到面板位置
requestAnimationFrame(() => {
const el = panelRefs.current[taskCode];
if (el) {
el.scrollIntoView({ behavior: "smooth", block: "start" });
}
});
}, []);
const setPanelRef = useCallback(
(taskCode: string) => (el: HTMLDivElement | null) => {
panelRefs.current[taskCode] = el;
},
[],
);
if (lines.length === 0) {
return <Empty description="暂无日志数据" />;
}
// 如果没有可解析的任务分组（全是无法解析的行），回退到普通展示
if (allGroups.length === 0) {
return <Empty description="无法解析任务分组" />;
}
const collapseItems = filteredGroups.map((group) => ({
key: group.taskCode,
label: <TaskPanelHeader group={group} />,
children: (
<div
style={{
maxHeight: 400,
overflow: "auto",
background: "#1e1e1e",
color: "#d4d4d4",
padding: 12,
borderRadius: 4,
}}
>
{group.entries.map((entry, i) => (
<LogLine key={i} entry={entry} />
))}
</div>
),
}));
return (
<div>
{/* 时间线概览 */}
<TaskTimeline groups={allGroups} onClickTask={handleTimelineClick} />
{/* 任务代码过滤 */}
{allGroups.length > 1 && (
<div style={{ marginBottom: 12 }}>
<Input
prefix={<SearchOutlined />}
placeholder="按任务代码过滤..."
value={taskFilter}
onChange={(e) => setTaskFilter(e.target.value)}
allowClear
style={{ width: 280 }}
/>
<Text type="secondary" style={{ marginLeft: 12, fontSize: 12 }}>
{filteredGroups.length} / {allGroups.length} 个任务
</Text>
</div>
)}
{/* 折叠面板 */}
{filteredGroups.length === 0 ? (
<Empty description="无匹配的任务" />
) : (
<>
{filteredGroups.map((g) => (
<div key={g.taskCode} ref={setPanelRef(g.taskCode)} />
))}
<Collapse
activeKey={activeKeys}
onChange={(keys) => setActiveKeys(keys as string[])}
items={collapseItems}
/>
</>
)}
</div>
);
};
export default TaskLogViewer;

45
apps/admin-web/src/pages/LogViewer.tsx
View File

@@ -7,13 +7,15 @@
*/
import React, { useState, useRef, useCallback, useEffect } from "react";
import { Input, Button, Space, message, Card, Typography, Tag, Badge } from "antd";
import { Input, Button, Space, message, Card, Typography, Tag, Badge, Segmented } from "antd";
import {
LinkOutlined, DisconnectOutlined, HistoryOutlined,
FileTextOutlined, SearchOutlined, ClearOutlined,
AppstoreOutlined, UnorderedListOutlined,
} from "@ant-design/icons";
import { apiClient } from "../api/client";
import LogStream from "../components/LogStream";
import TaskLogViewer from "../components/TaskLogViewer";
const { Title, Text } = Typography;
@@ -36,6 +38,8 @@ const LogViewer: React.FC = () => {
const [lines, setLines] = useState<string[]>([]);
const [filterKeyword, setFilterKeyword] = useState("");
const [connected, setConnected] = useState(false);
/** 展示模式：raw = 原始流，grouped = 按任务分组 */
const [viewMode, setViewMode] = useState<"raw" | "grouped">("grouped");
const wsRef = useRef<WebSocket | null>(null);
useEffect(() => {
@@ -116,20 +120,37 @@ const LogViewer: React.FC = () => {
<Button icon={<HistoryOutlined />} onClick={handleLoadHistory}>加载历史</Button>
<Button icon={<ClearOutlined />} onClick={handleClear} disabled={lines.length === 0}>清空</Button>
</Space>
<Input
prefix={<SearchOutlined />}
placeholder="过滤关键词..."
value={filterKeyword}
onChange={(e) => setFilterKeyword(e.target.value)}
allowClear
style={{ width: 220 }}
/>
<Space>
<Segmented
value={viewMode}
onChange={(v) => setViewMode(v as "raw" | "grouped")}
options={[
{ value: "grouped", icon: <AppstoreOutlined />, label: "按任务" },
{ value: "raw", icon: <UnorderedListOutlined />, label: "原始" },
]}
size="small"
/>
{viewMode === "raw" && (
<Input
prefix={<SearchOutlined />}
placeholder="过滤关键词..."
value={filterKeyword}
onChange={(e) => setFilterKeyword(e.target.value)}
allowClear
style={{ width: 220 }}
/>
)}
</Space>
</Space>
</Card>
{/* 日志流 */}
<div style={{ flex: 1, minHeight: 0 }}>
<LogStream executionId={executionId} lines={filteredLines} />
{/* 日志展示区域 */}
<div style={{ flex: 1, minHeight: 0, overflow: "auto" }}>
{viewMode === "grouped" ? (
<TaskLogViewer lines={lines} />
) : (
<LogStream executionId={executionId} lines={filteredLines} />
)}
</div>
</div>
);

296
apps/admin-web/src/pages/TaskConfig.tsx
View File

@@ -25,6 +25,12 @@ import {
Tooltip,
Segmented,
Spin,
Modal,
Form,
Select,
TimePicker,
Descriptions,
Tag,
} from "antd";
import {
SendOutlined,
@@ -35,16 +41,20 @@ import {
SyncOutlined,
ShopOutlined,
ApiOutlined,
ScheduleOutlined,
} from "@ant-design/icons";
import { useNavigate } from "react-router-dom";
import TaskSelector from "../components/TaskSelector";
import { validateTaskConfig, fetchFlows } from "../api/tasks";
import type { FlowDef, ProcessingModeDef } from "../api/tasks";
import { submitToQueue, executeDirectly } from "../api/execution";
import { createSchedule } from "../api/schedules";
import { useAuthStore } from "../store/authStore";
import BusinessDayHint from "../components/BusinessDayHint";
import type { RadioChangeEvent } from "antd";
import type { Dayjs } from "dayjs";
import type { TaskConfig as TaskConfigType } from "../types";
import dayjs from "dayjs";
import type { TaskConfig as TaskConfigType, ScheduleConfig } from "../types";
const { Title, Text } = Typography;
const { TextArea } = Input;
@@ -224,6 +234,12 @@ const TaskConfig: React.FC = () => {
const [forceFull, setForceFull] = useState(false);
const [useLocalJson, setUseLocalJson] = useState(false);
/* ---------- Pipeline 调优 ---------- */
const [pipelineWorkers, setPipelineWorkers] = useState<number | null>(null);
const [pipelineBatchSize, setPipelineBatchSize] = useState<number | null>(null);
const [pipelineRateMin, setPipelineRateMin] = useState<number | null>(null);
const [pipelineRateMax, setPipelineRateMax] = useState<number | null>(null);
/* ---------- CLI 预览 ---------- */
const [cliCommand, setCliCommand] = useState<string>("");
const [cliEdited, setCliEdited] = useState(false);
@@ -232,6 +248,12 @@ const TaskConfig: React.FC = () => {
/* ---------- 提交状态 ---------- */
const [submitting, setSubmitting] = useState(false);
/* ---------- 调度任务弹窗 ---------- */
const [scheduleModalOpen, setScheduleModalOpen] = useState(false);
const [scheduleSubmitting, setScheduleSubmitting] = useState(false);
const [scheduleType, setScheduleType] = useState<string>("daily");
const [scheduleForm] = Form.useForm();
/* ---------- 派生状态 ---------- */
const layers = flowDefs[flow]?.layers ?? [];
const showVerifyOption = processingMode === "verify_only";
@@ -262,6 +284,10 @@ const TaskConfig: React.FC = () => {
/* CHANGE [2026-02-19] intent: DWD 表正向勾选，选中=装载 */
dwd_only_tables: layers.includes("DWD") ? (selectedDwdTables.length > 0 ? selectedDwdTables : null) : null,
force_full: forceFull,
pipeline_workers: pipelineWorkers,
pipeline_batch_size: pipelineBatchSize,
pipeline_rate_min: pipelineRateMin,
pipeline_rate_max: pipelineRateMax,
extra_args: {},
};
};
@@ -288,7 +314,8 @@ const TaskConfig: React.FC = () => {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [flow, processingMode, fetchBeforeVerify, windowMode, lookbackHours, overlapSeconds,
windowStart, windowEnd, windowSplitDays, selectedTasks, selectedDwdTables,
dryRun, forceFull, useLocalJson, selectedConnectorStores]);
dryRun, forceFull, useLocalJson, selectedConnectorStores,
pipelineWorkers, pipelineBatchSize, pipelineRateMin, pipelineRateMax]);
/* ---------- 事件处理 ---------- */
const handleFlowChange = (e: RadioChangeEvent) => setFlow(e.target.value);
@@ -321,6 +348,67 @@ const TaskConfig: React.FC = () => {
}
};
/* ---------- 调度任务弹窗 ---------- */
const handleOpenScheduleModal = () => {
scheduleForm.resetFields();
scheduleForm.setFieldsValue({
schedule_config: {
schedule_type: "daily",
interval_value: 1,
interval_unit: "hours",
daily_time: dayjs("04:00", "HH:mm"),
weekly_days: [1],
weekly_time: dayjs("04:00", "HH:mm"),
cron_expression: "0 4 * * *",
},
});
setScheduleType("daily");
setScheduleModalOpen(true);
};
const handleScheduleSubmit = async () => {
try {
const values = await scheduleForm.validateFields();
setScheduleSubmitting(true);
const cfg = { ...values.schedule_config };
if (cfg.daily_time && typeof cfg.daily_time !== "string") {
cfg.daily_time = cfg.daily_time.format("HH:mm");
}
if (cfg.weekly_time && typeof cfg.weekly_time !== "string") {
cfg.weekly_time = cfg.weekly_time.format("HH:mm");
}
const scheduleConfig: ScheduleConfig = {
schedule_type: cfg.schedule_type ?? "daily",
interval_value: cfg.interval_value ?? 1,
interval_unit: cfg.interval_unit ?? "hours",
daily_time: cfg.daily_time ?? "04:00",
weekly_days: cfg.weekly_days ?? [1],
weekly_time: cfg.weekly_time ?? "04:00",
cron_expression: cfg.cron_expression ?? "0 4 * * *",
enabled: true,
start_date: null,
end_date: null,
};
const taskConfig = buildTaskConfig();
await createSchedule({
name: values.name,
task_codes: taskConfig.tasks,
task_config: taskConfig,
schedule_config: scheduleConfig,
run_immediately: !!values.run_immediately,
});
message.success("调度任务已创建");
setScheduleModalOpen(false);
} catch {
// 表单验证失败
} finally {
setScheduleSubmitting(false);
}
};
/* ---------- 样式常量 ---------- */
const cardStyle = { marginBottom: 12 };
const sectionTitleStyle: React.CSSProperties = {
@@ -464,6 +552,7 @@ const TaskConfig: React.FC = () => {
</Col>
</Row>
) : (
<>
<Row gutter={16}>
<Col span={12}>
<Text style={sectionTitleStyle}>开始日期</Text>
@@ -481,6 +570,10 @@ const TaskConfig: React.FC = () => {
/>
</Col>
</Row>
<div style={{ marginTop: 4 }}>
<BusinessDayHint />
</div>
</>
)}
<div style={{ marginTop: 12 }}>
@@ -526,6 +619,65 @@ const TaskConfig: React.FC = () => {
</div>
</Col>
</Row>
{/* Pipeline 调优参数 */}
<div style={{ marginTop: 12, borderTop: "1px solid #f0f0f0", paddingTop: 12 }}>
<Text type="secondary" style={{ fontSize: 12, marginBottom: 8, display: "block" }}>
Pipeline 调优（留空使用默认值）
</Text>
<Row gutter={[24, 8]}>
<Col span={6}>
<Text style={{ fontSize: 12 }}>并发 workers</Text>
<InputNumber
size="small"
min={1}
max={32}
placeholder="默认 3"
value={pipelineWorkers}
onChange={(v) => setPipelineWorkers(v)}
style={{ width: "100%" }}
/>
</Col>
<Col span={6}>
<Text style={{ fontSize: 12 }}>批量大小</Text>
<InputNumber
size="small"
min={1}
max={10000}
placeholder="默认 200"
value={pipelineBatchSize}
onChange={(v) => setPipelineBatchSize(v)}
style={{ width: "100%" }}
/>
</Col>
<Col span={6}>
<Text style={{ fontSize: 12 }}>限流下限(秒)</Text>
<InputNumber
size="small"
min={0}
max={60}
step={0.1}
placeholder="默认 1.0"
value={pipelineRateMin}
onChange={(v) => setPipelineRateMin(v)}
style={{ width: "100%" }}
/>
</Col>
<Col span={6}>
<Text style={{ fontSize: 12 }}>限流上限(秒)</Text>
<InputNumber
size="small"
min={0}
max={60}
step={0.1}
placeholder="默认 3.0"
value={pipelineRateMax}
onChange={(v) => setPipelineRateMax(v)}
style={{ width: "100%" }}
/>
</Col>
</Row>
</div>
</Card>
{/* ---- 任务选择（含 DWD 表过滤） ---- */}
@@ -587,26 +739,142 @@ const TaskConfig: React.FC = () => {
{/* ---- 操作按钮 ---- */}
<Card size="small" style={{ marginBottom: 24 }}>
<Space size="middle">
<Button
type="primary"
size="large"
icon={<SendOutlined />}
loading={submitting}
onClick={handleSubmitToQueue}
>
提交到队列
</Button>
<div style={{ display: "flex", alignItems: "center" }}>
<Space size="middle">
<Button
type="primary"
size="large"
icon={<SendOutlined />}
loading={submitting}
onClick={handleSubmitToQueue}
>
提交到队列
</Button>
<Button
size="large"
icon={<ScheduleOutlined />}
onClick={handleOpenScheduleModal}
>
添加到调度任务
</Button>
</Space>
<div style={{ flex: 1 }} />
<Button
size="large"
icon={<ThunderboltOutlined />}
loading={submitting}
onClick={handleExecuteDirectly}
>
直接执行
直接执行！
</Button>
</Space>
</div>
</Card>
{/* ---- 调度任务创建 Modal ---- */}
<Modal
title="添加到调度任务"
open={scheduleModalOpen}
onOk={handleScheduleSubmit}
onCancel={() => setScheduleModalOpen(false)}
confirmLoading={scheduleSubmitting}
destroyOnClose
width={560}
>
{/* 当前配置摘要（只读） */}
<Descriptions
column={1}
bordered
size="small"
style={{ marginBottom: 16 }}
title="当前任务配置（只读）"
>
<Descriptions.Item label="任务">
{selectedTasks.length > 0
? selectedTasks.map((t) => <Tag key={t}>{t}</Tag>)
: <Text type="secondary">未选择</Text>}
</Descriptions.Item>
<Descriptions.Item label="Flow">
<Tag>{flow}</Tag>
</Descriptions.Item>
<Descriptions.Item label="处理模式">{processingMode}</Descriptions.Item>
<Descriptions.Item label="时间窗口">
{windowMode === "lookback"
? `回溯 ${lookbackHours} 小时`
: `${windowStart?.format("YYYY-MM-DD") ?? "?"} ~ ${windowEnd?.format("YYYY-MM-DD") ?? "?"}`}
</Descriptions.Item>
</Descriptions>
<Form form={scheduleForm} layout="vertical" preserve={false}>
<Form.Item
name="name"
label="调度任务名称"
rules={[{ required: true, message: "请输入调度任务名称" }]}
>
<Input placeholder="例如：每日全量同步" />
</Form.Item>
<Form.Item
name={["schedule_config", "schedule_type"]}
label="调度类型"
rules={[{ required: true }]}
>
<Select
options={[
{ label: "一次性", value: "once" },
{ label: "固定间隔", value: "interval" },
{ label: "每日", value: "daily" },
{ label: "每周", value: "weekly" },
{ label: "Cron", value: "cron" },
]}
onChange={(v: string) => setScheduleType(v)}
/>
</Form.Item>
{scheduleType === "interval" && (
<Space>
<Form.Item name={["schedule_config", "interval_value"]} noStyle rules={[{ required: true }]}>
<InputNumber min={1} placeholder="间隔值" />
</Form.Item>
<Form.Item name={["schedule_config", "interval_unit"]} noStyle rules={[{ required: true }]}>
<Select style={{ width: 100 }} options={[
{ label: "分钟", value: "minutes" },
{ label: "小时", value: "hours" },
{ label: "天", value: "days" },
]} />
</Form.Item>
</Space>
)}
{scheduleType === "daily" && (
<Form.Item name={["schedule_config", "daily_time"]} label="执行时间" rules={[{ required: true }]}>
<TimePicker format="HH:mm" />
</Form.Item>
)}
{scheduleType === "weekly" && (
<>
<Form.Item name={["schedule_config", "weekly_days"]} label="星期" rules={[{ required: true }]}>
<Checkbox.Group options={[
{ label: "周一", value: 1 }, { label: "周二", value: 2 },
{ label: "周三", value: 3 }, { label: "周四", value: 4 },
{ label: "周五", value: 5 }, { label: "周六", value: 6 },
{ label: "周日", value: 0 },
]} />
</Form.Item>
<Form.Item name={["schedule_config", "weekly_time"]} label="执行时间" rules={[{ required: true }]}>
<TimePicker format="HH:mm" />
</Form.Item>
</>
)}
{scheduleType === "cron" && (
<Form.Item name={["schedule_config", "cron_expression"]} label="Cron 表达式" rules={[{ required: true }]}>
<Input placeholder="0 4 * * *" />
</Form.Item>
)}
<Form.Item name="run_immediately" valuePropName="checked" style={{ marginBottom: 0 }}>
<Checkbox>创建后立即执行一次</Checkbox>
</Form.Item>
</Form>
</Modal>
</div>
);
};

138
apps/admin-web/src/pages/TaskManager.tsx
View File

@@ -138,6 +138,14 @@ const QueueTab: React.FC = () => {
};
const columns: ColumnsType<QueuedTask> = [
{
title: '任务 ID', dataIndex: 'id', key: 'id', width: 120,
render: (id: string) => (
<Text copyable={{ text: id }} style={{ fontSize: 11 }}>
{id.slice(0, 8)}…
</Text>
),
},
{
title: '任务', dataIndex: ['config', 'tasks'], key: 'tasks',
render: (tasks: string[]) => (
@@ -234,6 +242,26 @@ const HistoryTab: React.FC = () => {
const [detail, setDetail] = useState<ExecutionLog | null>(null);
const [historyLogLines, setHistoryLogLines] = useState<string[]>([]);
const [logLoading, setLogLoading] = useState(false);
const [wsConnected, setWsConnected] = useState(false);
const historyWsRef = useRef<WebSocket | null>(null);
/** 关闭 WebSocket 连接并重置状态 */
const closeHistoryWs = useCallback(() => {
historyWsRef.current?.close();
historyWsRef.current = null;
setWsConnected(false);
}, []);
/* 组件卸载时清理 WebSocket */
useEffect(() => {
return () => { historyWsRef.current?.close(); };
}, []);
const handleCancelHistory = useCallback(async (id: string) => {
try { await cancelExecution(id); message.success('已发送终止信号'); load(); }
catch { message.error('终止失败'); }
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
const load = useCallback(async () => {
setLoading(true);
@@ -244,29 +272,73 @@ const HistoryTab: React.FC = () => {
useEffect(() => { load(); }, [load]);
/** 点击行时加载详情和日志 */
/** 点击行时加载详情和日志；running 任务走 WebSocket 实时流 */
const handleRowClick = useCallback(async (record: ExecutionLog) => {
setDetail(record);
setHistoryLogLines([]);
setLogLoading(true);
try {
const { data: logData } = await apiClient.get<{
execution_id: string;
output_log: string | null;
error_log: string | null;
}>(`/execution/${record.id}/logs`);
const parts: string[] = [];
if (logData.output_log) parts.push(logData.output_log);
if (logData.error_log) parts.push(logData.error_log);
setHistoryLogLines(parts.join('\n').split('\n').filter(Boolean));
} catch {
/* 日志可能不存在，静默处理 */
} finally {
setLogLoading(false);
closeHistoryWs();
if (record.status === 'running') {
// running 任务：通过 WebSocket 实时推送日志
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
const host = window.location.host;
const ws = new WebSocket(`${protocol}//${host}/ws/logs/${record.id}`);
historyWsRef.current = ws;
ws.onopen = () => { setWsConnected(true); setLogLoading(false); };
ws.onmessage = (event) => {
setHistoryLogLines((prev) => [...prev, event.data]);
};
ws.onclose = () => {
setWsConnected(false);
// 任务结束后刷新历史列表以更新状态
load();
};
ws.onerror = () => {
message.error('WebSocket 连接失败，回退到静态日志');
setWsConnected(false);
// 回退：用 REST API 拉取已有日志
apiClient.get<{
execution_id: string;
output_log: string | null;
error_log: string | null;
}>(`/execution/${record.id}/logs`).then(({ data: logData }) => {
const parts: string[] = [];
if (logData.output_log) parts.push(logData.output_log);
if (logData.error_log) parts.push(logData.error_log);
setHistoryLogLines(parts.join('\n').split('\n').filter(Boolean));
}).catch(() => {}).finally(() => setLogLoading(false));
};
} else {
// 已完成任务：REST API 一次性拉取
try {
const { data: logData } = await apiClient.get<{
execution_id: string;
output_log: string | null;
error_log: string | null;
}>(`/execution/${record.id}/logs`);
const parts: string[] = [];
if (logData.output_log) parts.push(logData.output_log);
if (logData.error_log) parts.push(logData.error_log);
setHistoryLogLines(parts.join('\n').split('\n').filter(Boolean));
} catch {
/* 日志可能不存在，静默处理 */
} finally {
setLogLoading(false);
}
}
}, []);
}, [closeHistoryWs, load]);
const columns: ColumnsType<ExecutionLog> = [
{
title: '执行 ID', dataIndex: 'id', key: 'id', width: 120,
render: (id: string) => (
<Text copyable={{ text: id }} style={{ fontSize: 11 }}>
{id.slice(0, 8)}…
</Text>
),
},
{
title: '任务', dataIndex: 'task_codes', key: 'task_codes',
render: (codes: string[]) => (
@@ -275,6 +347,12 @@ const HistoryTab: React.FC = () => {
</Text>
),
},
{
title: '调度 ID', dataIndex: 'schedule_id', key: 'schedule_id', width: 120,
render: (id: string | null) => id
? <Text copyable={{ text: id }} style={{ fontSize: 11 }}>{id.slice(0, 8)}…</Text>
: '—',
},
{
title: '状态', dataIndex: 'status', key: 'status', width: 90,
render: (s: string) => <Tag color={STATUS_COLOR[s] ?? 'default'}>{s}</Tag>,
@@ -287,6 +365,21 @@ const HistoryTab: React.FC = () => {
<Tag color={v === 0 ? 'success' : 'error'}>{v}</Tag>
) : '—',
},
{
title: '操作', key: 'action', width: 80, align: 'center',
render: (_: unknown, record: ExecutionLog) => {
if (record.status === 'running') {
return (
<Popconfirm title="确认终止该任务？" onConfirm={(e) => { e?.stopPropagation(); handleCancelHistory(record.id); }} onCancel={(e) => e?.stopPropagation()}>
<Button type="link" danger icon={<StopOutlined />} size="small" onClick={(e) => e.stopPropagation()}>
终止
</Button>
</Popconfirm>
);
}
return null;
},
},
];
return (
@@ -303,7 +396,18 @@ const HistoryTab: React.FC = () => {
/>
<Drawer
title="执行详情" open={!!detail} onClose={() => setDetail(null)}
title={
<Space>
<span>执行详情</span>
{detail?.status === 'running' && (
wsConnected
? <Tag color="processing">实时连接中</Tag>
: <Tag>未连接</Tag>
)}
</Space>
}
open={!!detail}
onClose={() => { closeHistoryWs(); setDetail(null); }}
width={720}
styles={{ body: { padding: 12 } }}
>

41
apps/admin-web/src/store/businessDayStore.ts Normal file
View File

@@ -0,0 +1,41 @@
/**
* 营业日配置全局状态 — Zustand store。
*
* - 启动时请求一次 /api/config/business-day
* - API 不可用时降级为默认值 8，console.warn 输出警告
*/
import { create } from "zustand";
import { fetchBusinessDayConfig } from "../api/businessDay";
const DEFAULT_START_HOUR = 8;
export interface BusinessDayState {
/** 营业日分割点小时值（0–23） */
startHour: number;
/** 是否已完成加载（含降级） */
loaded: boolean;
/** 启动时调用一次，从后端获取配置 */
init: () => Promise<void>;
}
export const useBusinessDayStore = create<BusinessDayState>((set, get) => ({
startHour: DEFAULT_START_HOUR,
loaded: false,
async init() {
// 避免重复请求
if (get().loaded) return;
try {
const config = await fetchBusinessDayConfig();
set({ startHour: config.business_day_start_hour, loaded: true });
} catch {
console.warn(
"[BusinessDay] 无法获取营业日配置，使用默认值:",
DEFAULT_START_HOUR,
);
set({ startHour: DEFAULT_START_HOUR, loaded: true });
}
},
}));

7
apps/admin-web/src/types/index.ts
View File

@@ -32,6 +32,11 @@ export interface TaskConfig {
dwd_only_tables: string[] | null;
/** 强制全量处理（跳过 hash 去重和变更对比） */
force_full: boolean;
/** Pipeline 调优参数（可选） */
pipeline_workers: number | null;
pipeline_batch_size: number | null;
pipeline_rate_min: number | null;
pipeline_rate_max: number | null;
extra_args: Record<string, unknown>;
}
@@ -102,6 +107,7 @@ export interface QueuedTask {
finished_at: string | null;
exit_code: number | null;
error_message: string | null;
schedule_id: string | null;
}
/** 执行历史记录 */
@@ -116,6 +122,7 @@ export interface ExecutionLog {
duration_ms: number | null;
command: string;
summary: Record<string, unknown> | null;
schedule_id: string | null;
}
/** 调度任务 */

130
apps/admin-web/src/utils/taskLogParser.ts Normal file
View File

@@ -0,0 +1,130 @@
/**
* 任务日志解析与分组工具函数。
*
* 解析 TaskLogBuffer 输出的日志格式：[TASK_CODE] timestamp level message
* 将扁平日志行按任务代码分组，提取时间线信息。
*/
/** 单条解析后的日志条目 */
export interface ParsedLogEntry {
/** 原始行文本 */
raw: string;
/** 任务代码（如 ODS_MEMBER） */
taskCode: string;
/** 时间戳字符串（原始格式） */
timestamp: string;
/** 日志级别：INFO / WARNING / ERROR / DEBUG */
level: string;
/** 日志消息正文 */
message: string;
}
/** 按任务分组后的日志集合 */
export interface TaskLogGroup {
taskCode: string;
entries: ParsedLogEntry[];
/** 该任务第一条日志的时间戳 */
startTime: string | null;
/** 该任务最后一条日志的时间戳 */
endTime: string | null;
/** 推断的任务状态：success / failed / running / unknown */
status: "success" | "failed" | "running" | "unknown";
/** 各级别日志计数 */
counts: { info: number; warning: number; error: number; debug: number };
}
/**
* 匹配 TaskLogBuffer 输出格式：[TASK_CODE] 2024-01-01 12:00:00 INFO 消息内容
* 也兼容：[TASK_CODE] 2024-01-01 12:00:00,123 INFO 消息内容
*/
const LOG_LINE_REGEX =
/^\[([A-Z][A-Z0-9_]*)\]\s+(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}(?:[.,]\d+)?)\s+(DEBUG|INFO|WARNING|ERROR|CRITICAL)\s+(.*)$/;
/**
* 解析单行日志。无法解析时返回 null。
*/
export function parseLogLine(line: string): ParsedLogEntry | null {
const match = LOG_LINE_REGEX.exec(line.trim());
if (!match) return null;
return {
raw: line,
taskCode: match[1],
timestamp: match[2],
level: match[3],
message: match[4],
};
}
/**
* 将日志行数组按任务代码分组。
* 无法解析的行归入 "_UNKNOWN" 分组。
*/
export function groupLogsByTask(lines: string[]): TaskLogGroup[] {
const groupMap = new Map<string, ParsedLogEntry[]>();
for (const line of lines) {
if (!line.trim()) continue;
const parsed = parseLogLine(line);
const code = parsed?.taskCode ?? "_UNKNOWN";
const entry: ParsedLogEntry = parsed ?? {
raw: line,
taskCode: "_UNKNOWN",
timestamp: "",
level: "INFO",
message: line,
};
const arr = groupMap.get(code);
if (arr) {
arr.push(entry);
} else {
groupMap.set(code, [entry]);
}
}
const groups: TaskLogGroup[] = [];
for (const [taskCode, entries] of groupMap) {
const counts = { info: 0, warning: 0, error: 0, debug: 0 };
for (const e of entries) {
const lvl = e.level.toLowerCase();
if (lvl === "info") counts.info++;
else if (lvl === "warning" || lvl === "critical") counts.warning++;
else if (lvl === "error") counts.error++;
else if (lvl === "debug") counts.debug++;
}
// 推断状态：有 ERROR 级别 → failed；否则看最后一条消息
let status: TaskLogGroup["status"] = "unknown";
if (counts.error > 0) {
status = "failed";
} else if (entries.length > 0) {
const lastMsg = entries[entries.length - 1].message.toLowerCase();
if (lastMsg.includes("完成") || lastMsg.includes("success") || lastMsg.includes("finished")) {
status = "success";
}
}
const timestamps = entries.map((e) => e.timestamp).filter(Boolean);
groups.push({
taskCode,
entries,
startTime: timestamps[0] ?? null,
endTime: timestamps.length > 0 ? timestamps[timestamps.length - 1] : null,
status,
counts,
});
}
return groups;
}
/**
* 按任务代码过滤分组。空关键词返回全部。
*/
export function filterTaskGroups(
groups: TaskLogGroup[],
keyword: string,
): TaskLogGroup[] {
const kw = keyword.trim().toUpperCase();
if (!kw) return groups;
return groups.filter((g) => g.taskCode.toUpperCase().includes(kw));
}

2
apps/admin-web/tsconfig.tsbuildinfo
View File

@@ -1 +1 @@
{"root":["./src/app.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/__tests__/flowlayers.test.ts","./src/__tests__/logfilter.test.ts","./src/api/client.ts","./src/api/dbviewer.ts","./src/api/envconfig.ts","./src/api/etlstatus.ts","./src/api/execution.ts","./src/api/schedules.ts","./src/api/tasks.ts","./src/components/dwdtableselector.tsx","./src/components/errorboundary.tsx","./src/components/logstream.tsx","./src/components/scheduletab.tsx","./src/components/taskselector.tsx","./src/pages/dbviewer.tsx","./src/pages/etlstatus.tsx","./src/pages/envconfig.tsx","./src/pages/logviewer.tsx","./src/pages/login.tsx","./src/pages/taskconfig.tsx","./src/pages/taskmanager.tsx","./src/store/authstore.ts","./src/types/index.ts"],"version":"5.8.3"}
{"root":["./src/app.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/__tests__/flowlayers.test.ts","./src/__tests__/logfilter.test.ts","./src/__tests__/tasklogparser.test.ts","./src/api/businessday.ts","./src/api/client.ts","./src/api/dbviewer.ts","./src/api/envconfig.ts","./src/api/etlstatus.ts","./src/api/execution.ts","./src/api/opspanel.ts","./src/api/schedules.ts","./src/api/tasks.ts","./src/components/businessdayhint.tsx","./src/components/dwdtableselector.tsx","./src/components/errorboundary.tsx","./src/components/logstream.tsx","./src/components/schedulehistorydrawer.tsx","./src/components/scheduletab.tsx","./src/components/tasklogviewer.tsx","./src/components/taskselector.tsx","./src/pages/dbviewer.tsx","./src/pages/etlstatus.tsx","./src/pages/envconfig.tsx","./src/pages/logviewer.tsx","./src/pages/login.tsx","./src/pages/opspanel.tsx","./src/pages/taskconfig.tsx","./src/pages/taskmanager.tsx","./src/store/authstore.ts","./src/store/businessdaystore.ts","./src/types/index.ts","./src/utils/tasklogparser.ts"],"version":"5.8.3"}

162
apps/backend/README.md
View File

@@ -1,42 +1,166 @@
# apps/backend - FastAPI 后端
# apps/backend — FastAPI 后端
为微信小程序提供 RESTful API，连接 zqyy_app 业务数据库，通过 FDW 只读访问 ETL 数据。
为微信小程序和管理后台提供 RESTful API，连接 `zqyy_app` 业务数据库，通过 FDW 只读访问 ETL 数据。
## 内部结构
## 技术栈
- Python 3.10+ / FastAPI 0.115+ / Uvicorn
- PostgreSQL（psycopg2 直连，纯 SQL，无 ORM）
- JWT 认证（python-jose + bcrypt）
- WebSocket（实时日志推送）
- neozqyy-shared（共享枚举、金额精度、时间工具）
## 目录结构
```
apps/backend/
├── app/
│ ├── main.py # FastAPI 入口，启用 OpenAPI 文档
│ ├── config.py # 配置加载
│ ├── database.py # zqyy_app 数据库连接
│ ├── routers/ # 路由模块
│ ├── middleware/ # 中间件（鉴权、日志等）
│ └── schemas/ # Pydantic 请求/响应模型
├── tests/ # 后端测试
├── pyproject.toml # 依赖声明
│ ├── main.py # FastAPI 入口（lifespan 管理 + 路由注册）
│ ├── config.py # 配置加载（根 .env < .env.local < 环境变量）
│ ├── database.py # 数据库连接（zqyy_app 读写 + etl_feiqiu 只读 RLS）
│ ├── auth/ # 认证模块
│ │ ├── dependencies.py # FastAPI 依赖注入（CurrentUser）
│ │ └── jwt.py # JWT 签发/验证/密码哈希
│ ├── routers/ # 17 个路由模块（详见 API 参考）
│ ├── schemas/ # Pydantic 请求/响应模型
│ ├── services/ # 业务逻辑层
│ ├── middleware/ # 中间件
│ └── ws/ # WebSocket（实时日志）
├── tests/ # 后端测试
├── pyproject.toml # 依赖声明
└── README.md
```
## 启动
```bash
# 确保已在根目录执行 uv sync --all-packages
# 安装依赖（在 monorepo 根目录）
uv sync --all-packages
# 启动开发服务器
cd apps/backend
uv run uvicorn app.main:app --host 127.0.0.1 --port 8000 --reload
```
API 文档自动生成于 http://localhost:8000/docs
API 文档自动生成于：
- Swagger UI: http://localhost:8000/docs
- ReDoc: http://localhost:8000/redoc
## 架构概览
### 双数据库连接
| 连接 | 数据库 | 用途 | 函数 |
|------|--------|------|------|
| 业务读写 | `zqyy_app` | 用户、队列、调度、执行日志 | `get_connection()` |
| ETL 只读 | `etl_feiqiu` | 数据查看器、游标监控 | `get_etl_readonly_connection(site_id)` |
ETL 只读连接自动设置 `default_transaction_read_only = on` 和 RLS `app.current_site_id`，确保门店数据隔离。
### 认证体系
后端支持两套认证：
1. 管理后台认证（`/api/auth/*`）：用户名 + 密码 → JWT
2. 小程序认证（`/api/xcx-auth/*`）：微信 code → openid → JWT
JWT 令牌分两种：
- 完整令牌：已审批用户，包含 `user_id` + `site_id` + `roles`
- 受限令牌（`limited=True`）：待审批用户，仅允许访问申请相关端点
依赖注入：
- `get_current_user`：要求完整令牌
- `get_current_user_or_limited`：允许受限令牌
### 配置加载
优先级（低 → 高）：根 `.env` → 应用 `.env.local` → 环境变量
关键配置项：
- `DB_HOST` / `DB_PORT` / `DB_USER` / `DB_PASSWORD` / `APP_DB_NAME` — 业务数据库
- `ETL_DB_*` — ETL 数据库（缺省复用业务库参数）
- `JWT_SECRET_KEY` — JWT 签名密钥（生产必须设置）
- `CORS_ORIGINS` — 允许的跨域来源
- `ETL_PROJECT_PATH` — ETL CLI 工作目录
- `WX_APPID` / `WX_SECRET` — 微信小程序配置
- `WX_DEV_MODE` — 开发模式（启用 mock 登录）
- `BUSINESS_DAY_START_HOUR` — 营业日分割点（默认 8 点）
### 后台服务
应用启动时通过 lifespan 拉起两个后台服务：
- `TaskQueue`：任务执行队列（从 `task_queue` 表消费，按 site_id 隔离）
- `Scheduler`：定时调度器（从 `scheduled_tasks` 表读取配置，自动入队）
同时注册触发器 job handler：
- `task_generator`：任务生成器（每日凌晨 4:00）
- `task_expiry_check`：任务过期检查（每小时）
- `recall_completion_check`：召回完成检测（ETL 数据更新后）
- `note_reclassify_backfill`：备注重分类回填（召回完成后）
## 路由总览
| 前缀 | 模块 | 说明 | 认证 |
|------|------|------|------|
| `/api/auth` | `auth.py` | 管理后台登录/刷新令牌 | 无 |
| `/api/xcx-auth` | `xcx_auth.py` | 小程序微信登录/申请/状态/店铺切换 | 部分 |
| `/api/xcx/tasks` | `xcx_tasks.py` | 小程序任务列表/置顶/放弃 | JWT |
| `/api/xcx/notes` | `xcx_notes.py` | 小程序备注 CRUD | JWT |
| `/api/admin/applications` | `admin_applications.py` | 管理端申请审核 | JWT |
| `/api/business-day` | `business_day.py` | 营业日配置查询 | JWT |
| `/api/tasks` | `tasks.py` | 任务注册表/Flow 定义/配置验证 | JWT |
| `/api/execution` | `execution.py` | 任务执行/队列管理/历史/日志 | JWT |
| `/api/schedules` | `schedules.py` | 调度任务 CRUD + 启停 | JWT |
| `/api/db` | `db_viewer.py` | ETL 数据库只读查看器 | JWT |
| `/api/etl-status` | `etl_status.py` | ETL 游标/执行记录监控 | JWT |
| `/api/env-config` | `env_config.py` | 环境变量查看/编辑 | JWT |
| `/api/xcx-test` | `xcx_test.py` | MVP 全链路验证 | 无 |
| `/api/wx-callback` | `wx_callback.py` | 微信消息推送回调 | 签名验证 |
| `/api/retention-clue` | `member_retention_clue.py` | 维客线索 CRUD | JWT |
| `/api/ops` | `ops_panel.py` | 运维面板（服务启停/Git/系统信息） | 无 |
| `/ws/logs` | `ws/logs.py` | WebSocket 实时日志推送 | — |
| `/health` | `main.py` | 健康检查 | 无 |
> 详细 API 端点说明见 [`docs/API-REFERENCE.md`](docs/API-REFERENCE.md)
## 服务层
| 模块 | 职责 |
|------|------|
| `wechat.py` | 微信 `code2session` 接口调用 |
| `matching.py` | 用户申请时的人员匹配（助教/员工） |
| `application.py` | 用户入驻申请的创建/审批/拒绝 |
| `role.py` | 用户权限查询/门店角色检查 |
| `scheduler.py` | 定时调度引擎（cron 解析 + next_run 计算） |
| `task_executor.py` | ETL CLI 子进程执行 + 日志广播 |
| `task_queue.py` | 任务队列管理（入队/消费/重排） |
| `task_registry.py` | ETL 任务/Flow/DWD 表静态注册表 |
| `cli_builder.py` | ETL CLI 命令构建器 |
| `task_generator.py` | 任务生成器（基于 WBI/NCI 指数） |
| `task_manager.py` | 任务管理（置顶/放弃/状态变更） |
| `task_expiry.py` | 任务过期检查与处理 |
| `note_service.py` | 备注服务（CRUD + 星星评分） |
| `note_reclassifier.py` | 备注重分类（召回完成后回填） |
| `recall_detector.py` | 召回完成检测（ETL 数据更新触发） |
| `trigger_scheduler.py` | 触发器调度器（cron/interval/event） |
## 依赖
- fastapi>=0.115, uvicorn[standard]>=0.34
- psycopg2-binary>=2.9, python-dotenv>=1.0
- neozqyy-shared（workspace 引用）
```
fastapi>=0.115
uvicorn[standard]>=0.34
psycopg2-binary>=2.9
python-dotenv>=1.0
python-jose[cryptography]>=3.3
bcrypt>=4.0
psutil>=5.9
neozqyy-shared（workspace 引用）
```
## Roadmap
- [ ] 用户管理与微信登录
- [ ] RBAC 权限中间件
- [ ] RBAC 权限中间件（基于 `auth.role_permissions` 表）
- [ ] 数据看板 API（助教业绩、财务日报、客户分析）
- [ ] 任务审批流 API
- [ ] FDW 数据查询接口（助教业绩、财务日报等）
- [ ] 消息推送（微信模板消息/订阅消息）
- [ ] API 限流与审计日志

1
apps/backend/app/ai/__init__.py Normal file
View File

@@ -0,0 +1 @@
# AI 集成模块：百炼 API 封装、8 个 AI 应用、事件调度、缓存管理

1
apps/backend/app/ai/apps/__init__.py Normal file
View File

@@ -0,0 +1 @@
# AI 应用子模块：app1_chat ~ app8_consolidation

273
apps/backend/app/ai/bailian_client.py Normal file
View File

@@ -0,0 +1,273 @@
"""百炼 API 统一封装层。
使用 openai Python SDK（百炼兼容 OpenAI 协议），提供流式和非流式两种调用模式。
所有 AI 应用通过此客户端统一调用阿里云通义千问。
"""
from __future__ import annotations
import asyncio
import copy
import json
import logging
from datetime import datetime
from typing import Any, AsyncGenerator
import openai
logger = logging.getLogger(__name__)
# ── 异常类 ──────────────────────────────────────────────────────────
class BailianApiError(Exception):
"""百炼 API 调用失败（重试耗尽后）。"""
def __init__(self, message: str, status_code: int | None = None):
super().__init__(message)
self.status_code = status_code
class BailianJsonParseError(Exception):
"""百炼 API 返回的 JSON 解析失败。"""
def __init__(self, message: str, raw_content: str = ""):
super().__init__(message)
self.raw_content = raw_content
class BailianAuthError(BailianApiError):
"""百炼 API Key 无效（HTTP 401）。"""
def __init__(self, message: str = "API Key 无效或已过期"):
super().__init__(message, status_code=401)
# ── 客户端 ──────────────────────────────────────────────────────────
class BailianClient:
"""百炼 API 统一封装层。
使用 openai.AsyncOpenAI 客户端，base_url 指向百炼端点。
提供流式（chat_stream）和非流式（chat_json）两种调用模式。
"""
# 重试配置
MAX_RETRIES = 3
BASE_INTERVAL = 1 # 秒
def __init__(self, api_key: str, base_url: str, model: str):
"""初始化百炼客户端。
Args:
api_key: 百炼 API Key（环境变量 BAILIAN_API_KEY）
base_url: 百炼 API 端点（环境变量 BAILIAN_BASE_URL）
model: 模型标识，如 qwen-plus（环境变量 BAILIAN_MODEL）
"""
self.model = model
self._client = openai.AsyncOpenAI(
api_key=api_key,
base_url=base_url,
)
async def chat_stream(
self,
messages: list[dict],
*,
temperature: float = 0.7,
max_tokens: int = 2000,
) -> AsyncGenerator[str, None]:
"""流式调用，逐 chunk yield 文本。用于应用 1 SSE。
Args:
messages: 消息列表
temperature: 温度参数，默认 0.7
max_tokens: 最大 token 数，默认 2000
Yields:
文本 chunk
"""
messages = self._inject_current_time(messages)
response = await self._call_with_retry(
model=self.model,
messages=messages,
temperature=temperature,
max_tokens=max_tokens,
stream=True,
)
async for chunk in response:
if chunk.choices and chunk.choices[0].delta.content:
yield chunk.choices[0].delta.content
async def chat_json(
self,
messages: list[dict],
*,
temperature: float = 0.3,
max_tokens: int = 4000,
) -> tuple[dict, int]:
"""非流式调用，返回解析后的 JSON dict 和 tokens_used。
用于应用 2-8 的结构化输出。使用 response_format={"type": "json_object"}
确保返回合法 JSON。
Args:
messages: 消息列表
temperature: 温度参数，默认 0.3（结构化输出用低温度）
max_tokens: 最大 token 数，默认 4000
Returns:
(parsed_json_dict, tokens_used) 元组
Raises:
BailianJsonParseError: 响应内容无法解析为 JSON
BailianApiError: API 调用失败（重试耗尽后）
"""
messages = self._inject_current_time(messages)
response = await self._call_with_retry(
model=self.model,
messages=messages,
temperature=temperature,
max_tokens=max_tokens,
stream=False,
response_format={"type": "json_object"},
)
raw_content = response.choices[0].message.content or ""
tokens_used = response.usage.total_tokens if response.usage else 0
try:
parsed = json.loads(raw_content)
except (json.JSONDecodeError, TypeError) as e:
logger.error("百炼 API 返回非法 JSON: %s", raw_content[:500])
raise BailianJsonParseError(
f"JSON 解析失败: {e}",
raw_content=raw_content,
) from e
return parsed, tokens_used
def _inject_current_time(self, messages: list[dict]) -> list[dict]:
"""纯函数：在首条消息的 content（JSON 字符串）中注入 current_time 字段。
- 深拷贝输入，不修改原始 messages
- 首条消息 content 尝试解析为 JSON，注入 current_time
- 如果首条消息 content 不是 JSON，则包装为 JSON
- 其余消息不变
- current_time 格式：ISO 8601 精确到秒，如 2026-03-08T14:30:00
Args:
messages: 原始消息列表
Returns:
注入 current_time 后的新消息列表
"""
if not messages:
return []
result = copy.deepcopy(messages)
first = result[0]
content = first.get("content", "")
now_str = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
try:
parsed = json.loads(content)
if isinstance(parsed, dict):
parsed["current_time"] = now_str
else:
# content 是合法 JSON 但不是 dict（如数组、字符串），包装为 dict
parsed = {"original_content": parsed, "current_time": now_str}
except (json.JSONDecodeError, TypeError):
# content 不是 JSON，包装为 dict
parsed = {"content": content, "current_time": now_str}
first["content"] = json.dumps(parsed, ensure_ascii=False)
return result
async def _call_with_retry(self, **kwargs: Any) -> Any:
"""带指数退避的重试封装。
重试策略：
- 最多重试 MAX_RETRIES 次（默认 3 次）
- 间隔：BASE_INTERVAL × 2^(n-1)，即 1s → 2s → 4s
- HTTP 4xx：不重试，直接抛出（401 → BailianAuthError）
- HTTP 5xx / 超时：重试
Args:
**kwargs: 传递给 openai client 的参数
Returns:
API 响应对象
Raises:
BailianAuthError: API Key 无效（HTTP 401）
BailianApiError: API 调用失败（重试耗尽后）
"""
is_stream = kwargs.get("stream", False)
last_error: Exception | None = None
for attempt in range(self.MAX_RETRIES):
try:
if is_stream:
# 流式调用：返回 async iterator
return await self._client.chat.completions.create(**kwargs)
else:
return await self._client.chat.completions.create(**kwargs)
except openai.AuthenticationError as e:
# 401：API Key 无效，不重试
logger.error("百炼 API 认证失败: %s", e)
raise BailianAuthError(str(e)) from e
except openai.BadRequestError as e:
# 400：请求参数错误，不重试
logger.error("百炼 API 请求参数错误: %s", e)
raise BailianApiError(str(e), status_code=400) from e
except openai.RateLimitError as e:
# 429：限流，不重试（属于 4xx）
logger.error("百炼 API 限流: %s", e)
raise BailianApiError(str(e), status_code=429) from e
except openai.PermissionDeniedError as e:
# 403：权限不足，不重试
logger.error("百炼 API 权限不足: %s", e)
raise BailianApiError(str(e), status_code=403) from e
except openai.NotFoundError as e:
# 404：资源不存在，不重试
logger.error("百炼 API 资源不存在: %s", e)
raise BailianApiError(str(e), status_code=404) from e
except openai.UnprocessableEntityError as e:
# 422：不可处理，不重试
logger.error("百炼 API 不可处理的请求: %s", e)
raise BailianApiError(str(e), status_code=422) from e
except (openai.InternalServerError, openai.APIConnectionError, openai.APITimeoutError) as e:
# 5xx / 超时 / 连接错误：重试
last_error = e
if attempt < self.MAX_RETRIES - 1:
wait_time = self.BASE_INTERVAL * (2 ** attempt)
logger.warning(
"百炼 API 调用失败（第 %d/%d 次），%ds 后重试: %s",
attempt + 1,
self.MAX_RETRIES,
wait_time,
e,
)
await asyncio.sleep(wait_time)
else:
logger.error(
"百炼 API 调用失败，已达最大重试次数 %d: %s",
self.MAX_RETRIES,
e,
)
# 重试耗尽
status_code = getattr(last_error, "status_code", None)
raise BailianApiError(
f"百炼 API 调用失败（重试 {self.MAX_RETRIES} 次后）: {last_error}",
status_code=status_code,
) from last_error

188
apps/backend/app/ai/cache_service.py Normal file
View File

@@ -0,0 +1,188 @@
"""
AI 缓存读写服务。
负责 biz.ai_cache 表的 CRUD 和保留策略管理。
所有查询和写入操作强制 site_id 隔离。
"""
from __future__ import annotations
import json
import logging
from datetime import datetime
from app.database import get_connection
logger = logging.getLogger(__name__)
class AICacheService:
"""AI 缓存读写服务。"""
def get_latest(
self,
cache_type: str,
site_id: int,
target_id: str,
) -> dict | None:
"""查询最新缓存记录。
按 (cache_type, site_id, target_id) 查询 created_at 最新的一条。
无记录时返回 None。
"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, cache_type, site_id, target_id,
result_json, score, triggered_by,
created_at, expires_at
FROM biz.ai_cache
WHERE cache_type = %s AND site_id = %s AND target_id = %s
ORDER BY created_at DESC
LIMIT 1
""",
(cache_type, site_id, target_id),
)
columns = [desc[0] for desc in cur.description]
row = cur.fetchone()
if row is None:
return None
return _row_to_dict(columns, row)
finally:
conn.close()
def get_history(
self,
cache_type: str,
site_id: int,
target_id: str,
limit: int = 2,
) -> list[dict]:
"""查询历史缓存记录（按 created_at DESC），用于 Prompt reference。
无记录时返回空列表。
"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, cache_type, site_id, target_id,
result_json, score, triggered_by,
created_at, expires_at
FROM biz.ai_cache
WHERE cache_type = %s AND site_id = %s AND target_id = %s
ORDER BY created_at DESC
LIMIT %s
""",
(cache_type, site_id, target_id, limit),
)
columns = [desc[0] for desc in cur.description]
rows = cur.fetchall()
return [_row_to_dict(columns, row) for row in rows]
finally:
conn.close()
def write_cache(
self,
cache_type: str,
site_id: int,
target_id: str,
result_json: dict,
triggered_by: str | None = None,
score: int | None = None,
expires_at: datetime | None = None,
) -> int:
"""写入缓存记录，返回 id。写入后清理超限记录。"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO biz.ai_cache
(cache_type, site_id, target_id, result_json,
triggered_by, score, expires_at)
VALUES (%s, %s, %s, %s, %s, %s, %s)
RETURNING id
""",
(
cache_type,
site_id,
target_id,
json.dumps(result_json, ensure_ascii=False),
triggered_by,
score,
expires_at,
),
)
row = cur.fetchone()
conn.commit()
cache_id: int = row[0]
except Exception:
conn.rollback()
raise
finally:
conn.close()
# 写入成功后清理超限记录（失败仅记录警告，不影响写入结果）
try:
deleted = self._cleanup_excess(cache_type, site_id, target_id)
if deleted > 0:
logger.info(
"清理超限缓存: cache_type=%s site_id=%s target_id=%s 删除=%d",
cache_type, site_id, target_id, deleted,
)
except Exception:
logger.warning(
"清理超限缓存失败: cache_type=%s site_id=%s target_id=%s",
cache_type, site_id, target_id,
exc_info=True,
)
return cache_id
def _cleanup_excess(
self,
cache_type: str,
site_id: int,
target_id: str,
max_count: int = 500,
) -> int:
"""清理超限记录，保留最近 max_count 条，返回删除数量。"""
conn = get_connection()
try:
with conn.cursor() as cur:
# 删除超出保留上限的最旧记录
cur.execute(
"""
DELETE FROM biz.ai_cache
WHERE id IN (
SELECT id FROM biz.ai_cache
WHERE cache_type = %s AND site_id = %s AND target_id = %s
ORDER BY created_at DESC
OFFSET %s
)
""",
(cache_type, site_id, target_id, max_count),
)
deleted = cur.rowcount
conn.commit()
return deleted
except Exception:
conn.rollback()
raise
finally:
conn.close()
def _row_to_dict(columns: list[str], row: tuple) -> dict:
"""将数据库行转换为 dict，处理特殊类型序列化。"""
result = {}
for col, val in zip(columns, row):
if isinstance(val, datetime):
result[col] = val.isoformat()
else:
result[col] = val
return result

160
apps/backend/app/ai/conversation_service.py Normal file
View File

@@ -0,0 +1,160 @@
"""
对话记录持久化服务。
负责 biz.ai_conversations 和 biz.ai_messages 两张表的 CRUD。
所有 8 个 AI 应用的每次调用都通过本服务记录对话和消息。
"""
from __future__ import annotations
import json
import logging
from datetime import datetime
from app.database import get_connection
logger = logging.getLogger(__name__)
class ConversationService:
"""AI 对话记录持久化服务。"""
def create_conversation(
self,
user_id: int | str,
nickname: str,
app_id: str,
site_id: int,
source_page: str | None = None,
source_context: dict | None = None,
) -> int:
"""创建对话记录，返回 conversation_id。
系统自动调用时 user_id 为 'system'。
"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO biz.ai_conversations
(user_id, nickname, app_id, site_id, source_page, source_context)
VALUES (%s, %s, %s, %s, %s, %s)
RETURNING id
""",
(
str(user_id),
nickname,
app_id,
site_id,
source_page,
json.dumps(source_context, ensure_ascii=False) if source_context else None,
),
)
row = cur.fetchone()
conn.commit()
return row[0]
except Exception:
conn.rollback()
raise
finally:
conn.close()
def add_message(
self,
conversation_id: int,
role: str,
content: str,
tokens_used: int | None = None,
) -> int:
"""添加消息记录，返回 message_id。"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO biz.ai_messages
(conversation_id, role, content, tokens_used)
VALUES (%s, %s, %s, %s)
RETURNING id
""",
(conversation_id, role, content, tokens_used),
)
row = cur.fetchone()
conn.commit()
return row[0]
except Exception:
conn.rollback()
raise
finally:
conn.close()
def get_conversations(
self,
user_id: int | str,
site_id: int,
page: int = 1,
page_size: int = 20,
) -> list[dict]:
"""查询用户历史对话列表，按 created_at 降序，分页。"""
offset = (page - 1) * page_size
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, user_id, nickname, app_id, site_id,
source_page, source_context, created_at
FROM biz.ai_conversations
WHERE user_id = %s AND site_id = %s
ORDER BY created_at DESC
LIMIT %s OFFSET %s
""",
(str(user_id), site_id, page_size, offset),
)
columns = [desc[0] for desc in cur.description]
rows = cur.fetchall()
return [
_row_to_dict(columns, row)
for row in rows
]
finally:
conn.close()
def get_messages(
self,
conversation_id: int,
) -> list[dict]:
"""查询对话的所有消息，按 created_at 升序。"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, conversation_id, role, content,
tokens_used, created_at
FROM biz.ai_messages
WHERE conversation_id = %s
ORDER BY created_at ASC
""",
(conversation_id,),
)
columns = [desc[0] for desc in cur.description]
rows = cur.fetchall()
return [
_row_to_dict(columns, row)
for row in rows
]
finally:
conn.close()
def _row_to_dict(columns: list[str], row: tuple) -> dict:
"""将数据库行转换为 dict，处理特殊类型序列化。"""
result = {}
for col, val in zip(columns, row):
if isinstance(val, datetime):
result[col] = val.isoformat()
else:
result[col] = val
return result

1
apps/backend/app/ai/prompts/__init__.py Normal file
View File

@@ -0,0 +1 @@
# AI Prompt 模板子模块

157
apps/backend/app/ai/schemas.py Normal file
View File

@@ -0,0 +1,157 @@
"""AI 集成层 Pydantic 模型定义。
覆盖请求/响应体、缓存类型枚举、线索模型、各应用结果模型。
"""
from __future__ import annotations
import enum
from pydantic import BaseModel, Field
# ── 请求/响应 ──
class ChatStreamRequest(BaseModel):
"""SSE 流式对话请求体"""
message: str
source_page: str | None = None
page_context: dict | None = None
screen_content: str | None = None
class SSEEvent(BaseModel):
"""SSE 事件"""
type: str # chunk / done / error
content: str | None = None
conversation_id: int | None = None
tokens_used: int | None = None
message: str | None = None
# ── 缓存类型枚举 ──
class CacheTypeEnum(str, enum.Enum):
APP2_FINANCE = "app2_finance"
APP3_CLUE = "app3_clue"
APP4_ANALYSIS = "app4_analysis"
APP5_TACTICS = "app5_tactics"
APP6_NOTE_ANALYSIS = "app6_note_analysis"
APP7_CUSTOMER_ANALYSIS = "app7_customer_analysis"
APP8_CLUE_CONSOLIDATED = "app8_clue_consolidated"
# ── 线索相关 ──
class App3CategoryEnum(str, enum.Enum):
"""App3 线索分类（3 个枚举值）"""
CUSTOMER_BASIC = "客户基础"
CONSUMPTION_HABIT = "消费习惯"
PLAY_PREFERENCE = "玩法偏好"
class App6CategoryEnum(str, enum.Enum):
"""App6/8 线索分类（6 个枚举值）"""
CUSTOMER_BASIC = "客户基础"
CONSUMPTION_HABIT = "消费习惯"
PLAY_PREFERENCE = "玩法偏好"
PROMO_PREFERENCE = "促销偏好"
SOCIAL_RELATION = "社交关系"
IMPORTANT_FEEDBACK = "重要反馈"
class ClueItem(BaseModel):
"""单条线索（App3/App6 共用）"""
category: str
summary: str
detail: str
emoji: str
class ConsolidatedClueItem(BaseModel):
"""整合后线索（App8，含 providers）"""
category: str
summary: str
detail: str
emoji: str
providers: str
# ── 各应用结果模型 ──
class App2InsightItem(BaseModel):
"""App2 财务洞察单条"""
seq: int
title: str
body: str
class App2Result(BaseModel):
"""App2 财务洞察结果"""
insights: list[App2InsightItem]
class App3Result(BaseModel):
"""App3 客户数据维客线索结果"""
clues: list[ClueItem]
class App4Result(BaseModel):
"""App4 关系分析结果"""
task_description: str
action_suggestions: list[str]
one_line_summary: str
class App5TacticsItem(BaseModel):
"""App5 话术单条"""
scenario: str
script: str
class App5Result(BaseModel):
"""App5 话术参考结果"""
tactics: list[App5TacticsItem]
class App6Result(BaseModel):
"""App6 备注分析结果"""
score: int = Field(ge=1, le=10)
clues: list[ClueItem]
class App7StrategyItem(BaseModel):
"""App7 客户分析策略单条"""
title: str
content: str
class App7Result(BaseModel):
"""App7 客户分析结果"""
strategies: list[App7StrategyItem]
summary: str
class App8Result(BaseModel):
"""App8 维客线索整理结果"""
clues: list[ConsolidatedClueItem]

133
apps/backend/app/config.py
View File

@@ -10,12 +10,79 @@ from pathlib import Path
from dotenv import load_dotenv
# CHANGE 2026-03-07 | 项目根目录定位：防止 junction/symlink 穿透到 D 盘
# 背景：C:\NeoZQYY 是 junction → D:\NeoZQYY\...\repo，
# Path(__file__).resolve() 和 absolute() 都可能解析到 D 盘，
# 导致加载 D 盘的 .env（路径全指向 D 盘），ETL 命令因此携带错误路径。
# 策略：环境变量 > 已知固定路径 > __file__ 推算（最后手段）
import logging as _logging
_cfg_logger = _logging.getLogger("app.config")
def _find_project_root() -> Path:
"""定位项目根目录，返回包含 .env 的路径。
优先级：
1. 环境变量 NEOZQYY_ROOT（最可靠，显式指定）
2. __file__ 向上推算，但用 junction 安全的方式
"""
# 1. 环境变量显式指定（部署时设置，最可靠）
env_root = os.environ.get("NEOZQYY_ROOT")
if env_root:
p = Path(env_root)
if (p / ".env").exists():
_cfg_logger.info("[ROOT] 策略1命中: NEOZQYY_ROOT=%s", p)
return p
_cfg_logger.warning("[ROOT] NEOZQYY_ROOT=%s 但 .env 不存在，跳过", env_root)
# 2. 从 __file__ 推算（apps/backend/app/config.py → 上 3 级）
raw_file = Path(__file__)
abs_file = raw_file.absolute()
candidate = abs_file.parents[3]
_cfg_logger.info(
"[ROOT] 策略2: __file__=%s | absolute=%s | candidate=%s",
raw_file, abs_file, candidate,
)
# CHANGE 2026-03-07 | 防护：如果推算路径包含 test/repo 或 prod/repo 等
# 多环境子目录，说明发生了 junction/symlink 穿透到 D 盘部署结构，
# 此时向上搜索找到真正的项目根（包含 .env 的最浅目录）
candidate_str = str(candidate)
if any(seg in candidate_str for seg in ("\\test\\", "\\prod\\", "/test/", "/prod/")):
_cfg_logger.warning(
"[ROOT] 检测到多环境子目录穿透: %s，启动向上搜索", candidate_str
)
elif (candidate / ".env").exists():
_cfg_logger.info("[ROOT] 策略2命中: %s", candidate)
return candidate
# 3. 向上搜索：应对 junction 穿透导致层级偏移的情况
cur = abs_file.parent
for i in range(10):
if (cur / ".env").exists():
_cfg_logger.info("[ROOT] 策略3命中(第%d级): %s", i, cur)
return cur
parent = cur.parent
if parent == cur:
break
cur = parent
_cfg_logger.warning("[ROOT] 所有策略均未命中，回退到: %s", candidate)
return candidate
_project_root = _find_project_root()
_cfg_logger.info("项目根目录: %s", _project_root)
# 根 .env（公共配置）
_root_env = Path(__file__).resolve().parents[3] / ".env"
_root_env = _project_root / ".env"
_cfg_logger.info("加载根 .env: %s (存在: %s)", _root_env, _root_env.exists())
load_dotenv(_root_env, override=False)
# 应用级 .env.local（私有覆盖，优先级更高）
_local_env = Path(__file__).resolve().parents[1] / ".env.local"
_local_env = _project_root / "apps" / "backend" / ".env.local"
load_dotenv(_local_env, override=True)
@@ -24,6 +91,14 @@ def get(key: str, default: str | None = None) -> str | None:
return os.getenv(key, default)
def _require_env(key: str) -> str:
"""必需的环境变量，缺失时立即报错。"""
raise RuntimeError(
f"必需的环境变量 {key} 未设置。"
f"请在 .env 中显式配置（当前 .env 路径: {_root_env}）"
)
# ---- 数据库连接参数 ----
DB_HOST: str = get("DB_HOST", "localhost")
DB_PORT: str = get("DB_PORT", "5432")
@@ -55,11 +130,55 @@ CORS_ORIGINS: list[str] = [
]
# ---- ETL 项目路径 ----
# ETL CLI 的工作目录（子进程 cwd），缺省时按 monorepo 相对路径推算
ETL_PROJECT_PATH: str = get(
"ETL_PROJECT_PATH",
str(Path(__file__).resolve().parents[3] / "apps" / "etl" / "connectors" / "feiqiu"),
)
# CHANGE 2026-03-06 | 必须在 .env 显式设置，禁止依赖 __file__ 推算
# 原因：__file__ 推算依赖 uvicorn 启动位置，不同部署环境会指向错误代码副本
ETL_PROJECT_PATH: str = get("ETL_PROJECT_PATH") or _require_env("ETL_PROJECT_PATH")
# ETL 子进程 Python 可执行路径
# CHANGE 2026-03-06 | 必须在 .env 显式设置，避免 PATH 歧义
ETL_PYTHON_EXECUTABLE: str = get("ETL_PYTHON_EXECUTABLE") or _require_env("ETL_PYTHON_EXECUTABLE")
# ---- 运维面板 ----
# CHANGE 2026-03-06 | 必须在 .env 显式设置，消除 __file__ 推算风险
OPS_SERVER_BASE: str = get("OPS_SERVER_BASE") or _require_env("OPS_SERVER_BASE")
# CHANGE 2026-03-07 | 启动时验证关键路径：
# 1. 路径必须实际存在于文件系统
# 2. 路径不得包含多环境子目录（test/repo、prod/repo），这是 junction 穿透的标志
_cfg_logger.info("ETL_PROJECT_PATH = %s", ETL_PROJECT_PATH)
_cfg_logger.info("ETL_PYTHON_EXECUTABLE = %s", ETL_PYTHON_EXECUTABLE)
_cfg_logger.info("OPS_SERVER_BASE = %s", OPS_SERVER_BASE)
for _var_name, _var_val in [
("ETL_PROJECT_PATH", ETL_PROJECT_PATH),
("ETL_PYTHON_EXECUTABLE", ETL_PYTHON_EXECUTABLE),
("OPS_SERVER_BASE", OPS_SERVER_BASE),
]:
# 检测 junction 穿透特征：路径中包含 \test\repo 或 \prod\repo
_normalized = _var_val.replace("/", "\\")
if "\\test\\repo" in _normalized or "\\prod\\repo" in _normalized:
_cfg_logger.error(
"路径穿透检测: %s=%s 包含多环境子目录，"
"说明 .env 来自 junction 穿透后的 D 盘副本。"
"当前 .env 路径: %s | NEOZQYY_ROOT: %s",
_var_name, _var_val, _root_env,
os.environ.get("NEOZQYY_ROOT", "<未设置>"),
)
raise RuntimeError(
f"配置路径异常: {_var_name}={_var_val} 包含多环境子目录"
f"（test/repo 或 prod/repo），疑似加载了错误的 .env。"
f" 当前 .env: {_root_env}。"
f" 请确保 NEOZQYY_ROOT 环境变量指向正确的项目根目录。"
)
# ---- 微信小程序 ----
WX_APPID: str = get("WX_APPID", "")
WX_SECRET: str = get("WX_SECRET", "")
# 开发模式：WX_DEV_MODE=true 时启用 mock 登录端点，跳过微信 code2Session
WX_DEV_MODE: bool = get("WX_DEV_MODE", "false").lower() in ("true", "1", "yes")
# ---- 营业日分割点 ----
BUSINESS_DAY_START_HOUR: int = int(get("BUSINESS_DAY_START_HOUR", "8"))
# ---- 通用 ----
TIMEZONE: str = get("TIMEZONE", "Asia/Shanghai")

71
apps/backend/app/main.py
View File

@@ -12,10 +12,12 @@ from fastapi.middleware.cors import CORSMiddleware
from app import config
# CHANGE 2026-02-19 | 新增 xcx_test 路由（MVP 验证）+ wx_callback 路由（微信消息推送）
# CHANGE 2026-02-22 | 新增 member_birthday 路由（助教手动补录会员生日）
# CHANGE 2026-02-23 | 新增 ops_panel 路由（运维控制面板）
# CHANGE 2026-02-25 | 新增 xcx_auth 路由（小程序微信登录 + 申请 + 状态查询 + 店铺切换）
from app.routers import auth, execution, schedules, tasks, env_config, db_viewer, etl_status, xcx_test, wx_callback, member_birthday, ops_panel, xcx_auth
# CHANGE 2026-02-26 | member_birthday 路由替换为 member_retention_clue（维客线索重构）
# CHANGE 2026-02-26 | 新增 admin_applications 路由（管理端申请审核）
# CHANGE 2026-02-27 | 新增 xcx_tasks / xcx_notes 路由（小程序核心业务）
from app.routers import auth, execution, schedules, tasks, env_config, db_viewer, etl_status, xcx_test, wx_callback, member_retention_clue, ops_panel, xcx_auth, admin_applications, business_day, xcx_tasks, xcx_notes
from app.services.scheduler import scheduler
from app.services.task_queue import task_queue
from app.ws.logs import ws_router
@@ -24,9 +26,37 @@ from app.ws.logs import ws_router
@asynccontextmanager
async def lifespan(app: FastAPI):
"""应用生命周期：启动时拉起后台服务，关闭时优雅停止。"""
# CHANGE 2026-03-07 | 启动横幅：打印关键路径，便于诊断连到了哪个实例
import sys
_banner = (
"\n"
"╔══════════════════════════════════════════════════════╗\n"
"║ NeoZQYY Backend — 启动诊断 ║\n"
"╠══════════════════════════════════════════════════════╣\n"
f"║ Python: {sys.executable}\n"
f"║ ROOT: {config._project_root}\n"
f"║ ETL_PATH: {config.ETL_PROJECT_PATH}\n"
f"║ ETL_PY: {config.ETL_PYTHON_EXECUTABLE}\n"
f"║ OPS_BASE: {config.OPS_SERVER_BASE}\n"
f"║ APP_DB: {config.APP_DB_NAME}\n"
f"║ .env: {config._root_env}\n"
"╚══════════════════════════════════════════════════════╝\n"
)
print(_banner, flush=True)
# 启动
task_queue.start()
scheduler.start()
# CHANGE 2026-02-27 | 注册触发器 job handler（核心业务模块）
from app.services.trigger_scheduler import register_job
from app.services import task_generator, task_expiry, recall_detector, note_reclassifier
register_job("task_generator", lambda **_kw: task_generator.run())
register_job("task_expiry_check", lambda **_kw: task_expiry.run())
register_job("recall_completion_check", recall_detector.run)
register_job("note_reclassify_backfill", note_reclassifier.run)
yield
# 关闭
await scheduler.stop()
@@ -63,12 +93,47 @@ app.include_router(etl_status.router)
app.include_router(ws_router)
app.include_router(xcx_test.router)
app.include_router(wx_callback.router)
app.include_router(member_birthday.router)
app.include_router(member_retention_clue.router)
app.include_router(ops_panel.router)
app.include_router(xcx_auth.router)
app.include_router(admin_applications.router)
app.include_router(business_day.router)
app.include_router(xcx_tasks.router)
app.include_router(xcx_notes.router)
@app.get("/health", tags=["系统"])
async def health_check():
"""健康检查端点，用于探活和监控。"""
return {"status": "ok"}
# CHANGE 2026-03-07 | 诊断端点：返回关键路径配置，用于确认连到的是哪个实例
@app.get("/debug/config-paths", tags=["系统"])
async def debug_config_paths():
"""返回当前后端实例的关键路径配置（仅开发环境使用）。"""
import sys
import os
import platform
from app.services.cli_builder import cli_builder as _cb
from app.schemas.tasks import TaskConfigSchema as _TCS
_test_cfg = _TCS(flow="api_ods_dwd", processing_mode="increment_only",
tasks=["DWD_LOAD_FROM_ODS"], store_id=123)
_test_cmd = _cb.build_command(
_test_cfg, config.ETL_PROJECT_PATH,
python_executable=config.ETL_PYTHON_EXECUTABLE,
)
_test_cmd_str = " ".join(_test_cmd)
return {
"hostname": platform.node(),
"python_executable": sys.executable,
"project_root": str(config._project_root),
"env_file": str(config._root_env),
"etl_python_executable": config.ETL_PYTHON_EXECUTABLE,
"etl_project_path": config.ETL_PROJECT_PATH,
"simulated_command": _test_cmd_str,
"NEOZQYY_ROOT_env": os.environ.get("NEOZQYY_ROOT", "<未设置>"),
"cwd": os.getcwd(),
}

124
apps/backend/app/middleware/permission.py Normal file
View File

@@ -0,0 +1,124 @@
# -*- coding: utf-8 -*-
"""
权限中间件 —— 基于 FastAPI 依赖注入的权限检查。
提供两个依赖工厂：
- require_permission(*codes)：检查用户 status=approved 且拥有指定权限
- require_approved()：仅检查用户 status=approved（不检查具体权限）
用法：
@router.get("/finance")
async def get_finance(
user: CurrentUser = Depends(require_permission("view_board_finance"))
):
...
@router.get("/tasks")
async def get_tasks(
user: CurrentUser = Depends(require_approved())
):
...
"""
from __future__ import annotations
import logging
from fastapi import Depends, HTTPException, status
from app.auth.dependencies import CurrentUser, get_current_user
from app.database import get_connection
from app.services.role import get_user_permissions
logger = logging.getLogger(__name__)
def _get_user_status(user_id: int) -> str | None:
"""从数据库查询用户当前 status。返回 None 表示用户不存在。"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"SELECT status FROM auth.users WHERE id = %s",
(user_id,),
)
row = cur.fetchone()
finally:
conn.close()
return row[0] if row else None
def require_permission(*permission_codes: str):
"""
权限依赖工厂：要求用户 status=approved 且拥有全部指定权限。
流程：
1. 通过 get_current_user 从 JWT 提取 user_id + site_id
2. 查询 auth.users.status —— 非 approved 则 403
3. 查询 user_site_roles + role_permissions 获取权限列表
4. 检查所需权限是否全部在列表中 —— 缺失则 403
5. 返回 CurrentUser 对象
"""
async def _dependency(
user: CurrentUser = Depends(get_current_user),
) -> CurrentUser:
# 查询数据库中的实时 status
db_status = _get_user_status(user.user_id)
if db_status is None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="用户不存在",
)
if db_status != "approved":
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="用户未通过审核，无法访问此资源",
)
# 检查具体权限
if permission_codes:
user_perms = await get_user_permissions(user.user_id, user.site_id)
missing = set(permission_codes) - set(user_perms)
if missing:
logger.warning(
"用户 %s 在 site_id=%s 下缺少权限: %s",
user.user_id,
user.site_id,
missing,
)
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="权限不足",
)
return user
return _dependency
def require_approved():
"""
审核状态依赖工厂：仅检查用户 status=approved，不检查具体权限。
用于通用的已认证端点，只需确认用户已通过审核即可访问。
"""
async def _dependency(
user: CurrentUser = Depends(get_current_user),
) -> CurrentUser:
db_status = _get_user_status(user.user_id)
if db_status is None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="用户不存在",
)
if db_status != "approved":
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="用户未通过审核，无法访问此资源",
)
return user
return _dependency

214
apps/backend/app/routers/admin_applications.py Normal file
View File

@@ -0,0 +1,214 @@
# -*- coding: utf-8 -*-
"""
管理端审核路由 —— 申请列表、详情、批准、拒绝。
端点清单：
- GET /api/admin/applications — 查询申请列表（可按 status 过滤）
- GET /api/admin/applications/{id} — 查询申请详情 + 候选匹配
- POST /api/admin/applications/{id}/approve — 批准申请
- POST /api/admin/applications/{id}/reject — 拒绝申请
所有端点要求 JWT + site_admin 或 tenant_admin 角色。
"""
from __future__ import annotations
import logging
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query, status
from app.auth.dependencies import CurrentUser
from app.database import get_connection
from app.middleware.permission import require_permission
from app.schemas.xcx_auth import (
ApplicationResponse,
ApproveRequest,
MatchCandidate,
RejectRequest,
)
from app.services.application import approve_application, reject_application
from app.services.matching import find_candidates
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/admin", tags=["管理端审核"])
# ── 管理端需要 site_admin 或 tenant_admin 权限 ─────────────
# require_permission() 不检查具体 permission code，
# 但会验证 status=approved；管理端路由额外在依赖中检查角色。
# 设计文档要求 site_admin / tenant_admin 角色，
# 这里通过检查 CurrentUser.roles 实现。
def _require_admin():
"""
管理端依赖：要求用户 status=approved 且角色包含 site_admin 或 tenant_admin。
复用 require_permission()（无具体权限码 → 仅检查 approved），
再额外校验角色列表。
"""
async def _dependency(
user: CurrentUser = Depends(require_permission()),
) -> CurrentUser:
admin_roles = {"site_admin", "tenant_admin"}
if not admin_roles.intersection(user.roles):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="需要管理员权限（site_admin 或 tenant_admin）",
)
return user
return _dependency
# ── GET /api/admin/applications ───────────────────────────
@router.get("/applications", response_model=list[ApplicationResponse])
async def list_applications(
status_filter: Optional[str] = Query(
None, alias="status", description="按状态过滤：pending / approved / rejected"
),
user: CurrentUser = Depends(_require_admin()),
):
"""查询申请列表，可按 status 过滤。"""
conn = get_connection()
try:
with conn.cursor() as cur:
if status_filter:
cur.execute(
"""
SELECT id, site_code, applied_role_text, status,
review_note, created_at::text, reviewed_at::text
FROM auth.user_applications
WHERE status = %s
ORDER BY created_at DESC
""",
(status_filter,),
)
else:
cur.execute(
"""
SELECT id, site_code, applied_role_text, status,
review_note, created_at::text, reviewed_at::text
FROM auth.user_applications
ORDER BY created_at DESC
"""
)
rows = cur.fetchall()
finally:
conn.close()
return [
ApplicationResponse(
id=r[0],
site_code=r[1],
applied_role_text=r[2],
status=r[3],
review_note=r[4],
created_at=r[5],
reviewed_at=r[6],
)
for r in rows
]
# ── GET /api/admin/applications/{id} ─────────────────────
@router.get("/applications/{application_id}")
async def get_application_detail(
application_id: int,
user: CurrentUser = Depends(_require_admin()),
):
"""查询申请详情 + 候选匹配。"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, user_id, site_code, site_id, applied_role_text,
phone, employee_number, status,
review_note, created_at::text, reviewed_at::text
FROM auth.user_applications
WHERE id = %s
""",
(application_id,),
)
row = cur.fetchone()
finally:
conn.close()
if row is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="申请不存在",
)
app_data = {
"id": row[0],
"user_id": row[1],
"site_code": row[2],
"site_id": row[3],
"applied_role_text": row[4],
"phone": row[5],
"employee_number": row[6],
"status": row[7],
"review_note": row[8],
"created_at": row[9],
"reviewed_at": row[10],
}
# 查找候选匹配
candidates_raw = await find_candidates(
site_id=app_data["site_id"],
phone=app_data["phone"],
employee_number=app_data["employee_number"],
)
candidates = [MatchCandidate(**c) for c in candidates_raw]
return {
"application": app_data,
"candidates": [c.model_dump() for c in candidates],
}
# ── POST /api/admin/applications/{id}/approve ────────────
@router.post("/applications/{application_id}/approve", response_model=ApplicationResponse)
async def approve(
application_id: int,
body: ApproveRequest,
user: CurrentUser = Depends(_require_admin()),
):
"""批准申请：分配角色 + 可选绑定。"""
result = await approve_application(
application_id=application_id,
reviewer_id=user.user_id,
role_id=body.role_id,
binding=body.binding,
review_note=body.review_note,
)
return ApplicationResponse(**result)
# ── POST /api/admin/applications/{id}/reject ─────────────
@router.post("/applications/{application_id}/reject", response_model=ApplicationResponse)
async def reject(
application_id: int,
body: RejectRequest,
user: CurrentUser = Depends(_require_admin()),
):
"""拒绝申请：记录拒绝原因。"""
result = await reject_application(
application_id=application_id,
reviewer_id=user.user_id,
review_note=body.review_note,
)
return ApplicationResponse(**result)

20
apps/backend/app/routers/business_day.py Normal file
View File

@@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
"""营业日配置 API
提供公开端点返回当前营业日分割点配置，供前端动态获取。
"""
from fastapi import APIRouter
from app import config
router = APIRouter(prefix="/api/config", tags=["业务配置"])
@router.get("/business-day")
async def get_business_day_config():
"""返回当前营业日分割点配置。
无需认证（公开配置），前端启动时调用一次缓存。
"""
return {"business_day_start_hour": config.BUSINESS_DAY_START_HOUR}

4
apps/backend/app/routers/execution.py
View File

@@ -202,7 +202,8 @@ async def get_execution_history(
cur.execute(
"""
SELECT id, site_id, task_codes, status, started_at,
finished_at, exit_code, duration_ms, command, summary
finished_at, exit_code, duration_ms, command, summary,
schedule_id
FROM task_execution_log
WHERE site_id = %s
ORDER BY started_at DESC
@@ -227,6 +228,7 @@ async def get_execution_history(
duration_ms=row[7],
command=row[8],
summary=row[9],
schedule_id=str(row[10]) if row[10] else None,
)
for row in rows
]

102
apps/backend/app/routers/member_retention_clue.py Normal file
View File

@@ -0,0 +1,102 @@
"""
维客线索路由。
- POST /api/retention-clue — 提交维客线索（UPSERT）
- GET /api/retention-clue/{member_id} — 查询某会员的全部维客线索
- DELETE /api/retention-clue/{clue_id} — 删除单条线索
"""
import logging
from fastapi import APIRouter, HTTPException, status
from app.database import get_connection
from app.schemas.member_retention_clue import RetentionClueSubmit, RetentionClueOut
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api", tags=["维客线索"])
@router.post("/retention-clue")
async def submit_retention_clue(body: RetentionClueSubmit):
"""
提交维客线索（INSERT）。
同一会员可有多条不同大类的线索。
"""
sql = """
INSERT INTO member_retention_clue
(member_id, category, summary, detail,
recorded_by_assistant_id, recorded_by_name, site_id, source)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id
"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(sql, (
body.member_id,
body.category.value,
body.summary,
body.detail,
body.recorded_by_assistant_id,
body.recorded_by_name,
body.site_id,
body.source.value,
))
row = cur.fetchone()
conn.commit()
return {"status": "ok", "id": row[0] if row else None}
except Exception:
conn.rollback()
logger.exception("维客线索写入失败: member_id=%s", body.member_id)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="线索提交失败，请稍后重试",
)
finally:
conn.close()
@router.get("/retention-clue/{member_id}", response_model=list[RetentionClueOut])
async def get_retention_clues(member_id: int, site_id: int):
"""查询某会员的全部维客线索，按录入时间倒序。"""
sql = """
SELECT id, member_id, category, summary, detail,
recorded_by_assistant_id, recorded_by_name, recorded_at, site_id, source
FROM member_retention_clue
WHERE member_id = %s AND site_id = %s
ORDER BY recorded_at DESC
"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(sql, (member_id, site_id))
rows = cur.fetchall()
cols = [d[0] for d in cur.description]
return [dict(zip(cols, r)) for r in rows]
finally:
conn.close()
@router.delete("/retention-clue/{clue_id}")
async def delete_retention_clue(clue_id: int):
"""删除单条维客线索。"""
sql = "DELETE FROM member_retention_clue WHERE id = %s"
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(sql, (clue_id,))
if cur.rowcount == 0:
raise HTTPException(status_code=404, detail="线索不存在")
conn.commit()
return {"status": "ok"}
except HTTPException:
raise
except Exception:
conn.rollback()
logger.exception("维客线索删除失败: id=%s", clue_id)
raise HTTPException(status_code=500, detail="删除失败")
finally:
conn.close()

7
apps/backend/app/routers/ops_panel.py
View File

@@ -17,12 +17,13 @@ import psutil
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from ..config import OPS_SERVER_BASE
router = APIRouter(prefix="/api/ops", tags=["运维面板"])
# ---- 环境定义 ----
# 服务器上的两套环境；开发机上回退到本机路径（方便调试）
_SERVER_BASE = Path("D:/NeoZQYY")
# CHANGE 2026-03-04 | 从 config 读取，消除硬编码 D 盘路径
_SERVER_BASE = Path(OPS_SERVER_BASE)
ENVIRONMENTS: dict[str, dict[str, Any]] = {
"test": {

111
apps/backend/app/routers/schedules.py
View File

@@ -1,12 +1,14 @@
# -*- coding: utf-8 -*-
"""调度任务 CRUD API
提供 5 个端点：
提供 8 个端点：
- GET /api/schedules — 列表（按 site_id 过滤）
- POST /api/schedules — 创建
- POST /api/schedules — 创建（支持 run_immediately）
- PUT /api/schedules/{id} — 更新
- DELETE /api/schedules/{id} — 删除
- PATCH /api/schedules/{id}/toggle — 启用/禁用
- GET /api/schedules/{id}/history — 调度任务执行历史
- POST /api/schedules/{id}/run — 手动执行一次（不更新调度间隔）
所有端点需要 JWT 认证，site_id 从 JWT 提取。
"""
@@ -17,7 +19,7 @@ import json
import logging
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi import APIRouter, Depends, HTTPException, Query, status
from app.auth.dependencies import CurrentUser, get_current_user
from app.database import get_connection
@@ -26,7 +28,10 @@ from app.schemas.schedules import (
ScheduleResponse,
UpdateScheduleRequest,
)
from app.schemas.execution import ExecutionHistoryItem
from app.schemas.tasks import TaskConfigSchema
from app.services.scheduler import calculate_next_run
from app.services.task_queue import task_queue
logger = logging.getLogger(__name__)
@@ -92,7 +97,7 @@ async def create_schedule(
body: CreateScheduleRequest,
user: CurrentUser = Depends(get_current_user),
) -> ScheduleResponse:
"""创建调度任务，自动计算 next_run_at。"""
"""创建调度任务，自动计算 next_run_at。支持 run_immediately 立即入队执行一次。"""
now = datetime.now(timezone.utc)
next_run = calculate_next_run(body.schedule_config, now)
@@ -124,7 +129,18 @@ async def create_schedule(
finally:
conn.close()
return _row_to_response(row)
response = _row_to_response(row)
# 立即执行一次（入队，不影响调度间隔）
if body.run_immediately:
try:
config = TaskConfigSchema(**body.task_config)
config = config.model_copy(update={"store_id": user.site_id})
task_queue.enqueue(config, user.site_id, schedule_id=response.id)
except Exception:
logger.warning("创建调度后立即执行入队失败 schedule_id=%s", response.id, exc_info=True)
return response
# ── PUT /api/schedules/{id} — 更新 ──────────────────────────
@@ -291,3 +307,88 @@ async def toggle_schedule(
conn.close()
return _row_to_response(updated_row)
# ── POST /api/schedules/{id}/run — 手动执行一次 ──────────────
@router.post("/{schedule_id}/run")
async def run_schedule_now(
schedule_id: str,
user: CurrentUser = Depends(get_current_user),
) -> dict:
"""手动触发调度任务执行一次，不更新 last_run_at / next_run_at / run_count。
读取调度任务的 task_config，构造 TaskConfigSchema 后入队执行。
"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"SELECT task_config, site_id FROM scheduled_tasks WHERE id = %s AND site_id = %s",
(schedule_id, user.site_id),
)
row = cur.fetchone()
conn.commit()
finally:
conn.close()
if row is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="调度任务不存在",
)
task_config_raw = row[0] if isinstance(row[0], dict) else json.loads(row[0])
config = TaskConfigSchema(**task_config_raw)
config = config.model_copy(update={"store_id": user.site_id})
task_id = task_queue.enqueue(config, user.site_id, schedule_id=schedule_id)
return {"message": "已提交到执行队列", "task_id": task_id}
# ── GET /api/schedules/{id}/history — 执行历史 ────────────────
@router.get("/{schedule_id}/history", response_model=list[ExecutionHistoryItem])
async def get_schedule_history(
schedule_id: str,
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=200),
user: CurrentUser = Depends(get_current_user),
) -> list[ExecutionHistoryItem]:
"""获取调度任务的执行历史记录，按开始时间倒序，支持分页。"""
offset = (page - 1) * page_size
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, site_id, task_codes, status, started_at, finished_at,
exit_code, duration_ms, command, summary, schedule_id
FROM task_execution_log
WHERE schedule_id = %s AND site_id = %s
ORDER BY started_at DESC
LIMIT %s OFFSET %s
""",
(schedule_id, user.site_id, page_size, offset),
)
rows = cur.fetchall()
conn.commit()
finally:
conn.close()
return [
ExecutionHistoryItem(
id=str(row[0]),
site_id=row[1],
task_codes=row[2] or [],
status=row[3],
started_at=row[4],
finished_at=row[5],
exit_code=row[6],
duration_ms=row[7],
command=row[8],
summary=row[9],
schedule_id=str(row[10]) if row[10] else None,
)
for row in rows
]

12
apps/backend/app/routers/tasks.py
View File

@@ -236,6 +236,16 @@ async def sync_check(
backend_codes = {t.code for t in ALL_TASKS}
# ETL 侧存在但后端故意不注册的任务（一次性初始化 / 尚未上线）
# 从 etl_only 差集中排除，避免同步检查误报
ETL_ONLY_EXPECTED: set[str] = {
"INIT_ODS_SCHEMA", # 一次性：ODS schema 初始化
"INIT_DWD_SCHEMA", # 一次性：DWD schema 初始化
"INIT_DWS_SCHEMA", # 一次性：DWS schema 初始化
"SEED_DWS_CONFIG", # 一次性：DWS 配置种子数据
"DWS_ASSISTANT_ORDER_CONTRIBUTION", # DWS 任务，后端暂未注册
}
try:
result = subprocess.run(
[sys.executable, "-c",
@@ -257,7 +267,7 @@ async def sync_check(
)
backend_only = sorted(backend_codes - etl_codes)
etl_only = sorted(etl_codes - backend_codes)
etl_only = sorted((etl_codes - backend_codes) - ETL_ONLY_EXPECTED)
return SyncCheckResponse(
in_sync=len(backend_only) == 0 and len(etl_only) == 0,

352
apps/backend/app/routers/xcx_auth.py
View File

@@ -9,6 +9,11 @@
- GET /api/xcx/me/sites — 查询关联店铺
- POST /api/xcx/switch-site — 切换当前店铺
- POST /api/xcx/refresh — 刷新令牌
- POST /api/xcx/dev-login — 开发模式 mock 登录（仅 WX_DEV_MODE=true）
- POST /api/xcx/dev-switch-role — 切换角色（仅 WX_DEV_MODE=true）
- POST /api/xcx/dev-switch-status — 切换用户状态（仅 WX_DEV_MODE=true）
- POST /api/xcx/dev-switch-binding — 切换人员绑定（仅 WX_DEV_MODE=true）
- GET /api/xcx/dev-context — 查询调试上下文（仅 WX_DEV_MODE=true）
"""
from __future__ import annotations
@@ -29,6 +34,7 @@ from app.auth.jwt import (
create_token_pair,
decode_refresh_token,
)
from app import config
from app.database import get_connection
from app.services.application import (
create_application,
@@ -37,6 +43,11 @@ from app.services.application import (
from app.schemas.xcx_auth import (
ApplicationRequest,
ApplicationResponse,
DevLoginRequest,
DevSwitchBindingRequest,
DevSwitchRoleRequest,
DevSwitchStatusRequest,
DevContextResponse,
RefreshTokenRequest,
SiteInfo,
SwitchSiteRequest,
@@ -45,6 +56,7 @@ from app.schemas.xcx_auth import (
WxLoginResponse,
)
from app.services.wechat import WeChatAuthError, code2session
from app.services.role import get_user_permissions
logger = logging.getLogger(__name__)
@@ -94,9 +106,9 @@ async def wx_login(body: WxLoginRequest):
流程：code → code2session(openid) → 查找/创建 auth.users → 签发 JWT。
- disabled 用户返回 403
- 新用户自动创建（status=pending）
- 新用户自动创建（status=new），前端引导至申请页
- approved 用户签发包含 site_id + roles 的完整令牌
- pending/rejected 用户签发受限令牌
- new/pending/rejected 用户签发受限令牌
"""
# 1. 调用微信 code2Session
try:
@@ -125,12 +137,12 @@ async def wx_login(body: WxLoginRequest):
row = cur.fetchone()
if row is None:
# 新用户：创建 pending 记录
# 新用户：创建 new 记录（尚未提交申请）
try:
cur.execute(
"""
INSERT INTO auth.users (wx_openid, wx_union_id, status)
VALUES (%s, %s, 'pending')
VALUES (%s, %s, 'new')
RETURNING id, status
""",
(openid, unionid),
@@ -166,7 +178,7 @@ async def wx_login(body: WxLoginRequest):
# approved 但无 site 绑定（异常边界），签发受限令牌
tokens = create_limited_token_pair(user_id)
else:
# pending / rejected → 受限令牌
# new / pending / rejected → 受限令牌
tokens = create_limited_token_pair(user_id)
finally:
@@ -415,3 +427,333 @@ async def refresh_token(body: RefreshTokenRequest):
user_status=user_status,
user_id=user_id,
)
# ── POST /api/xcx/dev-login（仅开发模式） ─────────────────
if config.WX_DEV_MODE:
@router.post("/dev-login", response_model=WxLoginResponse)
async def dev_login(body: DevLoginRequest):
"""
开发模式 mock 登录。
直接根据 openid 查找/创建用户，跳过微信 code2Session。
- 已有用户：status 参数为空时保留当前状态，非空时覆盖
- 新用户：status 参数为空时默认 new，非空时使用指定值
仅在 WX_DEV_MODE=true 时注册。
"""
openid = body.openid
target_status = body.status # 可能为 None
conn = get_connection()
try:
with conn.cursor() as cur:
# 查找已有用户
cur.execute(
"SELECT id, status FROM auth.users WHERE wx_openid = %s",
(openid,),
)
row = cur.fetchone()
if row is None:
# 新用户：使用指定状态或默认 new
init_status = target_status or "new"
cur.execute(
"""
INSERT INTO auth.users (wx_openid, status)
VALUES (%s, %s)
RETURNING id, status
""",
(openid, init_status),
)
row = cur.fetchone()
conn.commit()
else:
# 已有用户：仅在显式传入 status 时覆盖
if target_status is not None:
user_id_existing = row[0]
cur.execute(
"UPDATE auth.users SET status = %s, updated_at = NOW() WHERE id = %s",
(target_status, user_id_existing),
)
conn.commit()
row = (user_id_existing, target_status)
user_id, user_status = row
# 签发令牌（逻辑与正常登录一致）
if user_status == "approved":
default_site_id = _get_user_default_site(conn, user_id)
if default_site_id is not None:
roles = _get_user_roles_at_site(conn, user_id, default_site_id)
tokens = create_token_pair(user_id, default_site_id, roles=roles)
else:
tokens = create_limited_token_pair(user_id)
else:
tokens = create_limited_token_pair(user_id)
finally:
conn.close()
return WxLoginResponse(
access_token=tokens["access_token"],
refresh_token=tokens["refresh_token"],
token_type=tokens["token_type"],
user_status=user_status,
user_id=user_id,
)
# ── GET /api/xcx/dev-context（仅开发模式） ────────────────
@router.get("/dev-context", response_model=DevContextResponse)
async def dev_context(
user: CurrentUser = Depends(get_current_user_or_limited),
):
"""
返回当前用户的完整调试上下文。
包含：用户信息、当前门店、角色、权限、人员绑定、所有关联门店。
允许受限令牌访问（返回基础信息，门店/角色/权限为空）。
"""
conn = get_connection()
try:
with conn.cursor() as cur:
# 用户基本信息
cur.execute(
"SELECT wx_openid, status, nickname FROM auth.users WHERE id = %s",
(user.user_id,),
)
u_row = cur.fetchone()
if u_row is None:
raise HTTPException(status_code=404, detail="用户不存在")
openid, u_status, nickname = u_row
# 当前门店名称
site_name = None
if user.site_id:
cur.execute(
"SELECT site_name FROM auth.site_code_mapping WHERE site_id = %s",
(user.site_id,),
)
sn_row = cur.fetchone()
site_name = sn_row[0] if sn_row else None
# 当前门店下的角色
roles = _get_user_roles_at_site(conn, user.user_id, user.site_id) if user.site_id else []
# 当前门店下的权限
permissions = await get_user_permissions(user.user_id, user.site_id) if user.site_id else []
# 人员绑定
binding = None
if user.site_id:
cur.execute(
"""
SELECT assistant_id, staff_id, binding_type
FROM auth.user_assistant_binding
WHERE user_id = %s AND site_id = %s
LIMIT 1
""",
(user.user_id, user.site_id),
)
b_row = cur.fetchone()
if b_row:
binding = {
"assistant_id": b_row[0],
"staff_id": b_row[1],
"binding_type": b_row[2],
}
# 所有关联门店
cur.execute(
"""
SELECT usr.site_id,
COALESCE(scm.site_name, '') AS site_name,
r.code, r.name
FROM auth.user_site_roles usr
JOIN auth.roles r ON usr.role_id = r.id
LEFT JOIN auth.site_code_mapping scm ON usr.site_id = scm.site_id
WHERE usr.user_id = %s
ORDER BY usr.site_id, r.code
""",
(user.user_id,),
)
site_rows = cur.fetchall()
finally:
conn.close()
sites_map: dict[int, dict] = {}
for sid, sname, rcode, rname in site_rows:
if sid not in sites_map:
sites_map[sid] = {"site_id": sid, "site_name": sname, "roles": []}
sites_map[sid]["roles"].append({"code": rcode, "name": rname})
return DevContextResponse(
user_id=user.user_id,
openid=openid,
status=u_status,
nickname=nickname,
site_id=user.site_id,
site_name=site_name,
roles=roles,
permissions=permissions,
binding=binding,
all_sites=list(sites_map.values()),
)
# ── POST /api/xcx/dev-switch-role（仅开发模式） ───────────
@router.post("/dev-switch-role", response_model=WxLoginResponse)
async def dev_switch_role(
body: DevSwitchRoleRequest,
user: CurrentUser = Depends(get_current_user),
):
"""
切换当前用户在当前门店下的角色。
删除旧角色绑定，插入新角色绑定，重签 token。
"""
valid_roles = ("coach", "staff", "site_admin", "tenant_admin")
if body.role_code not in valid_roles:
raise HTTPException(
status_code=400,
detail=f"无效角色，可选: {', '.join(valid_roles)}",
)
conn = get_connection()
try:
with conn.cursor() as cur:
# 查询目标角色 ID
cur.execute(
"SELECT id FROM auth.roles WHERE code = %s",
(body.role_code,),
)
role_row = cur.fetchone()
if role_row is None:
raise HTTPException(status_code=400, detail=f"角色 {body.role_code} 不存在")
role_id = role_row[0]
# 删除当前门店下的所有角色
cur.execute(
"DELETE FROM auth.user_site_roles WHERE user_id = %s AND site_id = %s",
(user.user_id, user.site_id),
)
# 插入新角色
cur.execute(
"""
INSERT INTO auth.user_site_roles (user_id, site_id, role_id)
VALUES (%s, %s, %s)
ON CONFLICT (user_id, site_id, role_id) DO NOTHING
""",
(user.user_id, user.site_id, role_id),
)
conn.commit()
# 重签 token
roles = _get_user_roles_at_site(conn, user.user_id, user.site_id)
cur.execute("SELECT status FROM auth.users WHERE id = %s", (user.user_id,))
u_status = cur.fetchone()[0]
finally:
conn.close()
tokens = create_token_pair(user.user_id, user.site_id, roles=roles)
return WxLoginResponse(
access_token=tokens["access_token"],
refresh_token=tokens["refresh_token"],
token_type=tokens["token_type"],
user_status=u_status,
user_id=user.user_id,
)
# ── POST /api/xcx/dev-switch-status（仅开发模式） ─────────
@router.post("/dev-switch-status", response_model=WxLoginResponse)
async def dev_switch_status(
body: DevSwitchStatusRequest,
user: CurrentUser = Depends(get_current_user_or_limited),
):
"""
切换当前用户状态，重签 token。
允许受限令牌访问（pending 用户也需要能切换状态）。
"""
valid_statuses = ("new", "pending", "approved", "rejected", "disabled")
if body.status not in valid_statuses:
raise HTTPException(
status_code=400,
detail=f"无效状态，可选: {', '.join(valid_statuses)}",
)
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"UPDATE auth.users SET status = %s, updated_at = NOW() WHERE id = %s",
(body.status, user.user_id),
)
conn.commit()
# 根据新状态签发对应令牌
if body.status == "approved":
default_site_id = _get_user_default_site(conn, user.user_id)
if default_site_id is not None:
roles = _get_user_roles_at_site(conn, user.user_id, default_site_id)
tokens = create_token_pair(user.user_id, default_site_id, roles=roles)
else:
tokens = create_limited_token_pair(user.user_id)
else:
tokens = create_limited_token_pair(user.user_id)
finally:
conn.close()
return WxLoginResponse(
access_token=tokens["access_token"],
refresh_token=tokens["refresh_token"],
token_type=tokens["token_type"],
user_status=body.status,
user_id=user.user_id,
)
# ── POST /api/xcx/dev-switch-binding（仅开发模式） ────────
@router.post("/dev-switch-binding")
async def dev_switch_binding(
body: DevSwitchBindingRequest,
user: CurrentUser = Depends(get_current_user),
):
"""
切换当前用户在当前门店下的人员绑定。
删除旧绑定，插入新绑定。
"""
valid_types = ("assistant", "staff", "manager")
if body.binding_type not in valid_types:
raise HTTPException(
status_code=400,
detail=f"无效绑定类型，可选: {', '.join(valid_types)}",
)
conn = get_connection()
try:
with conn.cursor() as cur:
# 删除当前门店下的旧绑定
cur.execute(
"DELETE FROM auth.user_assistant_binding WHERE user_id = %s AND site_id = %s",
(user.user_id, user.site_id),
)
# 插入新绑定
cur.execute(
"""
INSERT INTO auth.user_assistant_binding
(user_id, site_id, assistant_id, staff_id, binding_type)
VALUES (%s, %s, %s, %s, %s)
""",
(user.user_id, user.site_id, body.assistant_id, body.staff_id, body.binding_type),
)
conn.commit()
finally:
conn.close()
return {"ok": True, "binding_type": body.binding_type}

67
apps/backend/app/routers/xcx_notes.py Normal file
View File

@@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
"""
小程序备注路由 —— 备注 CRUD（含星星评分）。
端点清单：
- POST /api/xcx/notes — 创建备注
- GET /api/xcx/notes — 查询备注列表（query: target_type, target_id）
- DELETE /api/xcx/notes/{id} — 删除备注
所有端点均需 JWT（approved 状态）。
"""
from __future__ import annotations
from fastapi import APIRouter, Depends, Query
from app.auth.dependencies import CurrentUser
from app.middleware.permission import require_approved
from app.schemas.xcx_notes import NoteCreateRequest, NoteOut
from app.services import note_service
router = APIRouter(prefix="/api/xcx/notes", tags=["小程序备注"])
@router.post("", response_model=NoteOut)
async def create_note(
body: NoteCreateRequest,
user: CurrentUser = Depends(require_approved()),
):
"""创建备注（含星星评分，可选关联任务）。"""
return await note_service.create_note(
site_id=user.site_id,
user_id=user.user_id,
target_type=body.target_type,
target_id=body.target_id,
content=body.content,
task_id=body.task_id,
rating_service_willingness=body.rating_service_willingness,
rating_revisit_likelihood=body.rating_revisit_likelihood,
)
@router.get("")
async def get_notes(
target_type: str = Query("member", description="目标类型"),
target_id: int = Query(..., description="目标 ID"),
user: CurrentUser = Depends(require_approved()),
):
"""查询某目标的备注列表（按创建时间倒序）。"""
return await note_service.get_notes(
site_id=user.site_id,
target_type=target_type,
target_id=target_id,
)
@router.delete("/{note_id}")
async def delete_note(
note_id: int,
user: CurrentUser = Depends(require_approved()),
):
"""删除备注（验证归属后硬删除）。"""
return await note_service.delete_note(
note_id=note_id,
user_id=user.user_id,
site_id=user.site_id,
)

71
apps/backend/app/routers/xcx_tasks.py Normal file
View File

@@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
"""
小程序任务路由 —— 任务列表、置顶、放弃、取消放弃。
端点清单：
- GET /api/xcx/tasks — 获取活跃任务列表
- POST /api/xcx/tasks/{id}/pin — 置顶任务
- POST /api/xcx/tasks/{id}/unpin — 取消置顶
- POST /api/xcx/tasks/{id}/abandon — 放弃任务
- POST /api/xcx/tasks/{id}/cancel-abandon — 取消放弃
所有端点均需 JWT（approved 状态）。
"""
from __future__ import annotations
from fastapi import APIRouter, Depends
from app.auth.dependencies import CurrentUser
from app.middleware.permission import require_approved
from app.schemas.xcx_tasks import AbandonRequest, TaskListItem
from app.services import task_manager
router = APIRouter(prefix="/api/xcx/tasks", tags=["小程序任务"])
@router.get("", response_model=list[TaskListItem])
async def get_tasks(
user: CurrentUser = Depends(require_approved()),
):
"""获取当前助教的活跃任务列表。"""
return await task_manager.get_task_list(user.user_id, user.site_id)
@router.post("/{task_id}/pin")
async def pin_task(
task_id: int,
user: CurrentUser = Depends(require_approved()),
):
"""置顶任务。"""
return await task_manager.pin_task(task_id, user.user_id, user.site_id)
@router.post("/{task_id}/unpin")
async def unpin_task(
task_id: int,
user: CurrentUser = Depends(require_approved()),
):
"""取消置顶。"""
return await task_manager.unpin_task(task_id, user.user_id, user.site_id)
@router.post("/{task_id}/abandon")
async def abandon_task(
task_id: int,
body: AbandonRequest,
user: CurrentUser = Depends(require_approved()),
):
"""放弃任务（需填写原因）。"""
return await task_manager.abandon_task(
task_id, user.user_id, user.site_id, body.reason
)
@router.post("/{task_id}/cancel-abandon")
async def cancel_abandon(
task_id: int,
user: CurrentUser = Depends(require_approved()),
):
"""取消放弃，恢复为活跃状态。"""
return await task_manager.cancel_abandon(task_id, user.user_id, user.site_id)

1
apps/backend/app/schemas/execution.py
View File

@@ -50,6 +50,7 @@ class ExecutionHistoryItem(BaseModel):
duration_ms: int | None = None
command: str | None = None
summary: dict[str, Any] | None = None
schedule_id: str | None = None
class ExecutionLogsResponse(BaseModel):

54
apps/backend/app/schemas/member_retention_clue.py Normal file
View File

@@ -0,0 +1,54 @@
"""
维客线索相关 Pydantic 模型。
大类枚举：客户基础信息、消费习惯、玩法偏好、促销偏好、社交关系、重要反馈
"""
from datetime import datetime
from enum import Enum
from typing import Optional
from pydantic import BaseModel, Field
class ClueCategory(str, Enum):
"""维客线索大类枚举"""
BASIC_INFO = "客户基础信息"
CONSUMPTION = "消费习惯"
PLAY_PREF = "玩法偏好"
PROMO_PREF = "促销偏好"
SOCIAL = "社交关系"
FEEDBACK = "重要反馈"
class ClueSource(str, Enum):
"""维客线索来源枚举"""
MANUAL = "manual" # 助教手动录入
AI_CONSUMPTION = "ai_consumption" # 应用 3：消费分析自动生成
AI_NOTE = "ai_note" # 应用 6：备注分析自动提取
class RetentionClueSubmit(BaseModel):
"""提交维客线索请求。"""
member_id: int = Field(..., gt=0, description="会员 ID")
category: ClueCategory = Field(..., description="线索大类")
summary: str = Field(..., min_length=1, max_length=200, description="摘要：重点信息")
detail: Optional[str] = Field(None, max_length=2000, description="详情：分析及扩展说明，可为空")
recorded_by_assistant_id: int = Field(..., gt=0, description="记录助教 ID")
recorded_by_name: str = Field(..., min_length=1, max_length=50, description="记录助教姓名")
site_id: int = Field(..., gt=0, description="门店 ID")
source: ClueSource = Field(default=ClueSource.MANUAL, description="线索来源")
class RetentionClueOut(BaseModel):
"""维客线索返回模型。"""
id: int
member_id: int
category: ClueCategory
summary: str
detail: Optional[str]
recorded_by_assistant_id: Optional[int]
recorded_by_name: Optional[str]
recorded_at: datetime
site_id: int
source: ClueSource = ClueSource.MANUAL

1
apps/backend/app/schemas/schedules.py
View File

@@ -32,6 +32,7 @@ class CreateScheduleRequest(BaseModel):
task_codes: list[str]
task_config: dict[str, Any]
schedule_config: ScheduleConfigSchema
run_immediately: bool = False
class UpdateScheduleRequest(BaseModel):

5
apps/backend/app/schemas/tasks.py
View File

@@ -47,6 +47,11 @@ class TaskConfigSchema(BaseModel):
store_id: int | None = None
dwd_only_tables: list[str] | None = None
force_full: bool = False
# Pipeline 调优参数（可选，不传则使用 ETL 默认值）
pipeline_workers: int | None = None
pipeline_batch_size: int | None = None
pipeline_rate_min: float | None = None
pipeline_rate_max: float | None = None
extra_args: dict[str, Any] = {}
@model_validator(mode="after")

106
apps/backend/app/schemas/xcx_auth.py
View File

@@ -25,6 +25,46 @@ class WxLoginResponse(BaseModel):
user_id: int
class DevLoginRequest(BaseModel):
"""开发模式 mock 登录请求（仅 WX_DEV_MODE=true 时可用）。"""
openid: str = Field(..., min_length=1, description="模拟的微信 openid")
status: str | None = Field(None, description="模拟的用户状态；为空时保留已有用户的当前状态，新用户默认 new")
# ── 开发调试端点（仅 WX_DEV_MODE=true） ─────────────────
class DevSwitchRoleRequest(BaseModel):
"""切换角色请求。替换当前用户在当前门店下的所有角色为指定角色。"""
role_code: str = Field(..., description="目标角色 code（coach/staff/site_admin/tenant_admin）")
class DevSwitchStatusRequest(BaseModel):
"""切换用户状态请求。"""
status: str = Field(..., description="目标状态（new/pending/approved/rejected/disabled）")
class DevSwitchBindingRequest(BaseModel):
"""切换人员绑定请求。"""
binding_type: str = Field(..., description="绑定类型（assistant/staff/manager）")
assistant_id: int | None = Field(None, description="助教 ID（binding_type=assistant 时必填）")
staff_id: int | None = Field(None, description="员工 ID（binding_type=staff/manager 时必填）")
class DevContextResponse(BaseModel):
"""开发调试上下文信息。"""
user_id: int
openid: str | None = None
status: str
nickname: str | None = None
site_id: int | None = None
site_name: str | None = None
roles: list[str] = []
permissions: list[str] = []
binding: dict | None = None
all_sites: list[dict] = []
# ── 用户申请 ──────────────────────────────────────────────
class ApplicationRequest(BaseModel):
@@ -102,3 +142,69 @@ class RejectRequest(BaseModel):
"""拒绝申请请求。"""
review_note: str = Field(..., min_length=1, description="拒绝原因")
# ── 开发调试端点（仅 WX_DEV_MODE=true） ─────────────────
class DevSwitchRoleRequest(BaseModel):
"""切换角色请求。替换当前用户在当前门店下的所有角色为指定角色。"""
role_code: str = Field(..., description="目标角色 code（coach/staff/site_admin/tenant_admin）")
class DevSwitchStatusRequest(BaseModel):
"""切换用户状态请求。"""
status: str = Field(..., description="目标状态（new/pending/approved/rejected/disabled）")
class DevSwitchBindingRequest(BaseModel):
"""切换人员绑定请求。"""
binding_type: str = Field(..., description="绑定类型（assistant/staff/manager）")
assistant_id: int | None = Field(None, description="助教 ID（binding_type=assistant 时必填）")
staff_id: int | None = Field(None, description="员工 ID（binding_type=staff/manager 时必填）")
class DevContextResponse(BaseModel):
"""开发调试上下文信息。"""
user_id: int
openid: str | None = None
status: str
nickname: str | None = None
site_id: int | None = None
site_name: str | None = None
roles: list[str] = []
permissions: list[str] = []
binding: dict | None = None
all_sites: list[dict] = []
# ── 开发调试端点（仅 WX_DEV_MODE=true） ─────────────────
class DevSwitchRoleRequest(BaseModel):
"""切换角色请求。替换当前用户在当前门店下的所有角色为指定角色。"""
role_code: str = Field(..., description="目标角色 code（coach/staff/site_admin/tenant_admin）")
class DevSwitchStatusRequest(BaseModel):
"""切换用户状态请求。"""
status: str = Field(..., description="目标状态（new/pending/approved/rejected/disabled）")
class DevSwitchBindingRequest(BaseModel):
"""切换人员绑定请求。"""
binding_type: str = Field(..., description="绑定类型（assistant/staff/manager）")
assistant_id: int | None = Field(None, description="助教 ID（binding_type=assistant 时必填）")
staff_id: int | None = Field(None, description="员工 ID（binding_type=staff/manager 时必填）")
class DevContextResponse(BaseModel):
"""开发调试上下文信息。"""
user_id: int
openid: str | None = None
status: str
nickname: str | None = None
site_id: int | None = None
site_name: str | None = None
roles: list[str] = []
permissions: list[str] = []
binding: dict | None = None
all_sites: list[dict] = []

35
apps/backend/app/schemas/xcx_notes.py Normal file
View File

@@ -0,0 +1,35 @@
"""
小程序备注相关 Pydantic 模型。
覆盖：备注创建请求、备注输出等场景。
"""
from __future__ import annotations
from pydantic import BaseModel, Field
class NoteCreateRequest(BaseModel):
"""创建备注请求（含星星评分，评分 1-5 范围约束）。"""
target_type: str = Field(default="member")
target_id: int
content: str = Field(..., min_length=1)
task_id: int | None = None
rating_service_willingness: int | None = Field(None, ge=1, le=5)
rating_revisit_likelihood: int | None = Field(None, ge=1, le=5)
class NoteOut(BaseModel):
"""备注输出模型（含评分 + AI 评分）。"""
id: int
type: str
content: str
rating_service_willingness: int | None
rating_revisit_likelihood: int | None
ai_score: int | None
ai_analysis: str | None
task_id: int | None
created_at: str
updated_at: str

34
apps/backend/app/schemas/xcx_tasks.py Normal file
View File

@@ -0,0 +1,34 @@
"""
小程序任务相关 Pydantic 模型。
覆盖：任务列表项、放弃请求等场景。
"""
from __future__ import annotations
from pydantic import BaseModel, Field
class TaskListItem(BaseModel):
"""任务列表项（含客户信息 + RS 指数 + 爱心 icon）。"""
id: int
task_type: str
status: str
priority_score: float | None
is_pinned: bool
expires_at: str | None
created_at: str
# 客户信息（FDW 读取）
member_id: int
member_name: str | None
member_phone: str | None
# RS 指数 + 爱心 icon
rs_score: float | None
heart_icon: str # 💖 / 🧡 / 💛 / 💙
class AbandonRequest(BaseModel):
"""放弃任务请求（reason 必填）。"""
reason: str = Field(..., min_length=1, description="放弃原因（必填）")

9
apps/backend/app/services/application.py
View File

@@ -98,6 +98,15 @@ async def create_application(
(nickname, user_id),
)
# 5. 更新用户状态为 pending（new → pending）
cur.execute(
"""
UPDATE auth.users SET status = 'pending', updated_at = NOW()
WHERE id = %s AND status IN ('new', 'rejected')
""",
(user_id,),
)
conn.commit()
finally:
conn.close()

10
apps/backend/app/services/cli_builder.py
View File

@@ -124,6 +124,16 @@ class CLIBuilder:
if config.store_id is not None:
cmd.extend(["--store-id", str(config.store_id)])
# -- Pipeline 调优参数 --
if config.pipeline_workers is not None:
cmd.extend(["--pipeline-workers", str(config.pipeline_workers)])
if config.pipeline_batch_size is not None:
cmd.extend(["--pipeline-batch-size", str(config.pipeline_batch_size)])
if config.pipeline_rate_min is not None:
cmd.extend(["--pipeline-rate-min", str(config.pipeline_rate_min)])
if config.pipeline_rate_max is not None:
cmd.extend(["--pipeline-rate-max", str(config.pipeline_rate_max)])
# -- 额外参数（只传递 CLI 支持的参数） --
for key, value in config.extra_args.items():
if value is not None and key in CLI_SUPPORTED_ARGS:

217
apps/backend/app/services/note_reclassifier.py Normal file
View File

@@ -0,0 +1,217 @@
# -*- coding: utf-8 -*-
"""
备注回溯重分类器（Note Reclassifier）
召回完成后，回溯检查是否有普通备注需重分类为回访备注。
查找 service_time 之后的第一条 normal 备注 → 更新为 follow_up →
触发 AI 应用 6 接口（占位）→ 根据 ai_score 生成 follow_up_visit 任务。
由 trigger_jobs 中的 note_reclassify_backfill 配置驱动（event: recall_completed）。
"""
import json
import logging
logger = logging.getLogger(__name__)
def _get_connection():
"""延迟导入 get_connection，避免模块级导入失败。"""
from app.database import get_connection
return get_connection()
def _insert_history(
cur,
task_id: int,
action: str,
old_status: str | None = None,
new_status: str | None = None,
old_task_type: str | None = None,
new_task_type: str | None = None,
detail: dict | None = None,
) -> None:
"""在 coach_task_history 中记录变更。"""
cur.execute(
"""
INSERT INTO biz.coach_task_history
(task_id, action, old_status, new_status,
old_task_type, new_task_type, detail)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""",
(
task_id,
action,
old_status,
new_status,
old_task_type,
new_task_type,
json.dumps(detail) if detail else None,
),
)
def ai_analyze_note(note_id: int) -> int | None:
"""
AI 应用 6 备注分析接口（占位）。
P5 AI 集成层实现后替换此占位函数。
当前返回 None 表示 AI 未就绪，跳过评分逻辑。
"""
return None
def run(payload: dict | None = None) -> dict:
"""
备注回溯主流程。
payload 包含: {site_id, assistant_id, member_id, service_time}
1. 查找 biz.notes 中该 (site_id, target_type='member', target_id=member_id)
在 service_time 之后提交的第一条 type='normal' 的备注
2. 将该备注 type 从 'normal' 更新为 'follow_up'
3. 触发 AI 应用 6 接口（P5 实现，本 SPEC 仅定义触发接口）：
- 调用 ai_analyze_note(note_id) → 返回 ai_score
4. 若 ai_score >= 6：
- 生成 follow_up_visit 任务，status='completed'（回溯完成）
5. 若 ai_score < 6：
- 生成 follow_up_visit 任务，status='active'（需助教重新备注）
返回: {"reclassified_count": int, "tasks_created": int}
"""
if not payload:
return {"reclassified_count": 0, "tasks_created": 0}
site_id = payload.get("site_id")
assistant_id = payload.get("assistant_id")
member_id = payload.get("member_id")
service_time = payload.get("service_time")
if not all([site_id, assistant_id, member_id, service_time]):
logger.warning("备注回溯缺少必要参数: %s", payload)
return {"reclassified_count": 0, "tasks_created": 0}
reclassified_count = 0
tasks_created = 0
conn = _get_connection()
try:
# ── 1. 查找 service_time 之后的第一条 normal 备注 ──
note_id = None
with conn.cursor() as cur:
cur.execute(
"""
SELECT id
FROM biz.notes
WHERE site_id = %s
AND target_type = 'member'
AND target_id = %s
AND type = 'normal'
AND created_at > %s
ORDER BY created_at ASC
LIMIT 1
""",
(site_id, member_id, service_time),
)
row = cur.fetchone()
if row:
note_id = row[0]
conn.commit()
if note_id is None:
logger.info(
"未找到符合条件的 normal 备注: site_id=%s, member_id=%s",
site_id, member_id,
)
return {"reclassified_count": 0, "tasks_created": 0}
# ── 2. 将备注 type 从 'normal' 更新为 'follow_up' ──
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"""
UPDATE biz.notes
SET type = 'follow_up', updated_at = NOW()
WHERE id = %s AND type = 'normal'
""",
(note_id,),
)
conn.commit()
reclassified_count = 1
# ── 3. 触发 AI 应用 6 接口（占位，当前返回 None） ──
ai_score = ai_analyze_note(note_id)
# ── 4/5. 根据 ai_score 生成 follow_up_visit 任务 ──
if ai_score is not None:
if ai_score >= 6:
# 回溯完成：生成 completed 任务
task_status = "completed"
else:
# 需助教重新备注：生成 active 任务
task_status = "active"
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"""
INSERT INTO biz.coach_tasks
(site_id, assistant_id, member_id, task_type,
status, completed_at, completed_task_type)
VALUES (
%s, %s, %s, 'follow_up_visit',
%s,
CASE WHEN %s = 'completed' THEN NOW() ELSE NULL END,
CASE WHEN %s = 'completed' THEN 'follow_up_visit' ELSE NULL END
)
RETURNING id
""",
(
site_id, assistant_id, member_id,
task_status, task_status, task_status,
),
)
new_task_row = cur.fetchone()
new_task_id = new_task_row[0]
# 记录任务创建历史
_insert_history(
cur,
new_task_id,
action="created_by_reclassify",
old_status=None,
new_status=task_status,
old_task_type=None,
new_task_type="follow_up_visit",
detail={
"note_id": note_id,
"ai_score": ai_score,
"source": "note_reclassifier",
},
)
conn.commit()
tasks_created = 1
else:
# AI 未就绪，跳过任务创建
logger.info(
"AI 接口未就绪，跳过任务创建: note_id=%s", note_id
)
except Exception:
logger.exception(
"备注回溯失败: site_id=%s, member_id=%s",
site_id, member_id,
)
conn.rollback()
finally:
conn.close()
logger.info(
"备注回溯完成: reclassified_count=%d, tasks_created=%d",
reclassified_count, tasks_created,
)
return {
"reclassified_count": reclassified_count,
"tasks_created": tasks_created,
}

326
apps/backend/app/services/note_service.py Normal file
View File

@@ -0,0 +1,326 @@
"""
备注服务
负责备注 CRUD、星星评分存储与读取。
备注类型根据关联任务自动确定：follow_up_visit 任务 → follow_up，否则 normal。
星星评分不参与回访完成判定，不参与 AI 分析，仅存储。
"""
import json
import logging
from fastapi import HTTPException
logger = logging.getLogger(__name__)
def _get_connection():
"""延迟导入 get_connection，避免纯函数测试时触发模块级导入失败。"""
from app.database import get_connection
return get_connection()
def _record_history(
cur,
task_id: int,
action: str,
old_status: str | None = None,
new_status: str | None = None,
old_task_type: str | None = None,
new_task_type: str | None = None,
detail: dict | None = None,
) -> None:
"""在 coach_task_history 中记录变更（复用 task_manager 的模式）。"""
cur.execute(
"""
INSERT INTO biz.coach_task_history
(task_id, action, old_status, new_status,
old_task_type, new_task_type, detail)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""",
(
task_id,
action,
old_status,
new_status,
old_task_type,
new_task_type,
json.dumps(detail) if detail else None,
),
)
def ai_analyze_note(note_id: int) -> int | None:
"""
AI 应用 6 备注分析接口（占位）。
P5 AI 集成层实现后替换此占位函数。
当前返回 None 表示 AI 未就绪，跳过评分逻辑。
"""
return None
async def create_note(
site_id: int,
user_id: int,
target_type: str,
target_id: int,
content: str,
task_id: int | None = None,
rating_service_willingness: int | None = None,
rating_revisit_likelihood: int | None = None,
) -> dict:
"""
创建备注。
1. 验证评分范围（1-5 或 NULL），不合法则 422
2. 确定 note type：
- 若 task_id 关联的任务 task_type='follow_up_visit' → type='follow_up'
- 否则 → type='normal'
3. INSERT INTO biz.notes
4. 若 type='follow_up'：
- 触发 AI 应用 6 分析（P5 实现）
- 若 ai_score >= 6 且关联任务 status='active' → 标记任务 completed
5. 返回创建的备注记录
注意：星星评分不参与回访完成判定，不参与 AI 分析，仅存储。
"""
# 验证评分范围
for label, val in [
("再次服务意愿评分", rating_service_willingness),
("再来店可能性评分", rating_revisit_likelihood),
]:
if val is not None and (val < 1 or val > 5):
raise HTTPException(
status_code=422, detail=f"{label}必须在 1-5 范围内"
)
conn = _get_connection()
try:
# 确定 note type
note_type = "normal"
task_info = None
if task_id is not None:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, task_type, status, site_id
FROM biz.coach_tasks
WHERE id = %s
""",
(task_id,),
)
row = cur.fetchone()
if row is None:
raise HTTPException(status_code=404, detail="关联任务不存在")
task_info = {
"id": row[0],
"task_type": row[1],
"status": row[2],
"site_id": row[3],
}
if task_info["site_id"] != site_id:
raise HTTPException(status_code=403, detail="权限不足")
if task_info["task_type"] == "follow_up_visit":
note_type = "follow_up"
# INSERT 备注
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"""
INSERT INTO biz.notes
(site_id, user_id, target_type, target_id, type,
content, rating_service_willingness,
rating_revisit_likelihood, task_id)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id, site_id, user_id, target_type, target_id,
type, content, rating_service_willingness,
rating_revisit_likelihood, task_id,
ai_score, ai_analysis, created_at, updated_at
""",
(
site_id, user_id, target_type, target_id, note_type,
content, rating_service_willingness,
rating_revisit_likelihood, task_id,
),
)
row = cur.fetchone()
note = {
"id": row[0],
"site_id": row[1],
"user_id": row[2],
"target_type": row[3],
"target_id": row[4],
"type": row[5],
"content": row[6],
"rating_service_willingness": row[7],
"rating_revisit_likelihood": row[8],
"task_id": row[9],
"ai_score": row[10],
"ai_analysis": row[11],
"created_at": row[12].isoformat() if row[12] else None,
"updated_at": row[13].isoformat() if row[13] else None,
}
# 若 type='follow_up'，触发 AI 分析并可能标记任务完成
if note_type == "follow_up" and task_id is not None:
ai_score = ai_analyze_note(note["id"])
if ai_score is not None:
# 更新备注的 ai_score
cur.execute(
"""
UPDATE biz.notes
SET ai_score = %s, updated_at = NOW()
WHERE id = %s
""",
(ai_score, note["id"]),
)
note["ai_score"] = ai_score
# 若 ai_score >= 6 且关联任务 status='active' → 标记任务 completed
if ai_score >= 6 and task_info and task_info["status"] == "active":
cur.execute(
"""
UPDATE biz.coach_tasks
SET status = 'completed',
completed_at = NOW(),
completed_task_type = task_type,
updated_at = NOW()
WHERE id = %s AND status = 'active'
""",
(task_id,),
)
_record_history(
cur,
task_id,
action="completed_by_note",
old_status="active",
new_status="completed",
old_task_type=task_info["task_type"],
new_task_type=task_info["task_type"],
detail={
"note_id": note["id"],
"ai_score": ai_score,
},
)
conn.commit()
return note
except HTTPException:
conn.rollback()
raise
except Exception:
conn.rollback()
raise
finally:
conn.close()
async def get_notes(
site_id: int, target_type: str, target_id: int
) -> list[dict]:
"""
查询某目标的备注列表。
按 created_at DESC 排序，包含星星评分和 AI 评分。
"""
conn = _get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, site_id, user_id, target_type, target_id,
type, content, rating_service_willingness,
rating_revisit_likelihood, task_id,
ai_score, ai_analysis, created_at, updated_at
FROM biz.notes
WHERE site_id = %s
AND target_type = %s
AND target_id = %s
ORDER BY created_at DESC
""",
(site_id, target_type, target_id),
)
rows = cur.fetchall()
conn.commit()
return [
{
"id": r[0],
"site_id": r[1],
"user_id": r[2],
"target_type": r[3],
"target_id": r[4],
"type": r[5],
"content": r[6],
"rating_service_willingness": r[7],
"rating_revisit_likelihood": r[8],
"task_id": r[9],
"ai_score": r[10],
"ai_analysis": r[11],
"created_at": r[12].isoformat() if r[12] else None,
"updated_at": r[13].isoformat() if r[13] else None,
}
for r in rows
]
finally:
conn.close()
async def delete_note(note_id: int, user_id: int, site_id: int) -> dict:
"""
删除备注。
验证备注归属（user_id + site_id）后执行硬删除。
- 不存在 → 404
- 不属于当前用户 → 403
"""
conn = _get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, user_id, site_id
FROM biz.notes
WHERE id = %s
""",
(note_id,),
)
row = cur.fetchone()
if not row:
raise HTTPException(status_code=404, detail="资源不存在")
if row[2] != site_id or row[1] != user_id:
raise HTTPException(status_code=403, detail="权限不足")
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"DELETE FROM biz.notes WHERE id = %s",
(note_id,),
)
conn.commit()
return {"id": note_id, "deleted": True}
except HTTPException:
conn.rollback()
raise
except Exception:
conn.rollback()
raise
finally:
conn.close()

266
apps/backend/app/services/recall_detector.py Normal file
View File

@@ -0,0 +1,266 @@
# -*- coding: utf-8 -*-
"""
召回完成检测器（Recall Completion Detector）
ETL 数据更新后，通过 FDW 读取助教服务记录，
匹配活跃任务标记为 completed，记录 completed_at 和 completed_task_type 快照，
触发 recall_completed 事件通知备注回溯重分类器。
由 trigger_jobs 中的 recall_completion_check 配置驱动（event: etl_data_updated）。
"""
import json
import logging
logger = logging.getLogger(__name__)
def _get_connection():
"""延迟导入 get_connection，避免模块级导入失败。"""
from app.database import get_connection
return get_connection()
def _insert_history(
cur,
task_id: int,
action: str,
old_status: str | None = None,
new_status: str | None = None,
old_task_type: str | None = None,
new_task_type: str | None = None,
detail: dict | None = None,
) -> None:
"""在 coach_task_history 中记录变更。"""
cur.execute(
"""
INSERT INTO biz.coach_task_history
(task_id, action, old_status, new_status,
old_task_type, new_task_type, detail)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""",
(
task_id,
action,
old_status,
new_status,
old_task_type,
new_task_type,
json.dumps(detail) if detail else None,
),
)
def run(payload: dict | None = None) -> dict:
"""
召回完成检测主流程。
1. 从 trigger_jobs 读取 last_run_at 作为增量过滤基准
2. 获取所有 distinct site_id（从 active 任务中）
3. 对每个 site_id，SET LOCAL app.current_site_id 后
通过 FDW 读取 v_dwd_assistant_service_log 中 service_time > last_run_at 的新增服务记录
4. 对每条服务记录，查找 biz.coach_tasks 中匹配的
(site_id, assistant_id, member_id) 且 status='active' 的任务
5. 将匹配任务标记为 completed：
- status = 'completed'
- completed_at = 服务时间
- completed_task_type = 当前 task_type（快照）
6. 记录 coach_task_history
7. 触发 fire_event('recall_completed', {site_id, assistant_id, member_id, service_time})
返回: {"completed_count": int}
"""
completed_count = 0
conn = _get_connection()
try:
# ── 1. 读取 last_run_at ──
with conn.cursor() as cur:
cur.execute(
"""
SELECT last_run_at
FROM biz.trigger_jobs
WHERE job_name = 'recall_completion_check'
"""
)
row = cur.fetchone()
last_run_at = row[0] if row else None
conn.commit()
# ── 2. 获取所有有 active 任务的 distinct site_id ──
with conn.cursor() as cur:
cur.execute(
"""
SELECT DISTINCT site_id
FROM biz.coach_tasks
WHERE status = 'active'
"""
)
site_ids = [r[0] for r in cur.fetchall()]
conn.commit()
# ── 3. 逐 site_id 读取新增服务记录 ──
for site_id in site_ids:
try:
count = _process_site(conn, site_id, last_run_at)
completed_count += count
except Exception:
logger.exception(
"处理门店召回检测失败: site_id=%s", site_id
)
conn.rollback()
finally:
conn.close()
logger.info("召回完成检测完成: completed_count=%d", completed_count)
return {"completed_count": completed_count}
def _process_site(conn, site_id: int, last_run_at) -> int:
"""
处理单个门店的召回完成检测。
通过 FDW 读取新增服务记录，匹配 active 任务并标记 completed。
返回本门店完成的任务数。
"""
completed = 0
# 通过 FDW 读取新增服务记录（需要 SET LOCAL 启用 RLS）
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"SET LOCAL app.current_site_id = %s", (str(site_id),)
)
if last_run_at is not None:
cur.execute(
"""
SELECT DISTINCT assistant_id, member_id, service_time
FROM fdw_etl.v_dwd_assistant_service_log
WHERE service_time > %s
ORDER BY service_time ASC
""",
(last_run_at,),
)
else:
# 首次运行，读取所有服务记录
cur.execute(
"""
SELECT DISTINCT assistant_id, member_id, service_time
FROM fdw_etl.v_dwd_assistant_service_log
ORDER BY service_time ASC
"""
)
service_records = cur.fetchall()
conn.commit()
# ── 4-7. 逐条服务记录匹配并处理 ──
for assistant_id, member_id, service_time in service_records:
try:
count = _process_service_record(
conn, site_id, assistant_id, member_id, service_time
)
completed += count
except Exception:
logger.exception(
"处理服务记录失败: site_id=%s, assistant_id=%s, member_id=%s",
site_id,
assistant_id,
member_id,
)
conn.rollback()
return completed
def _process_service_record(
conn,
site_id: int,
assistant_id: int,
member_id: int,
service_time,
) -> int:
"""
处理单条服务记录：匹配 active 任务并标记 completed。
每条服务记录独立事务，失败不影响其他。
返回本次完成的任务数。
"""
completed = 0
with conn.cursor() as cur:
cur.execute("BEGIN")
# 查找匹配的 active 任务
cur.execute(
"""
SELECT id, task_type
FROM biz.coach_tasks
WHERE site_id = %s
AND assistant_id = %s
AND member_id = %s
AND status = 'active'
""",
(site_id, assistant_id, member_id),
)
active_tasks = cur.fetchall()
if not active_tasks:
conn.commit()
return 0
# 将所有匹配的 active 任务标记为 completed
for task_id, task_type in active_tasks:
cur.execute(
"""
UPDATE biz.coach_tasks
SET status = 'completed',
completed_at = %s,
completed_task_type = %s,
updated_at = NOW()
WHERE id = %s AND status = 'active'
""",
(service_time, task_type, task_id),
)
_insert_history(
cur,
task_id,
action="completed",
old_status="active",
new_status="completed",
old_task_type=task_type,
new_task_type=task_type,
detail={
"service_time": str(service_time),
"completed_task_type": task_type,
},
)
completed += 1
conn.commit()
# ── 7. 触发 recall_completed 事件 ──
# 延迟导入 fire_event 避免循环依赖
try:
from app.services.trigger_scheduler import fire_event
fire_event(
"recall_completed",
{
"site_id": site_id,
"assistant_id": assistant_id,
"member_id": member_id,
"service_time": str(service_time),
},
)
except Exception:
logger.exception(
"触发 recall_completed 事件失败: site_id=%s, assistant_id=%s, member_id=%s",
site_id,
assistant_id,
member_id,
)
return completed

2
apps/backend/app/services/scheduler.py
View File

@@ -213,7 +213,7 @@ class Scheduler:
# 入队
try:
queue_id = task_queue.enqueue(config, site_id)
queue_id = task_queue.enqueue(config, site_id, schedule_id=task_id)
logger.info(
"调度任务 [%s] 入队成功 → queue_id=%s site_id=%s",
task_id, queue_id, site_id,

72
apps/backend/app/services/task_executor.py
View File

@@ -17,19 +17,24 @@ from __future__ import annotations
import asyncio
import logging
import subprocess
import sys
import threading
import time
from datetime import datetime, timezone
from typing import Any
from ..config import ETL_PROJECT_PATH
# CHANGE 2026-03-07 | 只保留模块引用，execute() 中实时读取属性值
# 禁止 from ..config import ETL_PROJECT_PATH（值拷贝，reload 后过期）
from .. import config as _config_module
from ..database import get_connection
from ..schemas.tasks import TaskConfigSchema
from ..services.cli_builder import cli_builder
logger = logging.getLogger(__name__)
# 实例标识：用于区分多后端实例写入同一 DB 的记录
import platform as _platform
_INSTANCE_HOST = _platform.node() # hostname
class TaskExecutor:
"""管理 ETL CLI 子进程的生命周期"""
@@ -112,21 +117,58 @@ class TaskExecutor:
execution_id: str,
queue_id: str | None = None,
site_id: int | None = None,
schedule_id: str | None = None,
) -> None:
"""以子进程方式调用 ETL CLI。
使用 subprocess.Popen + 线程读取，兼容 Windows（避免
asyncio.create_subprocess_exec 在 Windows 上的 NotImplementedError）。
"""
# CHANGE 2026-03-07 | 实时从 config 模块读取，避免 import 时复制的值过期
etl_path = _config_module.ETL_PROJECT_PATH
etl_python = _config_module.ETL_PYTHON_EXECUTABLE
cmd = cli_builder.build_command(
config, ETL_PROJECT_PATH, python_executable=sys.executable
config, etl_path, python_executable=etl_python
)
command_str = " ".join(cmd)
# CHANGE 2026-03-07 | 运行时防护：拒绝执行包含非预期路径的命令
# 检测两种异常：
# 1. D 盘路径（junction 穿透）
# 2. 多环境子目录（test/repo、prod/repo）
_cmd_normalized = command_str.replace("/", "\\")
_bad_patterns = []
if "D:\\" in command_str or "D:/" in command_str:
_bad_patterns.append("D盘路径")
if "\\test\\repo" in _cmd_normalized or "\\prod\\repo" in _cmd_normalized:
_bad_patterns.append("多环境子目录(test/repo或prod/repo)")
if _bad_patterns:
_issues = " + ".join(_bad_patterns)
logger.error(
"路径防护触发：命令包含 %s，拒绝执行。"
" command=%s | ETL_PY=%s | ETL_PATH=%s"
" | NEOZQYY_ROOT=%s | config.__file__=%s",
_issues, command_str, etl_python, etl_path,
__import__('os').environ.get("NEOZQYY_ROOT", "<未设置>"),
_config_module.__file__,
)
raise RuntimeError(
f"ETL 命令包含异常路径（{_issues}），拒绝执行。"
f" 请检查 .env 中 ETL_PYTHON_EXECUTABLE 和 ETL_PROJECT_PATH 配置。"
f" 当前值: ETL_PY={etl_python}, ETL_PATH={etl_path}"
)
effective_site_id = site_id or config.store_id
# CHANGE 2026-03-07 | 在 command 前缀中注入实例标识，
# 便于在多后端实例共享同一 DB 时区分记录来源
command_str_with_host = f"[{_INSTANCE_HOST}] {command_str}"
logger.info(
"启动 ETL 子进程 [%s]: %s (cwd=%s)",
execution_id, command_str, ETL_PROJECT_PATH,
execution_id, command_str, etl_path,
)
self._log_buffers[execution_id] = []
@@ -140,7 +182,8 @@ class TaskExecutor:
task_codes=config.tasks,
status="running",
started_at=started_at,
command=command_str,
command=command_str_with_host,
schedule_id=schedule_id,
)
exit_code: int | None = None
@@ -226,7 +269,7 @@ class TaskExecutor:
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=ETL_PROJECT_PATH,
cwd=_config_module.ETL_PROJECT_PATH,
env=env,
text=True,
encoding="utf-8",
@@ -302,18 +345,30 @@ class TaskExecutor:
status: str,
started_at: datetime,
command: str,
schedule_id: str | None = None,
) -> None:
"""插入一条执行日志记录（running 状态）。"""
try:
conn = get_connection()
try:
with conn.cursor() as cur:
# 如果调用方未传 schedule_id，尝试从 task_queue 回查
effective_schedule_id = schedule_id
if effective_schedule_id is None and queue_id is not None:
cur.execute(
"SELECT schedule_id FROM task_queue WHERE id = %s",
(queue_id,),
)
row = cur.fetchone()
if row and row[0]:
effective_schedule_id = str(row[0])
cur.execute(
"""
INSERT INTO task_execution_log
(id, queue_id, site_id, task_codes, status,
started_at, command)
VALUES (%s, %s, %s, %s, %s, %s, %s)
started_at, command, schedule_id)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""",
(
execution_id,
@@ -323,6 +378,7 @@ class TaskExecutor:
status,
started_at,
command,
effective_schedule_id,
),
)
conn.commit()

117
apps/backend/app/services/task_expiry.py Normal file
View File

@@ -0,0 +1,117 @@
# -*- coding: utf-8 -*-
"""
有效期轮询器（Task Expiry Checker）
每小时运行一次，检查 expires_at 不为 NULL 且已过期的 active 任务，
将其标记为 inactive 并记录 history。
由 trigger_jobs 中的 task_expiry_check 配置驱动。
"""
import json
import logging
logger = logging.getLogger(__name__)
def _get_connection():
"""延迟导入 get_connection，避免模块级导入失败。"""
from app.database import get_connection
return get_connection()
def _insert_history(
cur,
task_id: int,
action: str,
old_status: str | None = None,
new_status: str | None = None,
old_task_type: str | None = None,
new_task_type: str | None = None,
detail: dict | None = None,
) -> None:
"""在 coach_task_history 中记录变更。"""
cur.execute(
"""
INSERT INTO biz.coach_task_history
(task_id, action, old_status, new_status,
old_task_type, new_task_type, detail)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""",
(
task_id,
action,
old_status,
new_status,
old_task_type,
new_task_type,
json.dumps(detail) if detail else None,
),
)
def run() -> dict:
"""
有效期轮询主流程。
1. SELECT id, task_type FROM biz.coach_tasks
WHERE expires_at IS NOT NULL AND expires_at < NOW() AND status = 'active'
2. 逐条 UPDATE status = 'inactive'
3. INSERT coach_task_history (action='expired')
每条过期任务独立事务，失败不影响其他。
返回: {"expired_count": int}
"""
expired_count = 0
conn = _get_connection()
try:
# 查询所有已过期的 active 任务
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, task_type
FROM biz.coach_tasks
WHERE expires_at IS NOT NULL
AND expires_at < NOW()
AND status = 'active'
"""
)
expired_tasks = cur.fetchall()
conn.commit()
# 逐条处理，每条独立事务
for task_id, task_type in expired_tasks:
try:
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"""
UPDATE biz.coach_tasks
SET status = 'inactive', updated_at = NOW()
WHERE id = %s AND status = 'active'
""",
(task_id,),
)
_insert_history(
cur,
task_id,
action="expired",
old_status="active",
new_status="inactive",
old_task_type=task_type,
new_task_type=task_type,
)
conn.commit()
expired_count += 1
except Exception:
logger.exception("处理过期任务失败: task_id=%s", task_id)
conn.rollback()
finally:
conn.close()
logger.info("有效期轮询完成: expired_count=%d", expired_count)
return {"expired_count": expired_count}

483
apps/backend/app/services/task_generator.py Normal file
View File

@@ -0,0 +1,483 @@
# -*- coding: utf-8 -*-
"""
任务生成器（Task Generator）
每日 4:00 运行，基于 WBI/NCI/RS 指数为每个助教生成/更新任务。
本模块包含：
- TaskPriority 枚举：任务类型优先级定义
- TASK_TYPE_PRIORITY 映射：task_type 字符串 → 优先级
- IndexData 数据类：客户-助教对的指数数据
- determine_task_type()：根据指数确定任务类型（纯函数）
- should_replace_task()：判断是否应替换现有任务（纯函数）
- compute_heart_icon()：根据 RS 指数计算爱心 icon 档位（纯函数）
"""
from decimal import Decimal
from dataclasses import dataclass
from enum import IntEnum
class TaskPriority(IntEnum):
"""任务类型优先级，数值越小优先级越高。"""
HIGH_PRIORITY_RECALL = 0
PRIORITY_RECALL = 0
FOLLOW_UP_VISIT = 1
RELATIONSHIP_BUILDING = 2
TASK_TYPE_PRIORITY: dict[str, TaskPriority] = {
"high_priority_recall": TaskPriority.HIGH_PRIORITY_RECALL,
"priority_recall": TaskPriority.PRIORITY_RECALL,
"follow_up_visit": TaskPriority.FOLLOW_UP_VISIT,
"relationship_building": TaskPriority.RELATIONSHIP_BUILDING,
}
@dataclass
class IndexData:
"""某客户-助教对的指数数据。"""
site_id: int
assistant_id: int
member_id: int
wbi: Decimal # 流失回赢指数
nci: Decimal # 新客转化指数
rs: Decimal # 关系强度指数
has_active_recall: bool # 是否有活跃召回任务
has_follow_up_note: bool # 召回完成后是否有回访备注
def determine_task_type(index_data: IndexData) -> str | None:
"""
根据指数数据确定应生成的任务类型。
优先级规则（高 → 低）：
1. max(WBI, NCI) > 7 → high_priority_recall
2. max(WBI, NCI) > 5 → priority_recall
3. RS < 6 → relationship_building
4. 不满足任何条件 → None（不生成任务）
返回: task_type 字符串或 None
"""
priority_score = max(index_data.wbi, index_data.nci)
if priority_score > 7:
return "high_priority_recall"
if priority_score > 5:
return "priority_recall"
if index_data.rs < 6:
return "relationship_building"
return None
def should_replace_task(existing_type: str, new_type: str) -> bool:
"""
判断新任务类型是否应替换现有任务类型。
规则：类型不同即替换，相同类型不替换。
"""
if existing_type == new_type:
return False
return True
def compute_heart_icon(rs_score: Decimal) -> str:
"""
根据 RS 指数计算爱心 icon 档位。
档位规则：
- RS > 8.5 → 💖
- 7 < RS ≤ 8.5 → 🧡
- 5 < RS ≤ 7 → 💛
- RS ≤ 5 → 💙
"""
if rs_score > Decimal("8.5"):
return "💖"
if rs_score > Decimal("7"):
return "🧡"
if rs_score > Decimal("5"):
return "💛"
return "💙"
# ── run() 主流程 ──────────────────────────────────────────────
import logging
logger = logging.getLogger(__name__)
def _get_connection():
"""延迟导入 get_connection，避免纯函数测试时触发模块级导入失败。"""
from app.database import get_connection
return get_connection()
def run() -> dict:
"""
任务生成器主流程。
1. 通过 auth.user_assistant_binding 获取所有已绑定助教
2. 对每个助教-客户对，通过 FDW 读取 WBI/NCI/RS 指数
3. 调用 determine_task_type() 确定任务类型
4. 检查已存在的 active 任务：相同 task_type → 跳过；
不同 task_type → 关闭旧任务 + 创建新任务 + 记录 history
5. 处理 follow_up_visit 的 48 小时滞留机制（expires_at 填充）
6. 更新 trigger_jobs 时间戳
返回: {"created": int, "replaced": int, "skipped": int}
"""
stats = {"created": 0, "replaced": 0, "skipped": 0}
conn = _get_connection()
try:
# ── 1. 获取所有已绑定助教 ──
with conn.cursor() as cur:
cur.execute(
"""
SELECT DISTINCT site_id, assistant_id
FROM auth.user_assistant_binding
WHERE assistant_id IS NOT NULL
"""
)
bindings = cur.fetchall()
conn.commit()
# ── 2. 逐助教处理 ──
for site_id, assistant_id in bindings:
try:
_process_assistant(conn, site_id, assistant_id, stats)
except Exception:
logger.exception(
"处理助教失败: site_id=%s, assistant_id=%s",
site_id,
assistant_id,
)
conn.rollback()
# ── 6. 更新 trigger_jobs 时间戳 ──
with conn.cursor() as cur:
cur.execute(
"""
UPDATE biz.trigger_jobs
SET last_run_at = NOW()
WHERE job_name = 'task_generator'
"""
)
conn.commit()
finally:
conn.close()
logger.info(
"任务生成器完成: created=%d, replaced=%d, skipped=%d",
stats["created"],
stats["replaced"],
stats["skipped"],
)
return stats
def _process_assistant(
conn, site_id: int, assistant_id: int, stats: dict
) -> None:
"""处理单个助教下所有客户-助教对的任务生成。"""
# 通过 FDW 读取该助教关联的客户指数数据
# 需要 SET LOCAL app.current_site_id 以启用 RLS
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"SET LOCAL app.current_site_id = %s", (str(site_id),)
)
# 读取 WBI（流失回赢指数）
cur.execute(
"""
SELECT member_id, COALESCE(display_score, 0)
FROM fdw_etl.v_dws_member_winback_index
"""
)
wbi_map = {row[0]: Decimal(str(row[1])) for row in cur.fetchall()}
# 读取 NCI（新客转化指数）
cur.execute(
"""
SELECT member_id, COALESCE(display_score, 0)
FROM fdw_etl.v_dws_member_newconv_index
"""
)
nci_map = {row[0]: Decimal(str(row[1])) for row in cur.fetchall()}
# 读取 RS（关系强度指数）— 按 assistant_id 过滤
cur.execute(
"""
SELECT member_id, COALESCE(rs_display, 0)
FROM fdw_etl.v_dws_member_assistant_relation_index
WHERE assistant_id = %s
""",
(assistant_id,),
)
rs_map = {row[0]: Decimal(str(row[1])) for row in cur.fetchall()}
conn.commit()
# 合并所有涉及的 member_id
all_member_ids = set(wbi_map.keys()) | set(nci_map.keys()) | set(rs_map.keys())
# 逐客户处理，每对独立事务
for member_id in all_member_ids:
try:
_process_member_task(
conn,
site_id,
assistant_id,
member_id,
wbi_map.get(member_id, Decimal("0")),
nci_map.get(member_id, Decimal("0")),
rs_map.get(member_id, Decimal("0")),
stats,
)
except Exception:
logger.exception(
"处理客户任务失败: site_id=%s, assistant_id=%s, member_id=%s",
site_id,
assistant_id,
member_id,
)
conn.rollback()
def _process_member_task(
conn,
site_id: int,
assistant_id: int,
member_id: int,
wbi: Decimal,
nci: Decimal,
rs: Decimal,
stats: dict,
) -> None:
"""
处理单个客户-助教对的任务生成/更新。
每对独立事务，失败不影响其他。
"""
index_data = IndexData(
site_id=site_id,
assistant_id=assistant_id,
member_id=member_id,
wbi=wbi,
nci=nci,
rs=rs,
# follow_up_visit 条件由外部传入；当前简化：不自动生成 follow_up_visit
# （follow_up_visit 由召回完成检测器触发，不在 task_generator 主动生成）
has_active_recall=True,
has_follow_up_note=True,
)
new_task_type = determine_task_type(index_data)
if new_task_type is None:
# 不满足任何条件 → 检查是否有需要填充 expires_at 的 follow_up_visit
_handle_no_task_condition(conn, site_id, assistant_id, member_id)
stats["skipped"] += 1
return
priority_score = max(wbi, nci)
with conn.cursor() as cur:
cur.execute("BEGIN")
# ── 4. 检查已存在的 active 任务 ──
cur.execute(
"""
SELECT id, task_type, expires_at, created_at
FROM biz.coach_tasks
WHERE site_id = %s
AND assistant_id = %s
AND member_id = %s
AND status = 'active'
ORDER BY created_at DESC
""",
(site_id, assistant_id, member_id),
)
existing_tasks = cur.fetchall()
# 检查是否已有相同 task_type 的 active 任务
same_type_exists = any(row[1] == new_task_type for row in existing_tasks)
if same_type_exists:
# 相同 task_type → 跳过
conn.commit()
stats["skipped"] += 1
return
# 不同 task_type 的 active 任务 → 关闭旧任务 + 创建新任务
for task_id, old_type, old_expires_at, old_created_at in existing_tasks:
if should_replace_task(old_type, new_task_type):
# 特殊处理：旧任务是 follow_up_visit → 填充 expires_at 而非直接 inactive
if old_type == "follow_up_visit" and old_expires_at is None:
# 需求 5.2: follow_up_visit 被高优先级任务顶替时，
# 填充 expires_at = created_at + 48h，保持 active
cur.execute(
"""
UPDATE biz.coach_tasks
SET expires_at = created_at + INTERVAL '48 hours',
updated_at = NOW()
WHERE id = %s
""",
(task_id,),
)
_insert_history(
cur,
task_id,
action="expires_at_filled",
old_status="active",
new_status="active",
old_task_type=old_type,
new_task_type=old_type,
detail={"reason": "higher_priority_task_created"},
)
else:
# 关闭旧任务
cur.execute(
"""
UPDATE biz.coach_tasks
SET status = 'inactive', updated_at = NOW()
WHERE id = %s
""",
(task_id,),
)
_insert_history(
cur,
task_id,
action="type_change_close",
old_status="active",
new_status="inactive",
old_task_type=old_type,
new_task_type=new_task_type,
)
stats["replaced"] += 1
# ── 创建新任务 ──
# follow_up_visit 生成时 expires_at = NULL（需求 4.1）
expires_at_val = None
# 需求 4.4: 若新任务是 follow_up_visit 且已存在有 expires_at 的旧 follow_up_visit
# → 旧任务已在上面被标记为 inactive，新任务 expires_at = NULL
cur.execute(
"""
INSERT INTO biz.coach_tasks
(site_id, assistant_id, member_id, task_type, status,
priority_score, expires_at, parent_task_id)
VALUES (%s, %s, %s, %s, 'active', %s, %s, %s)
RETURNING id
""",
(
site_id,
assistant_id,
member_id,
new_task_type,
float(priority_score),
expires_at_val,
# parent_task_id: 关联最近被关闭的旧任务（如有）
existing_tasks[0][0] if existing_tasks else None,
),
)
new_task_id = cur.fetchone()[0]
_insert_history(
cur,
new_task_id,
action="created",
old_status=None,
new_status="active",
old_task_type=existing_tasks[0][1] if existing_tasks else None,
new_task_type=new_task_type,
)
stats["created"] += 1
conn.commit()
def _handle_no_task_condition(
conn, site_id: int, assistant_id: int, member_id: int
) -> None:
"""
当不满足任何任务生成条件时，检查是否有 follow_up_visit 需要填充 expires_at。
需求 4.2: 当 follow_up_visit 的触发条件不再满足时，
填充 expires_at = created_at + 48h。
"""
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"""
SELECT id, expires_at
FROM biz.coach_tasks
WHERE site_id = %s
AND assistant_id = %s
AND member_id = %s
AND task_type = 'follow_up_visit'
AND status = 'active'
AND expires_at IS NULL
""",
(site_id, assistant_id, member_id),
)
rows = cur.fetchall()
for task_id, _ in rows:
cur.execute(
"""
UPDATE biz.coach_tasks
SET expires_at = created_at + INTERVAL '48 hours',
updated_at = NOW()
WHERE id = %s
""",
(task_id,),
)
_insert_history(
cur,
task_id,
action="expires_at_filled",
old_status="active",
new_status="active",
detail={"reason": "condition_no_longer_met"},
)
conn.commit()
def _insert_history(
cur,
task_id: int,
action: str,
old_status: str | None = None,
new_status: str | None = None,
old_task_type: str | None = None,
new_task_type: str | None = None,
detail: dict | None = None,
) -> None:
"""在 coach_task_history 中记录变更。"""
import json
cur.execute(
"""
INSERT INTO biz.coach_task_history
(task_id, action, old_status, new_status,
old_task_type, new_task_type, detail)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""",
(
task_id,
action,
old_status,
new_status,
old_task_type,
new_task_type,
json.dumps(detail) if detail else None,
),
)

395
apps/backend/app/services/task_manager.py Normal file
View File

@@ -0,0 +1,395 @@
"""
任务管理服务
负责任务 CRUD、置顶、放弃、取消放弃等操作。
通过 FDW 读取客户信息和 RS 指数，计算爱心 icon 档位。
"""
import json
import logging
from decimal import Decimal
from fastapi import HTTPException
from app.services.task_generator import compute_heart_icon
logger = logging.getLogger(__name__)
def _get_connection():
"""延迟导入 get_connection，避免纯函数测试时触发模块级导入失败。"""
from app.database import get_connection
return get_connection()
def _record_history(
cur,
task_id: int,
action: str,
old_status: str | None = None,
new_status: str | None = None,
old_task_type: str | None = None,
new_task_type: str | None = None,
detail: dict | None = None,
) -> None:
"""在 coach_task_history 中记录变更。"""
cur.execute(
"""
INSERT INTO biz.coach_task_history
(task_id, action, old_status, new_status,
old_task_type, new_task_type, detail)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""",
(
task_id,
action,
old_status,
new_status,
old_task_type,
new_task_type,
json.dumps(detail) if detail else None,
),
)
def _get_assistant_id(conn, user_id: int, site_id: int) -> int:
"""
通过 user_assistant_binding 获取 assistant_id。
找不到绑定关系时抛出 403。
"""
with conn.cursor() as cur:
cur.execute(
"""
SELECT assistant_id
FROM auth.user_assistant_binding
WHERE user_id = %s AND site_id = %s AND assistant_id IS NOT NULL
LIMIT 1
""",
(user_id, site_id),
)
row = cur.fetchone()
if not row:
raise HTTPException(status_code=403, detail="权限不足")
return row[0]
def _verify_task_ownership(
conn, task_id: int, assistant_id: int, site_id: int, required_status: str | None = None
) -> dict:
"""
验证任务归属并返回任务信息。
- 任务不存在 → 404
- 不属于当前助教 → 403
- required_status 不匹配 → 409
"""
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, task_type, status, is_pinned, abandon_reason,
assistant_id, site_id
FROM biz.coach_tasks
WHERE id = %s
""",
(task_id,),
)
row = cur.fetchone()
if not row:
raise HTTPException(status_code=404, detail="资源不存在")
task = {
"id": row[0],
"task_type": row[1],
"status": row[2],
"is_pinned": row[3],
"abandon_reason": row[4],
"assistant_id": row[5],
"site_id": row[6],
}
if task["site_id"] != site_id or task["assistant_id"] != assistant_id:
raise HTTPException(status_code=403, detail="权限不足")
if required_status and task["status"] != required_status:
raise HTTPException(status_code=409, detail="任务状态不允许此操作")
return task
async def get_task_list(user_id: int, site_id: int) -> list[dict]:
"""
获取助教的活跃任务列表。
1. 通过 auth.user_assistant_binding 获取 assistant_id
2. 查询 biz.coach_tasks WHERE status='active'
3. 通过 FDW 读取客户基本信息（dim_member）和 RS 指数
4. 计算爱心 icon 档位
5. 排序：is_pinned DESC, priority_score DESC, created_at ASC
FDW 查询需要 SET LOCAL app.current_site_id。
"""
conn = _get_connection()
try:
assistant_id = _get_assistant_id(conn, user_id, site_id)
# 查询活跃任务
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, task_type, status, priority_score, is_pinned,
expires_at, created_at, member_id
FROM biz.coach_tasks
WHERE site_id = %s
AND assistant_id = %s
AND status = 'active'
ORDER BY is_pinned DESC, priority_score DESC NULLS LAST, created_at ASC
""",
(site_id, assistant_id),
)
tasks = cur.fetchall()
conn.commit()
if not tasks:
return []
member_ids = list({t[7] for t in tasks})
# 通过 FDW 读取客户信息和 RS 指数（需要 SET LOCAL app.current_site_id）
member_info_map: dict[int, dict] = {}
rs_map: dict[int, Decimal] = {}
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"SET LOCAL app.current_site_id = %s", (str(site_id),)
)
# 读取客户基本信息
cur.execute(
"""
SELECT member_id, member_name, member_phone
FROM fdw_etl.v_dim_member
WHERE member_id = ANY(%s)
""",
(member_ids,),
)
for row in cur.fetchall():
member_info_map[row[0]] = {
"member_name": row[1],
"member_phone": row[2],
}
# 读取 RS 指数
cur.execute(
"""
SELECT member_id, COALESCE(rs_display, 0)
FROM fdw_etl.v_dws_member_assistant_relation_index
WHERE assistant_id = %s
AND member_id = ANY(%s)
""",
(assistant_id, member_ids),
)
for row in cur.fetchall():
rs_map[row[0]] = Decimal(str(row[1]))
conn.commit()
# 组装结果
result = []
for task_row in tasks:
(task_id, task_type, status, priority_score,
is_pinned, expires_at, created_at, member_id) = task_row
info = member_info_map.get(member_id, {})
rs_score = rs_map.get(member_id, Decimal("0"))
heart_icon = compute_heart_icon(rs_score)
result.append({
"id": task_id,
"task_type": task_type,
"status": status,
"priority_score": float(priority_score) if priority_score else None,
"is_pinned": is_pinned,
"expires_at": expires_at.isoformat() if expires_at else None,
"created_at": created_at.isoformat() if created_at else None,
"member_id": member_id,
"member_name": info.get("member_name"),
"member_phone": info.get("member_phone"),
"rs_score": float(rs_score),
"heart_icon": heart_icon,
})
return result
finally:
conn.close()
async def pin_task(task_id: int, user_id: int, site_id: int) -> dict:
"""
置顶任务。
验证任务归属后设置 is_pinned=TRUE，记录 history。
"""
conn = _get_connection()
try:
assistant_id = _get_assistant_id(conn, user_id, site_id)
task = _verify_task_ownership(
conn, task_id, assistant_id, site_id, required_status="active"
)
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"""
UPDATE biz.coach_tasks
SET is_pinned = TRUE, updated_at = NOW()
WHERE id = %s
""",
(task_id,),
)
_record_history(
cur,
task_id,
action="pin",
old_status="active",
new_status="active",
old_task_type=task["task_type"],
new_task_type=task["task_type"],
detail={"is_pinned": True},
)
conn.commit()
return {"id": task_id, "is_pinned": True}
finally:
conn.close()
async def unpin_task(task_id: int, user_id: int, site_id: int) -> dict:
"""
取消置顶。
验证任务归属后设置 is_pinned=FALSE。
"""
conn = _get_connection()
try:
assistant_id = _get_assistant_id(conn, user_id, site_id)
task = _verify_task_ownership(
conn, task_id, assistant_id, site_id, required_status="active"
)
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"""
UPDATE biz.coach_tasks
SET is_pinned = FALSE, updated_at = NOW()
WHERE id = %s
""",
(task_id,),
)
conn.commit()
return {"id": task_id, "is_pinned": False}
finally:
conn.close()
async def abandon_task(
task_id: int, user_id: int, site_id: int, reason: str
) -> dict:
"""
放弃任务。
1. 验证 reason 非空（空或纯空白 → 422）
2. 验证任务归属和 status='active'
3. 设置 status='abandoned', abandon_reason=reason
4. 记录 coach_task_history
"""
if not reason or not reason.strip():
raise HTTPException(status_code=422, detail="放弃原因不能为空")
conn = _get_connection()
try:
assistant_id = _get_assistant_id(conn, user_id, site_id)
task = _verify_task_ownership(
conn, task_id, assistant_id, site_id, required_status="active"
)
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"""
UPDATE biz.coach_tasks
SET status = 'abandoned',
abandon_reason = %s,
updated_at = NOW()
WHERE id = %s
""",
(reason, task_id),
)
_record_history(
cur,
task_id,
action="abandon",
old_status="active",
new_status="abandoned",
old_task_type=task["task_type"],
new_task_type=task["task_type"],
detail={"abandon_reason": reason},
)
conn.commit()
return {"id": task_id, "status": "abandoned"}
finally:
conn.close()
async def cancel_abandon(task_id: int, user_id: int, site_id: int) -> dict:
"""
取消放弃。
1. 验证任务归属和 status='abandoned'
2. 恢复 status='active', 清空 abandon_reason
3. 记录 coach_task_history
"""
conn = _get_connection()
try:
assistant_id = _get_assistant_id(conn, user_id, site_id)
task = _verify_task_ownership(
conn, task_id, assistant_id, site_id, required_status="abandoned"
)
with conn.cursor() as cur:
cur.execute("BEGIN")
cur.execute(
"""
UPDATE biz.coach_tasks
SET status = 'active',
abandon_reason = NULL,
updated_at = NOW()
WHERE id = %s
""",
(task_id,),
)
_record_history(
cur,
task_id,
action="cancel_abandon",
old_status="abandoned",
new_status="active",
old_task_type=task["task_type"],
new_task_type=task["task_type"],
)
conn.commit()
return {"id": task_id, "status": "active"}
finally:
conn.close()

55
apps/backend/app/services/task_queue.py
View File

@@ -16,6 +16,7 @@ from __future__ import annotations
import asyncio
import json
import logging
import platform
import uuid
from dataclasses import dataclass, field
from typing import Any
@@ -25,6 +26,11 @@ from ..schemas.tasks import TaskConfigSchema
logger = logging.getLogger(__name__)
# CHANGE 2026-03-07 | 实例标识：用于多后端实例共享同一 DB 时的任务隔离
# 背景：发现有另一台机器（宿主机 D 盘）的后端也在消费同一个 task_queue，
# 导致任务被错误实例执行。通过 enqueued_by 列实现"谁入队谁消费"。
_INSTANCE_ID = platform.node()
# 后台循环轮询间隔（秒）
POLL_INTERVAL_SECONDS = 2
@@ -43,6 +49,7 @@ class QueuedTask:
finished_at: Any = None
exit_code: int | None = None
error_message: str | None = None
schedule_id: str | None = None
class TaskQueue:
@@ -56,12 +63,13 @@ class TaskQueue:
# 入队
# ------------------------------------------------------------------
def enqueue(self, config: TaskConfigSchema, site_id: int) -> str:
def enqueue(self, config: TaskConfigSchema, site_id: int, schedule_id: str | None = None) -> str:
"""将任务配置入队，自动分配 position。
Args:
config: 任务配置
site_id: 门店 ID（门店隔离）
schedule_id: 关联的调度任务 ID（可选）
Returns:
新创建的队列任务 ID（UUID 字符串）
@@ -84,18 +92,19 @@ class TaskQueue:
max_pos = cur.fetchone()[0]
new_pos = max_pos + 1
# CHANGE 2026-03-07 | 写入 enqueued_by 实现多实例任务隔离
cur.execute(
"""
INSERT INTO task_queue (id, site_id, config, status, position)
VALUES (%s, %s, %s, 'pending', %s)
INSERT INTO task_queue (id, site_id, config, status, position, schedule_id, enqueued_by)
VALUES (%s, %s, %s, 'pending', %s, %s, %s)
""",
(task_id, site_id, json.dumps(config_json), new_pos),
(task_id, site_id, json.dumps(config_json), new_pos, schedule_id, _INSTANCE_ID),
)
conn.commit()
finally:
conn.close()
logger.info("任务入队 [%s] site_id=%s position=%s", task_id, site_id, new_pos)
logger.info("任务入队 [%s] site_id=%s position=%s schedule_id=%s", task_id, site_id, new_pos, schedule_id)
return task_id
# ------------------------------------------------------------------
@@ -114,19 +123,21 @@ class TaskQueue:
conn = get_connection()
try:
with conn.cursor() as cur:
# 选取 position 最小的 pending 任务并锁定
# CHANGE 2026-03-07 | 只消费本实例入队的任务（enqueued_by 匹配）
# 背景：多后端实例共享同一 DB 时，防止 A 实例消费 B 实例入队的任务
cur.execute(
"""
SELECT id, site_id, config, status, position,
created_at, started_at, finished_at,
exit_code, error_message
exit_code, error_message, schedule_id
FROM task_queue
WHERE site_id = %s AND status = 'pending'
AND (enqueued_by = %s OR enqueued_by IS NULL)
ORDER BY position ASC
LIMIT 1
FOR UPDATE SKIP LOCKED
""",
(site_id,),
(site_id, _INSTANCE_ID),
)
row = cur.fetchone()
if row is None:
@@ -144,6 +155,7 @@ class TaskQueue:
finished_at=row[7],
exit_code=row[8],
error_message=row[9],
schedule_id=str(row[10]) if row[10] else None,
)
# 更新状态为 running
@@ -261,10 +273,11 @@ class TaskQueue:
# ------------------------------------------------------------------
def list_pending(self, site_id: int) -> list[QueuedTask]:
"""列出指定门店的所有 pending 任务，按 position 升序。"""
"""列出指定门店的所有 pending 任务（仅限本实例入队的），按 position 升序。"""
conn = get_connection()
try:
with conn.cursor() as cur:
# CHANGE 2026-03-07 | 只列出本实例入队的 pending 任务
cur.execute(
"""
SELECT id, site_id, config, status, position,
@@ -272,9 +285,10 @@ class TaskQueue:
exit_code, error_message
FROM task_queue
WHERE site_id = %s AND status = 'pending'
AND (enqueued_by = %s OR enqueued_by IS NULL)
ORDER BY position ASC
""",
(site_id,),
(site_id, _INSTANCE_ID),
)
rows = cur.fetchall()
conn.commit()
@@ -298,18 +312,20 @@ class TaskQueue:
]
def has_running(self, site_id: int) -> bool:
"""检查指定门店是否有 running 状态的任务。"""
"""检查指定门店是否有本实例的 running 状态任务。"""
conn = get_connection()
try:
with conn.cursor() as cur:
# CHANGE 2026-03-07 | 只检查本实例的 running 任务
cur.execute(
"""
SELECT EXISTS(
SELECT 1 FROM task_queue
WHERE site_id = %s AND status = 'running'
AND (enqueued_by = %s OR enqueued_by IS NULL)
)
""",
(site_id,),
(site_id, _INSTANCE_ID),
)
result = cur.fetchone()[0]
conn.commit()
@@ -333,7 +349,10 @@ class TaskQueue:
from .task_executor import task_executor
self._running = True
logger.info("TaskQueue process_loop 启动")
logger.info(
"TaskQueue process_loop 启动 (instance_id=%s，仅消费本实例入队的任务)",
_INSTANCE_ID,
)
while self._running:
try:
@@ -369,6 +388,7 @@ class TaskQueue:
asyncio.create_task(
self._execute_and_update(
executor, config, execution_id, task.id, site_id,
schedule_id=task.schedule_id,
)
)
@@ -379,6 +399,7 @@ class TaskQueue:
execution_id: str,
queue_id: str,
site_id: int,
schedule_id: str | None = None,
) -> None:
"""执行任务并更新队列状态。"""
try:
@@ -387,6 +408,7 @@ class TaskQueue:
execution_id=execution_id,
queue_id=queue_id,
site_id=site_id,
schedule_id=schedule_id,
)
# 执行完成后根据 executor 的结果更新 task_queue 状态
self._update_queue_status_from_log(queue_id)
@@ -395,15 +417,18 @@ class TaskQueue:
self._mark_failed(queue_id, "执行过程中发生未捕获异常")
def _get_pending_site_ids(self) -> list[int]:
"""获取所有有 pending 任务的 site_id 列表。"""
"""获取所有有 pending 任务的 site_id 列表（仅限本实例入队的）。"""
conn = get_connection()
try:
with conn.cursor() as cur:
# CHANGE 2026-03-07 | 只查本实例入队的 pending 任务
cur.execute(
"""
SELECT DISTINCT site_id FROM task_queue
WHERE status = 'pending'
"""
AND (enqueued_by = %s OR enqueued_by IS NULL)
""",
(_INSTANCE_ID,),
)
rows = cur.fetchall()
conn.commit()

2
apps/backend/app/services/task_registry.py
View File

@@ -191,6 +191,8 @@ DWD_TABLES: list[DwdTableDefinition] = [
DwdTableDefinition("dwd.dim_goods_category", "商品分类维度", "商品", "ods.stock_goods_category_tree", is_dimension=True),
DwdTableDefinition("dwd.dim_groupbuy_package", "团购套餐维度", "团购", "ods.group_buy_packages", is_dimension=True),
DwdTableDefinition("dwd.dim_groupbuy_package_ex", "团购套餐维度（扩展）", "团购", "ods.group_buy_packages", is_dimension=True),
DwdTableDefinition("dwd.dim_staff", "员工维度", "人事", "ods.staff_info_master", is_dimension=True),
DwdTableDefinition("dwd.dim_staff_ex", "员工维度（扩展）", "人事", "ods.staff_info_master", is_dimension=True),
# 事实表
DwdTableDefinition("dwd.dwd_settlement_head", "结算主表", "结算", "ods.settlement_records"),
DwdTableDefinition("dwd.dwd_settlement_head_ex", "结算主表（扩展）", "结算", "ods.settlement_records"),

161
apps/backend/app/services/trigger_scheduler.py Normal file
View File

@@ -0,0 +1,161 @@
# -*- coding: utf-8 -*-
"""
触发器调度框架（Trigger Scheduler）
统一管理 cron/interval/event 三种触发方式，驱动后台任务执行。
- cron/interval 类型通过 check_scheduled_jobs() 轮询 next_run_at 触发
- event 类型通过 fire_event() 方法直接触发
- 每个 job 独立事务，失败不影响其他触发器
"""
from __future__ import annotations
import logging
from datetime import datetime, timedelta, timezone
from typing import Any, Callable
logger = logging.getLogger(__name__)
def _get_connection():
"""延迟导入 get_connection，避免纯函数测试时触发模块级导入失败。"""
from app.database import get_connection
return get_connection()
# job_type → 执行函数的注册表
_JOB_REGISTRY: dict[str, Callable] = {}
def register_job(job_type: str, handler: Callable) -> None:
"""注册 job_type 对应的执行函数。"""
_JOB_REGISTRY[job_type] = handler
def fire_event(event_name: str, payload: dict[str, Any] | None = None) -> int:
"""
触发事件驱动型任务。
查找 trigger_condition='event' 且 trigger_config.event_name 匹配的 enabled job，
立即执行对应的 handler。
返回: 执行的 job 数量
"""
conn = _get_connection()
executed = 0
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, job_type, job_name
FROM biz.trigger_jobs
WHERE status = 'enabled'
AND trigger_condition = 'event'
AND trigger_config->>'event_name' = %s
""",
(event_name,),
)
rows = cur.fetchall()
for job_id, job_type, job_name in rows:
handler = _JOB_REGISTRY.get(job_type)
if not handler:
logger.warning(
"未注册的 job_type: %s (job_name=%s)", job_type, job_name
)
continue
try:
handler(payload=payload)
executed += 1
# 更新 last_run_at
with conn.cursor() as cur:
cur.execute(
"UPDATE biz.trigger_jobs SET last_run_at = NOW() WHERE id = %s",
(job_id,),
)
conn.commit()
except Exception:
logger.exception("触发器 %s 执行失败", job_name)
conn.rollback()
finally:
conn.close()
return executed
def check_scheduled_jobs() -> int:
"""
检查 cron/interval 类型的到期 job 并执行。
由 Scheduler 后台循环调用。
返回: 执行的 job 数量
"""
conn = _get_connection()
executed = 0
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, job_type, job_name, trigger_condition, trigger_config
FROM biz.trigger_jobs
WHERE status = 'enabled'
AND trigger_condition IN ('cron', 'interval')
AND (next_run_at IS NULL OR next_run_at <= NOW())
ORDER BY next_run_at ASC NULLS FIRST
""",
)
rows = cur.fetchall()
for job_id, job_type, job_name, trigger_condition, trigger_config in rows:
handler = _JOB_REGISTRY.get(job_type)
if not handler:
logger.warning("未注册的 job_type: %s", job_type)
continue
try:
handler()
executed += 1
# 计算 next_run_at 并更新
next_run = _calculate_next_run(trigger_condition, trigger_config)
with conn.cursor() as cur:
cur.execute(
"""
UPDATE biz.trigger_jobs
SET last_run_at = NOW(), next_run_at = %s
WHERE id = %s
""",
(next_run, job_id),
)
conn.commit()
except Exception:
logger.exception("触发器 %s 执行失败", job_name)
conn.rollback()
finally:
conn.close()
return executed
def _calculate_next_run(
trigger_condition: str, trigger_config: dict
) -> datetime | None:
"""
根据触发条件和配置计算下次运行时间。
- interval: now + interval_seconds
- cron: 复用 scheduler._parse_simple_cron
- event: 返回 None（无 next_run_at）
"""
now = datetime.now(timezone.utc)
if trigger_condition == "interval":
seconds = trigger_config.get("interval_seconds", 3600)
return now + timedelta(seconds=seconds)
elif trigger_condition == "cron":
# 延迟导入：支持从 monorepo 根目录和 apps/backend/ 两种路径导入
try:
from app.services.scheduler import _parse_simple_cron
except ModuleNotFoundError:
from apps.backend.app.services.scheduler import _parse_simple_cron
return _parse_simple_cron(
trigger_config.get("cron_expression", "0 4 * * *"), now
)
return None # event 类型无 next_run_at

6
apps/backend/app/services/wechat.py
View File

@@ -12,7 +12,7 @@ import logging
import httpx
from app.config import get
from app.config import WX_APPID, WX_SECRET
logger = logging.getLogger(__name__)
@@ -59,8 +59,8 @@ async def code2session(code: str) -> dict:
WeChatAuthError: 微信接口返回非零 errcode 时抛出
RuntimeError: WX_APPID / WX_SECRET 环境变量缺失时抛出
"""
appid = get("WX_APPID", "")
secret = get("WX_SECRET", "")
appid = WX_APPID
secret = WX_SECRET
if not appid or not secret:
raise RuntimeError("微信配置缺失：WX_APPID 或 WX_SECRET 未设置")

101
apps/backend/docs/API-REFERENCE.md
View File

@@ -51,8 +51,8 @@
}
```
说明：
- 首次登录自动创建 `auth.users` 记录（status=pending）
- pending 用户获得受限令牌（`limited=True`），仅可访问申请相关端点
- 首次登录自动创建 `auth.users` 记录（status=new），前端引导至申请页
- new/pending/rejected 用户获得受限令牌（`limited=True`），仅可访问申请相关端点
- approved 用户获得完整令牌，包含 `site_id` 和 `roles`
### POST `/api/xcx-auth/apply`
@@ -114,6 +114,17 @@
{ "refresh_token": "..." }
```
### POST `/api/xcx-auth/dev-login`
开发模式 mock 登录（仅 `WX_DEV_MODE=true` 时注册）。
请求体：
```json
{ "openid": "模拟openid", "status": "approved" }
```
说明：
- `status` 可选，为空时保留已有用户当前状态，新用户默认 `new`
- 仅开发/测试环境可用
---
## 3. 任务配置 `/api/tasks`
@@ -326,8 +337,90 @@ MVP 全链路验证端点，从 `test."xcx-test"` 表读取数据。
### GET/POST `/api/wx-callback`
微信消息推送回调。GET 用于签名验证，POST 用于接收消息。
### POST `/api/member-birthday`
助教手动补录会员生日。
---
## 11. 管理端申请审核 `/api/admin/applications`
### GET `/api/admin/applications`
获取待审核申请列表。需管理后台 JWT。
### POST `/api/admin/applications/{id}/approve`
批准申请。
### POST `/api/admin/applications/{id}/reject`
拒绝申请。
---
## 12. 营业日配置 `/api/business-day`
### GET `/api/business-day/config`
获取营业日分割点配置（`BUSINESS_DAY_START_HOUR`）。
---
## 13. 小程序任务 `/api/xcx/tasks`
所有端点需 JWT（approved 状态）。
### GET `/api/xcx/tasks`
获取当前助教的活跃任务列表。
响应：`TaskListItem[]`
### POST `/api/xcx/tasks/{id}/pin`
置顶任务。
### POST `/api/xcx/tasks/{id}/unpin`
取消置顶。
### POST `/api/xcx/tasks/{id}/abandon`
放弃任务（需填写原因）。
请求体：
```json
{ "reason": "放弃原因" }
```
### POST `/api/xcx/tasks/{id}/cancel-abandon`
取消放弃，恢复为活跃状态。
---
## 14. 小程序备注 `/api/xcx/notes`
所有端点需 JWT（approved 状态）。
### POST `/api/xcx/notes`
创建备注（含星星评分，可选关联任务）。
请求体：
```json
{
"target_type": "member",
"target_id": 1,
"content": "备注内容",
"task_id": null,
"rating_service_willingness": 4,
"rating_revisit_likelihood": 3
}
```
### GET `/api/xcx/notes`
查询某目标的备注列表（按创建时间倒序）。
查询参数：
- `target_type`：目标类型（默认 `member`）
- `target_id`：目标 ID（必填）
### DELETE `/api/xcx/notes/{id}`
删除备注（验证归属后硬删除）。
---
## 15. 维客线索 `/api/member-retention-clue`
替代原 `member-birthday` 端点，提供维客线索管理能力。
### WebSocket `/ws/logs/{execution_id}`
实时日志推送。连接后自动接收指定执行的日志流。

1
apps/backend/pyproject.toml
View File

@@ -17,6 +17,7 @@ dependencies = [
"python-jose[cryptography]>=3.3",
"bcrypt>=4.0",
"psutil>=5.9",
"openai>=1.30",
]
[tool.uv.sources]

11
apps/etl/connectors/feiqiu/.env
View File

@@ -26,7 +26,7 @@ SCHEMA_ETL=meta
# API 配置（上游 SaaS API）
# ------------------------------------------------------------------------------
API_BASE=https://pc.ficoo.vip/apiprod/admin/v1/
API_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnQtdHlwZSI6IjQiLCJ1c2VyLXR5cGUiOiIxIiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjoiMTIiLCJyb2xlLWlkIjoiMTIiLCJ0ZW5hbnQtaWQiOiIyNzkwNjgzMTYwNzA5OTU3Iiwibmlja25hbWUiOiLnp5_miLfnrqHnkIblkZjvvJrmganmgakxIiwic2l0ZS1pZCI6IjAiLCJtb2JpbGUiOiIxMzgxMDUwMjMwNCIsInNpZCI6IjI5NTA0ODk2NTgzOTU4NDUiLCJzdGFmZi1pZCI6IjMwMDk5MTg2OTE1NTkwNDUiLCJvcmctaWQiOiIwIiwicm9sZS10eXBlIjoiMyIsInJlZnJlc2hUb2tlbiI6IjN4d3IwYjNWN01jemlvcFYyZnZibmtpMVg4MEhxNVFvOFRMcHh3RkNkQUk9IiwicmVmcmVzaEV4cGlyeVRpbWUiOiIyMDI2LzMvMSDkuIvljYgxMDo1MDozOCIsIm5lZWRDaGVja1Rva2VuIjoiZmFsc2UiLCJleHAiOjE3NzIzNzY2MzgsImlzcyI6InRlc3QiLCJhdWQiOiJVc2VyIn0.k_f4jnSGKOKPoZC22bVSrAo9A1FfRqvsNiGw-Vmc0qQ
API_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnQtdHlwZSI6IjQiLCJ1c2VyLXR5cGUiOiIxIiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjoiMTIiLCJyb2xlLWlkIjoiMTIiLCJ0ZW5hbnQtaWQiOiIyNzkwNjgzMTYwNzA5OTU3Iiwibmlja25hbWUiOiLnp5_miLfnrqHnkIblkZjvvJrmganmgakxIiwic2l0ZS1pZCI6IjAiLCJtb2JpbGUiOiIxMzgxMDUwMjMwNCIsInNpZCI6IjI5NTA0ODk2NTgzOTU4NDUiLCJzdGFmZi1pZCI6IjMwMDk5MTg2OTE1NTkwNDUiLCJvcmctaWQiOiIwIiwicm9sZS10eXBlIjoiMyIsInJlZnJlc2hUb2tlbiI6IlI5THQvRkVjSGZubkdiOTZJZ3lmdWhjaXU5WnIwREQrZFh1amhVY1RCSDQ9IiwicmVmcmVzaEV4cGlyeVRpbWUiOiIyMDI2LzMvMTEg5LiL5Y2INjo0MjozMSIsIm5lZWRDaGVja1Rva2VuIjoiZmFsc2UiLCJleHAiOjE3NzMyMjU3NTEsImlzcyI6InRlc3QiLCJhdWQiOiJVc2VyIn0.8H5V3W0NfGJrcYo9Ex-35D-SzxhC2tRaZGrgo2reYr4
API_TIMEOUT=20
API_PAGE_SIZE=200
API_RETRY_MAX=3
@@ -45,6 +45,13 @@ WRITE_PRETTY_JSON=true
# ------------------------------------------------------------------------------
PIPELINE_FLOW=FULL
# ------------------------------------------------------------------------------
# 管道限流配置（RateLimiter 请求间隔，秒）
# CHANGE 2026-03-06 | 从默认 5-20s 降至 0.1-2s，大幅缩短 ODS 请求耗时
# ------------------------------------------------------------------------------
PIPELINE_RATE_MIN=0.1
PIPELINE_RATE_MAX=2.0
# ------------------------------------------------------------------------------
# 时间窗口配置
# ------------------------------------------------------------------------------
@@ -166,7 +173,7 @@ DWD_FACT_UPSERT=true
# ------------------------------------------------------------------------------
# 任务列表配置
# ------------------------------------------------------------------------------
RUN_TASKS=PRODUCTS,TABLES,MEMBERS,ASSISTANTS,PACKAGES_DEF,ORDERS,PAYMENTS,REFUNDS,COUPON_USAGE,INVENTORY_CHANGE,TOPUPS,TABLE_DISCOUNT,ASSISTANT_ABOLISH,LEDGER
RUN_TASKS=PRODUCTS,TABLES,MEMBERS,ASSISTANTS,PACKAGES_DEF,ORDERS,PAYMENTS,REFUNDS,COUPON_USAGE,INVENTORY_CHANGE,TOPUPS,TABLE_DISCOUNT,LEDGER
INDEX_LOOKBACK_DAYS=60
# ------------------------------------------------------------------------------

12
apps/etl/connectors/feiqiu/api/client.py
View File

@@ -107,6 +107,11 @@ class APIClient:
"""
return self._post_json(endpoint, params)
# CHANGE [2026-03-06] intent: 补齐公共 post() 方法，UnifiedPipeline 详情拉取模式需要调用 self.api.post()
def post(self, endpoint: str, params: dict | None = None) -> dict:
"""发送 POST JSON 请求（与 get 相同，语义更明确的别名）。"""
return self._post_json(endpoint, params)
def _post_json(self, endpoint: str, payload: dict | None = None) -> dict:
if not self.base_url:
raise ValueError("API base_url 未配置")
@@ -292,3 +297,10 @@ class APIClient:
return v
return []
# AI_CHANGELOG:
# - 日期: 2026-03-06 08:37:26
# - Prompt: P20260306-083206
# - 直接原因: APIClient 缺少公共 post() 方法，UnifiedPipeline 详情拉取模式调用 self.api.post() 失败
# - 变更摘要: 新增 post() 作为 _post_json() 的公共别名，与已有 get() 对齐
# - 风险与验证: 极低风险，纯别名转发；166 个单元测试通过

43
apps/etl/connectors/feiqiu/api/rate_limiter.py Normal file
View File

@@ -0,0 +1,43 @@
"""请求间隔控制器，支持取消信号中断等待。"""
import random
import time
import threading
class RateLimiter:
"""请求间隔控制器，在相邻 API 请求之间插入随机等待时间，防止触发上游风控。
等待期间以 0.5s 为单位轮询 cancel_event，支持快速响应取消信号。
"""
def __init__(self, min_interval: float = 5.0, max_interval: float = 20.0):
if min_interval > max_interval:
raise ValueError(
f"min_interval({min_interval}) 不能大于 max_interval({max_interval})"
)
self._min = min_interval
self._max = max_interval
self._last_interval: float = 0.0
def wait(self, cancel_event: threading.Event | None = None) -> bool:
"""等待随机间隔。返回 False 表示被取消信号中断。
将等待时间拆分为 0.5s 小段，每段检查 cancel_event，
以便在取消信号到达时快速退出（最多延迟 0.5s）。
"""
interval = random.uniform(self._min, self._max)
self._last_interval = interval
remaining = interval
while remaining > 0:
if cancel_event and cancel_event.is_set():
return False
sleep_time = min(0.5, remaining)
time.sleep(sleep_time)
remaining -= sleep_time
return True
@property
def last_interval(self) -> float:
"""最近一次 wait() 生成的随机间隔值。"""
return self._last_interval

11
apps/etl/connectors/feiqiu/api/recording_client.py
View File

@@ -36,6 +36,11 @@ class RecordingAPIClient:
self.last_dump: dict[str, Any] | None = None
# ------------------------------------------------------------------ 公共 API
# CHANGE [2026-03-06] intent: 补齐 post() 代理，使 RecordingAPIClient 完整覆盖 APIClient 公共接口
def post(self, endpoint: str, params: dict | None = None) -> dict:
"""委托给底层 APIClient 的 post 方法（详情拉取等非分页请求使用）。"""
return self.base.post(endpoint, params)
def get_source_hint(self, endpoint: str) -> str:
"""Return the JSON dump path for this endpoint (for source_file lineage)."""
return str(self.output_dir / endpoint_to_filename(endpoint))
@@ -193,6 +198,12 @@ def build_recording_client(
# AI_CHANGELOG:
# - 日期: 2026-03-06 08:37:26
# - Prompt: P20260306-083206
# - 直接原因: RecordingAPIClient 缺少 post() 方法，UnifiedPipeline 详情拉取模式调用失败
# - 变更摘要: 新增 post() 方法委托给 self.base.post()，补齐代理接口覆盖
# - 风险与验证: 极低风险，纯委托转发；166 个单元测试通过
#
# - 日期: 2026-02-14
# - Prompt: P20260214-040231（审计收口补录）
# - 直接原因: 默认时区 Asia/Taipei 与运营地区（中国大陆）不符

36
apps/etl/connectors/feiqiu/cli/main.py
View File

@@ -282,6 +282,32 @@ def parse_args():
parser.add_argument("--idle-end", help="闲时窗口结束(HH:MM)")
parser.add_argument("--allow-empty-advance", action="store_true", help="允许空结果推进窗口")
# Pipeline 管道参数（覆盖 PipelineConfig 全局默认值）
parser.add_argument(
"--pipeline-workers",
dest="pipeline_workers",
type=int,
help="Pipeline 处理线程数（覆盖 pipeline.workers，默认 2）",
)
parser.add_argument(
"--pipeline-batch-size",
dest="pipeline_batch_size",
type=int,
help="Pipeline 批量写入阈值（覆盖 pipeline.batch_size，默认 100）",
)
parser.add_argument(
"--pipeline-rate-min",
dest="pipeline_rate_min",
type=float,
help="Pipeline 限流最小间隔秒数（覆盖 pipeline.rate_min，默认 5.0）",
)
parser.add_argument(
"--pipeline-rate-max",
dest="pipeline_rate_max",
type=float,
help="Pipeline 限流最大间隔秒数（覆盖 pipeline.rate_max，默认 20.0）",
)
# 强制全量更新（跳过 ODS hash 去重 + DWD 变更对比，无条件写入）
parser.add_argument(
"--force-full",
@@ -406,6 +432,16 @@ def build_cli_overrides(args) -> dict:
# 强制全量更新
if args.force_full:
overrides.setdefault("run", {})["force_full_update"] = True
# Pipeline 管道参数 → pipeline.* 命名空间（供 PipelineConfig.from_app_config() 读取）
if getattr(args, "pipeline_workers", None) is not None:
overrides.setdefault("pipeline", {})["workers"] = args.pipeline_workers
if getattr(args, "pipeline_batch_size", None) is not None:
overrides.setdefault("pipeline", {})["batch_size"] = args.pipeline_batch_size
if getattr(args, "pipeline_rate_min", None) is not None:
overrides.setdefault("pipeline", {})["rate_min"] = args.pipeline_rate_min
if getattr(args, "pipeline_rate_max", None) is not None:
overrides.setdefault("pipeline", {})["rate_max"] = args.pipeline_rate_max
# 任务
if args.tasks:

3
apps/etl/connectors/feiqiu/config/defaults.py
View File

@@ -4,6 +4,7 @@
DEFAULTS = {
"app": {
"timezone": "Asia/Shanghai",
"business_day_start_hour": 8,
"store_id": "",
# CHANGE 2026-02-15 | 对齐新库 etl_feiqiu 六层架构
"schema_oltp": "ods",
@@ -52,7 +53,6 @@ DEFAULTS = {
"INVENTORY_CHANGE",
"TOPUPS",
"TABLE_DISCOUNT",
"ASSISTANT_ABOLISH",
"LEDGER",
],
"dws_tasks": [],
@@ -178,5 +178,4 @@ TASK_TABLES = "TABLES"
TASK_PACKAGES_DEF = "PACKAGES_DEF"
TASK_TOPUPS = "TOPUPS"
TASK_TABLE_DISCOUNT = "TABLE_DISCOUNT"
TASK_ASSISTANT_ABOLISH = "ASSISTANT_ABOLISH"
TASK_LEDGER = "LEDGER"

4
apps/etl/connectors/feiqiu/config/env_parser.py
View File

@@ -7,6 +7,7 @@ from copy import deepcopy
ENV_MAP = {
"TIMEZONE": ("app.timezone",),
"BUSINESS_DAY_START_HOUR": ("app.business_day_start_hour",),
"STORE_ID": ("app.store_id",),
"SCHEMA_OLTP": ("app.schema_oltp",),
"SCHEMA_ETL": ("app.schema_etl",),
@@ -114,6 +115,9 @@ ENV_MAP = {
"DATA_SOURCE": ("run.data_source",),
# API 额外请求头（JSON 对象格式）
"API_HEADERS_EXTRA": ("api.headers_extra",),
# Pipeline 管道限流参数
"PIPELINE_RATE_MIN": ("pipeline.rate_min",),
"PIPELINE_RATE_MAX": ("pipeline.rate_max",),
}

75
apps/etl/connectors/feiqiu/config/pipeline_config.py Normal file
View File

@@ -0,0 +1,75 @@
# -*- coding: utf-8 -*-
"""统一管道配置数据类。
支持全局默认值 + 任务级覆盖的三级回退：
pipeline.<task_code>.* → pipeline.* → 硬编码默认值
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .settings import AppConfig
@dataclass(frozen=True)
class PipelineConfig:
"""统一管道配置，支持全局默认 + 任务级覆盖。"""
workers: int = 2 # ProcessingPool 工作线程数
queue_size: int = 100 # 处理队列容量
batch_size: int = 100 # WriteWorker 批量写入阈值
batch_timeout: float = 5.0 # WriteWorker 等待超时（秒）
rate_min: float = 0.1 # RateLimiter 最小间隔（秒）
rate_max: float = 2.0 # RateLimiter 最大间隔（秒）
max_consecutive_failures: int = 10 # 连续失败中断阈值
def __post_init__(self) -> None:
if self.workers < 1:
raise ValueError(f"workers 必须 >= 1，当前值: {self.workers}")
if self.queue_size < 1:
raise ValueError(f"queue_size 必须 >= 1，当前值: {self.queue_size}")
if self.batch_size < 1:
raise ValueError(f"batch_size 必须 >= 1，当前值: {self.batch_size}")
if self.rate_min > self.rate_max:
raise ValueError(
f"rate_min({self.rate_min}) 不能大于 rate_max({self.rate_max})"
)
@classmethod
def from_app_config(
cls,
config: AppConfig,
task_code: str | None = None,
) -> PipelineConfig:
"""从 AppConfig 加载，支持 pipeline.<task_code>.* 任务级覆盖。
回退优先级：
1. pipeline.<task_code_lower>.<key> （任务级，仅 task_code 非空时查找）
2. pipeline.<key> （全局级）
3. 字段硬编码默认值
"""
def _get(key: str, default): # noqa: ANN001
# 任务级覆盖
if task_code:
val = config.get(f"pipeline.{task_code.lower()}.{key}")
if val is not None:
return type(default)(val)
# 全局级
val = config.get(f"pipeline.{key}")
if val is not None:
return type(default)(val)
# 硬编码默认值
return default
return cls(
workers=_get("workers", 2),
queue_size=_get("queue_size", 100),
batch_size=_get("batch_size", 100),
batch_timeout=_get("batch_timeout", 5.0),
rate_min=_get("rate_min", 5.0),
rate_max=_get("rate_max", 20.0),
max_consecutive_failures=_get("max_consecutive_failures", 10),
)

6
apps/etl/connectors/feiqiu/config/settings.py
View File

@@ -111,6 +111,12 @@ class AppConfig:
missing.append("app.store_id")
if missing:
raise SystemExit("缺少必需配置: " + ", ".join(missing))
# business_day_start_hour 范围校验（0–23 整数）
hour = cfg["app"].get("business_day_start_hour", 8)
if not isinstance(hour, int) or not (0 <= hour <= 23):
raise SystemExit("app.business_day_start_hour 必须为 0–23 的整数")
def get(self, key: str, default=None):
"""获取配置值（支持点号路径）"""

10
apps/etl/connectors/feiqiu/database/connection.py
View File

@@ -20,7 +20,15 @@ class DatabaseConnection:
# 生产环境要求：数据库连接超时不得超过 20 秒。
timeout_val = max(1, min(int(timeout_val), 20))
conn = psycopg2.connect(self._dsn, connect_timeout=timeout_val)
# CHANGE 2026-03-06 | intent: 修复 Windows GBK 环境下 psycopg2 连接握手的 UnicodeDecodeError
# assumptions: libpq 默认使用系统 locale 的 client_encoding，Windows 中文系统为 GBK/CP936
# 边界: 显式指定 client_encoding=utf8 确保连接层始终使用 UTF-8，与数据库 server_encoding 一致
# 验证: web-admin 手动触发 ETL 全量 flow，不再出现 0xd6 解码错误
conn = psycopg2.connect(
self._dsn,
connect_timeout=timeout_val,
options="-c client_encoding=utf8",
)
conn.autocommit = False
# 会话参数（时区、语句超时等）

23
apps/etl/connectors/feiqiu/database/operations.py
View File

@@ -1,5 +1,9 @@
# -*- coding: utf-8 -*-
"""数据库批量操作"""
"""数据库批量操作
AI_CHANGELOG
- 2026-03-06 09:17:16 | Prompt: P20260306-084752（摘录：DWD 并行装载全部失败 _dsn 属性缺失）| Direct cause：DatabaseOperations 组合模式未透传 _dsn/_session/_connect_timeout | Summary：新增 3 个 property 透传底层 DatabaseConnection 属性 | Verify：334 单元测试通过 + getDiagnostics 无问题
"""
import psycopg2.extras
import re
@@ -9,6 +13,23 @@ class DatabaseOperations:
def __init__(self, connection):
self._connection = connection
self.conn = connection.conn
# [CHANGE P20260306-084752] intent: 透传底层 DatabaseConnection 的连接参数，
# DwdLoadTask._process_single_table 需要 _dsn/_session/_connect_timeout
# 为每个线程创建独立连接
# assumptions: _connection 始终是 DatabaseConnection 实例，具有这三个属性
# verify: 334 单元测试通过，DWD 并行装载不再 AttributeError
@property
def _dsn(self):
return self._connection._dsn
@property
def _session(self):
return self._connection._session
@property
def _connect_timeout(self):
return self._connection._connect_timeout
def batch_execute(self, sql: str, rows: list, page_size: int = 1000):
"""批量执行SQL"""

2
apps/etl/connectors/feiqiu/docs/business-rules/dws_metrics.md
View File

@@ -12,7 +12,7 @@
### 1.1 助教日报（dws_assistant_daily_detail）
- 目标表：`dws.dws_assistant_daily_detail`
- 数据来源：`dwd_assistant_service_log`、`dwd_assistant_trash_event`、`dim_assistant`（SCD2）
- 数据来源：`dwd_assistant_service_log`、`dwd_assistant_service_log_ex`（提供 `is_trash` 标记）、`dim_assistant`（SCD2）
- 粒度：门店 × 助教 × 日期
- 核心指标：服务次数（总/基础课/附加课/包厢课）、计费秒数与小时数、台账金额、去重客户数与台桌数、废除统计
- 课程类型分类：通过 `cfg_skill_type` 映射 `skill_id` → `BASE`/`BONUS`/`ROOM`

6
apps/etl/connectors/feiqiu/docs/database/DWD/Ex/BD_manual_dim_groupbuy_package_ex.md
View File

@@ -42,6 +42,12 @@
| 23 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 |
| 24 | scd2_is_current | INTEGER | YES | | 当前版本标记 |
| 25 | scd2_version | INTEGER | YES | | 版本号 |
| 26 | table_area_ids | JSONB | YES | | 可用台区 ID 列表（来自详情接口 tableAreaId） |
| 27 | table_area_names | JSONB | YES | | 可用台区名称列表（来自详情接口 tableAreaNameList） |
| 28 | assistant_services | JSONB | YES | | 助教服务关联数组（来自详情接口 packageCouponAssistants） |
| 29 | groupon_site_infos | JSONB | YES | | 关联门店信息数组（来自详情接口 grouponSiteInfos） |
> 字段 26-29 由迁移脚本 `db/etl_feiqiu/migrations/2026-03-05__add_detail_fields_to_dim_groupbuy_package_ex.sql` 新增，数据来源为 `ods.group_buy_package_details`（通过 LEFT JOIN `coupon_id = groupbuy_package_id` 合并）。
## 样本数据

2
apps/etl/connectors/feiqiu/docs/database/DWD/Ex/BD_manual_dwd_assistant_service_log_ex.md
View File

@@ -33,7 +33,7 @@
| 14 | is_confirm | INTEGER | YES | | 是否确认。**枚举值**: 2(5003)=**[待确认]** |
| 15 | is_single_order | INTEGER | YES | | 是否独立订单。**枚举值**: 1(5003)=是 |
| 16 | is_not_responding | INTEGER | YES | | 无响应。**枚举值**: 0(5003)=正常 |
| 17 | is_trash | INTEGER | YES | | 是否废单。**枚举值**: 0(5003)=正常 |
| 17 | is_trash | INTEGER | YES | | 是否废单。**枚举值**: 0=正常, 1=已作废。⚠️ 此字段是判断助教服务是否作废的唯一依据，替代已废弃的 `dwd_assistant_trash_event` 表（2026-02-22 DROP）。DWS 层助教日报等任务通过此字段过滤废单统计。 |
| 18 | trash_applicant_id | BIGINT | YES | | 废单申请人 ID（当前数据全为 0） |
| 19 | trash_applicant_name | VARCHAR(64) | YES | | 废单申请人姓名（当前数据全为空） |
| 20 | trash_reason | VARCHAR(255) | YES | | 废单原因（当前数据全为空） |

4
apps/etl/connectors/feiqiu/docs/database/DWD/main/BD_manual_billiards_dwd.md
View File

@@ -30,7 +30,7 @@
| 序号 | 表名 | 说明 | 主键 | 扩展表 | 文档链接 |
|------|------|------|------|--------|----------|
| 1 | dwd_assistant_service_log | 助教服务流水 | assistant_service_id | dwd_assistant_service_log_ex | [主表](BD_manual_dwd_assistant_service_log.md) / [扩展表](BD_manual_dwd_assistant_service_log_ex.md) |
| 2 | dwd_assistant_trash_event | 助教服务作废 | assistant_trash_event_id | dwd_assistant_trash_event_ex | [主表](BD_manual_dwd_assistant_trash_event.md) / [扩展表](BD_manual_dwd_assistant_trash_event_ex.md) |
| 2 | ~~dwd_assistant_trash_event~~ | ~~助教服务作废~~ | — | — | ⚠️ 已于 2026-02-22 废弃，作废判断改用 `dwd_assistant_service_log_ex.is_trash` |
| 3 | dwd_groupbuy_redemption | 团购券核销 | redemption_id | dwd_groupbuy_redemption_ex | [主表](BD_manual_dwd_groupbuy_redemption.md) / [扩展表](BD_manual_dwd_groupbuy_redemption_ex.md) |
| 4 | dwd_member_balance_change | 会员余额变动 | balance_change_id | dwd_member_balance_change_ex | [主表](BD_manual_dwd_member_balance_change.md) / [扩展表](BD_manual_dwd_member_balance_change_ex.md) |
| 5 | dwd_payment | 支付流水 | payment_id | 无 | [主表](BD_manual_dwd_payment.md) |
@@ -118,7 +118,7 @@ SELECT * FROM dwd.dwd_payment ORDER BY pay_time DESC NULLS LAST LIMIT 1;
| dwd_table_fee_adjust | 2,849 |
| dwd_assistant_service_log | 1,090 |
| dwd_recharge_order | 455 |
| dwd_assistant_trash_event | 98 |
| ~~dwd_assistant_trash_event~~ | ~~98~~ | ⚠️ 已废弃（2026-02-22） |
| dwd_refund | 45 |
---

2
apps/etl/connectors/feiqiu/docs/database/DWD/main/BD_manual_dim_table.md
View File

@@ -42,7 +42,7 @@
| M7 | 2 | 麻将/麻将棋牌 |
| M8 | 1 | 麻将/麻将棋牌 |
| K包 | 4 | K包/K歌/KTV |
| VIP包厢 | 4 | 台球/打球/中八/追分 (V5为 台球/打球/斯诺克) |
| VIP包厢 | 4 | 🎱 中式/追分 (V1-V4)、斯诺克 (V5) |
| 斯诺克区 | 4 | 台球/打球/斯诺克 |
| 666 | 2 | 麻将/麻将棋牌 |
| TV台 | 1 | 台球/打球/中八/追分 |

21
apps/etl/connectors/feiqiu/docs/database/DWD/main/BD_manual_dwd_settlement_head.md
View File

@@ -35,7 +35,7 @@
| 16 | member_card_type_name | VARCHAR(100) | YES | | 卡类型名称（当前数据全为空） |
| 17 | is_bind_member | BOOLEAN | YES | | 是否绑定会员。**枚举值**: False=否 |
| 18 | member_discount_amount | NUMERIC(18,2) | YES | | 会员折扣金额 |
| 19 | consume_money | NUMERIC(18,2) | YES | | 消费总金额（元） |
| 19 | consume_money | NUMERIC(18,2) | YES | | 消费总金额（元）。⚠️ **口径不稳定**：存在三种历史口径（A/B/C），DWS 层不应直接使用，应使用 `items_sum = table_charge_money + goods_money + assistant_pd_money + assistant_cx_money + electricity_money`。详见 [consume_money 口径](../../../../docs/reports/DWD-DOC/consume/consume-money-caliber.md) |
| 20 | table_charge_money | NUMERIC(18,2) | YES | | 台费金额 |
| 21 | goods_money | NUMERIC(18,2) | YES | | 商品金额 |
| 22 | real_goods_money | NUMERIC(18,2) | YES | | 实收商品金额 |
@@ -71,19 +71,30 @@ LIMIT 1;
```
**使用示例**
```sql
-- 每日营收统计
-- 每日营收统计（使用 items_sum 口径，不使用 consume_money）
SELECT
DATE(pay_time) AS pay_date,
COUNT(*) AS order_count,
SUM(consume_money) AS total_consume,
SUM(table_charge_money + goods_money + assistant_pd_money
+ assistant_cx_money + electricity_money) AS total_items_sum,
SUM(pay_amount) AS total_pay
FROM dwd.dwd_settlement_head
WHERE settle_type IN (1, 3)
GROUP BY DATE(pay_time)
ORDER BY pay_date DESC;
-- 台费 vs 商品 vs 助教收入
SELECT
SUM(table_charge_money) AS table_revenue,
SUM(goods_money) AS goods_revenue,
SUM(assistant_pd_money + assistant_cx_money) AS assistant_revenue
FROM dwd.dwd_settlement_head;
SUM(assistant_pd_money) AS assistant_pd_revenue,
SUM(assistant_cx_money) AS assistant_cx_revenue
FROM dwd.dwd_settlement_head
WHERE settle_type IN (1, 3);
```
**支付渠道恒等式（100% 成立）**
```
balance_amount = recharge_card_amount + gift_card_amount -- 储值卡 = 充值卡 + 礼品卡
pay_amount = point_amount + cash_amount -- 实付 = 积分 + 现金（互斥）
```
> `balance_amount` 是独立支付渠道，`recharge_card_amount`/`gift_card_amount` 是其分账明细，不可重复计算。

91
apps/etl/connectors/feiqiu/docs/database/DWS/main/BD_manual_cfg_area_category.md
View File

@@ -1,6 +1,6 @@
# cfg_area_category 台区分类映射表
> 生成时间：2026-02-03
> 生成时间：2026-02-03 | 更新时间：2026-03-07
## 表信息
@@ -9,8 +9,9 @@
| Schema | dws |
| 表名 | cfg_area_category |
| 主键 | category_id |
| 唯一约束 | (source_area_name, COALESCE(source_table_name, '')) |
| 数据来源 | 手工维护/seed脚本（基于dim_table实际数据） |
| 说明 | 将dim_table.site_table_area_name映射到财务报表区域分类 |
| 说明 | 将dim_table的台区/台桌映射到项目分类，支持台桌级细分 |
## 字段说明
@@ -18,57 +19,47 @@
|------|--------|------|------|------|------|
| 1 | category_id | SERIAL | NO | PK | 分类ID（自增） |
| 2 | source_area_name | VARCHAR(100) | NO | UK | 源区域名称（来自dim_table.site_table_area_name） |
| 3 | category_code | VARCHAR(20) | NO | | 分类代码。**枚举值**: BILLIARD, BILLIARD_VIP, SNOOKER, MAHJONG, KTV, SPECIAL, OTHER |
| 4 | category_name | VARCHAR(50) | NO | | 分类名称 |
| 5 | match_type | VARCHAR(10) | NO | | 匹配类型。**枚举值**: EXACT（精确）, LIKE（模糊）, DEFAULT（兜底） |
| 6 | match_priority | INTEGER | NO | | 匹配优先级（数字越小优先级越高） |
| 7 | is_active | BOOLEAN | NO | | 是否启用 |
| 8 | description | TEXT | YES | | 说明 |
| 9 | created_at | TIMESTAMPTZ | NO | | 创建时间 |
| 10 | updated_at | TIMESTAMPTZ | NO | | 更新时间 |
| 3 | source_table_name | VARCHAR(100) | YES | UK | 源台桌名称（来自dim_table.table_name），NULL表示区域级映射 |
| 4 | category_code | VARCHAR(20) | NO | | 分类代码。**枚举值**: BILLIARD, SNOOKER, MAHJONG, KTV, SPECIAL, OTHER |
| 5 | category_name | VARCHAR(50) | NO | | 分类名称（含emoji） |
| 6 | display_name | VARCHAR(50) | YES | | 显示名称（用于筛选器） |
| 7 | short_name | VARCHAR(20) | YES | | 简写（用于列表标签） |
| 8 | match_type | VARCHAR(10) | NO | | 匹配类型。**枚举值**: EXACT（精确）, LIKE（模糊）, DEFAULT（兜底） |
| 9 | match_priority | INTEGER | NO | | 匹配优先级（数字越小优先级越高） |
| 10 | is_active | BOOLEAN | NO | | 是否启用 |
| 11 | description | TEXT | YES | | 说明 |
| 12 | created_at | TIMESTAMPTZ | NO | | 创建时间 |
| 13 | updated_at | TIMESTAMPTZ | NO | | 更新时间 |
## 分类映射示例
## 变更说明（2026-03-07）
| 源区域名称 | 分类代码 | 分类名称 |
|------------|----------|----------|
| A区 | BILLIARD | 台球散台 |
| B区 | BILLIARD | 台球散台 |
| C区 | BILLIARD | 台球散台 |
| TV台 | BILLIARD | 台球散台 |
| VIP包厢 | BILLIARD_VIP | 台球VIP |
| 斯诺克区 | SNOOKER | 斯诺克 |
| 麻将房 | MAHJONG | 麻将棋牌 |
| M7 | MAHJONG | 麻将棋牌 |
| M8 | MAHJONG | 麻将棋牌 |
| 666 | MAHJONG | 麻将棋牌 |
| 发财 | MAHJONG | 麻将棋牌 |
| K包 | KTV | K歌娱乐 |
| k包活动区 | KTV | K歌娱乐 |
| 幸会158 | KTV | K歌娱乐 |
| 补时长 | SPECIAL | 补时长 |
### 新增字段
- `source_table_name`：支持台桌级细分映射（如 VIP包厢 V5 → SNOOKER）
- `display_name`：前端筛选器显示名称
- `short_name`：列表中的简写标签
## 使用说明
### 删除类型
- `BILLIARD_VIP` 已废弃，VIP包厢 V1-V4 归入 `BILLIARD`，V5 归入 `SNOOKER`
**取值方式**
### 唯一约束变更
- 从 `(source_area_name)` 改为 `(source_area_name, COALESCE(source_table_name, ''))`
```sql
-- 将台区名称映射到分类
SELECT
dt.site_table_area_name,
COALESCE(ac.category_code, 'OTHER') AS category_code,
COALESCE(ac.category_name, '其他') AS category_name
FROM dwd.dim_table dt
LEFT JOIN dws.cfg_area_category ac
ON dt.site_table_area_name = ac.source_area_name
AND ac.is_active = TRUE
WHERE dt.scd2_is_current = 1;
## 匹配优先级
-- 按分类汇总收入
SELECT
COALESCE(ac.category_name, '其他') AS category_name,
SUM(tfl.ledger_amount) AS total_amount
FROM dwd.dwd_table_fee_log tfl
LEFT JOIN dwd.dim_table dt ON dt.table_id = tfl.site_table_id
LEFT JOIN dws.cfg_area_category ac ON dt.site_table_area_name = ac.source_area_name
GROUP BY COALESCE(ac.category_name, '其他');
```
| 优先级 | 匹配方式 | 说明 |
|--------|---------|------|
| 5 | 台桌级精确 | source_area_name + source_table_name 都匹配 |
| 10 | 区域级精确 | source_area_name 匹配，source_table_name 为 NULL |
| 50 | 模糊匹配 | source_area_name 包含模式匹配 |
| 999 | 兜底 | 无法匹配的区域归入 OTHER |
## 分类映射
| 分类代码 | 显示名称 | 简写 | 源区域 |
|----------|---------|------|--------|
| BILLIARD | 🎱 中式/追分 | 🎱 | A区、B区、C区、TV台、VIP包厢(V1-V4) |
| SNOOKER | 斯诺克 | 斯 | 斯诺克区、VIP包厢(V5) |
| MAHJONG | 🀄 麻将/棋牌 | 🀄 | 麻将房、M7、M8、666、发财 |
| KTV | 🎤 团建/K歌 | 🎤 | K包、k包活动区、幸会158 |
| SPECIAL | 补时长 | 补 | 补时长 |
| OTHER | 其他 | 他 | 兜底 |

36
apps/etl/connectors/feiqiu/docs/database/DWS/main/BD_manual_dws_assistant_daily_detail.md
View File

@@ -10,7 +10,7 @@
| 表名 | dws_assistant_daily_detail |
| 主键 | id |
| 唯一键 | (site_id, assistant_id, stat_date) |
| 数据来源 | dwd_assistant_service_log + dwd_assistant_trash_event |
| 数据来源 | dwd_assistant_service_log + dwd_assistant_service_log_ex |
| 更新频率 | 每小时增量更新 |
| 说明 | 以"助教+日期"为粒度，汇总每日业绩明细 |
@@ -25,7 +25,7 @@
| 5 | assistant_nickname | VARCHAR(50) | YES | 助教花名（冗余，便于查询展示） |
| 6 | stat_date | DATE | NO | 统计日期 |
| 7 | assistant_level_code | INTEGER | YES | 助教等级代码（SCD2口径：取stat_date当日生效的等级） |
| 8 | assistant_level_name | VARCHAR(20) | YES | 助教等级名称 |
| 8 | assistant_level_name | VARCHAR(20) | YES | 助教等级名称（由 `level_code` 静态映射得出，不依赖 SCD2 返回值） |
| 9 | total_service_count | INTEGER | NO | 总服务次数 |
| 10 | base_service_count | INTEGER | NO | 基础课服务次数 |
| 11 | bonus_service_count | INTEGER | NO | 附加课服务次数 |
@@ -46,8 +46,12 @@
| 26 | unique_tables | INTEGER | NO | 服务台桌数（去重） |
| 27 | trashed_seconds | INTEGER | NO | 被废除的服务时长（秒） |
| 28 | trashed_count | INTEGER | NO | 被废除的服务次数 |
| 29 | created_at | TIMESTAMPTZ | NO | 创建时间 |
| 30 | updated_at | TIMESTAMPTZ | NO | 更新时间 |
| 29 | penalty_minutes | NUMERIC(10,2) | YES | 惩罚分钟数（定档折算）。公式：`actual_minutes × (1 - per_hour_contribution / 24)`，per_hour_contribution ≥ 24 时为 0 |
| 30 | penalty_reason | TEXT | YES | 惩罚原因描述（NULL=无违规） |
| 31 | is_exempt | BOOLEAN | NO | 是否豁免惩罚（豁免助教不计算惩罚） |
| 32 | per_hour_contribution | NUMERIC(10,2) | YES | 每小时贡献金额（= `base_ledger_amount / base_hours / overlap_count`，NULL=无违规或豁免） |
| 33 | created_at | TIMESTAMPTZ | NO | 创建时间 |
| 34 | updated_at | TIMESTAMPTZ | NO | 更新时间 |
## 数据来源
@@ -68,17 +72,21 @@ WHERE is_delete = 0
GROUP BY site_id, DATE(start_use_time), site_assistant_id, nickname;
```
### 废除记录：dwd_assistant_trash_event
### 废除记录：dwd_assistant_service_log_ex
> ⚠️ `dwd_assistant_trash_event` 已于 2026-02-22 废弃，作废判断改用 `dwd_assistant_service_log_ex.is_trash`（0=正常，1=作废）。
```sql
SELECT
site_id,
DATE(create_time) AS stat_date,
assistant_no,
assistant_name,
SUM(charge_minutes_raw * 60) AS trashed_seconds,
COUNT(*) AS trashed_count
FROM dwd.dwd_assistant_trash_event
GROUP BY site_id, DATE(create_time), assistant_no, assistant_name;
s.site_id,
DATE(s.start_use_time) AS stat_date,
s.site_assistant_id AS assistant_id,
SUM(CASE WHEN ex.is_trash = 1 THEN s.income_seconds ELSE 0 END) AS trashed_seconds,
COUNT(CASE WHEN ex.is_trash = 1 THEN 1 END) AS trashed_count
FROM dwd.dwd_assistant_service_log s
LEFT JOIN dwd.dwd_assistant_service_log_ex ex ON s.assistant_service_id = ex.assistant_service_id
WHERE s.is_delete = 0
GROUP BY s.site_id, DATE(s.start_use_time), s.site_assistant_id;
```
## 使用说明
@@ -115,4 +123,4 @@ GROUP BY assistant_id, DATE_TRUNC('month', stat_date);
|------|------|
| 可回溯 | ✅ 完全可回溯 |
| 数据范围 | 2025-07-21 ~ 至今 |
| 依赖表 | dwd_assistant_service_log, dwd_assistant_trash_event, dim_assistant |
| 依赖表 | dwd_assistant_service_log, dwd_assistant_service_log_ex, dim_assistant, dim_table, cfg_skill_type |

136
apps/etl/connectors/feiqiu/docs/database/DWS/main/BD_manual_dws_assistant_project_tag.md Normal file
View File

@@ -0,0 +1,136 @@
# dws_assistant_project_tag 助教项目标签表
> 生成时间：2026-03-07
## 表信息
| 属性 | 值 |
|------|-----|
| Schema | dws |
| 表名 | dws_assistant_project_tag |
| 主键 | id |
| 唯一键 | (site_id, assistant_id, time_window, category_code) |
| 数据来源 | dwd_assistant_service_log + dim_table + cfg_area_category |
| 更新频率 | 每日全量重建（按 site_id 删除后重新插入） |
| 说明 | 按时间窗口计算助教在四大项目的工作时长占比，≥25% 分配标签 |
## 字段说明
| 序号 | 字段名 | 类型 | 可空 | 说明 |
|------|--------|------|------|------|
| 1 | id | BIGSERIAL | NO | 自增主键 |
| 2 | site_id | BIGINT | NO | 门店ID |
| 3 | tenant_id | BIGINT | NO | 租户ID |
| 4 | assistant_id | BIGINT | NO | 助教ID |
| 5 | time_window | VARCHAR(40) | NO | 时间窗口枚举值 |
| 6 | category_code | VARCHAR(30) | NO | 项目分类代码（BILLIARD/SNOOKER/MAHJONG/KTV） |
| 7 | category_name | VARCHAR(50) | NO | 项目显示名称（如 🎱 中式/追分） |
| 8 | short_name | VARCHAR(10) | NO | 项目简写（如 🎱） |
| 9 | duration_seconds | BIGINT | NO | 该项目总工作时长（秒） |
| 10 | total_seconds | BIGINT | NO | 所有四大项目总时长（秒） |
| 11 | percentage | NUMERIC(5,4) | NO | 占比（0~1，四位小数） |
| 12 | is_tagged | BOOLEAN | NO | 占比≥0.25 时为 TRUE |
| 13 | computed_at | TIMESTAMPTZ | NO | 计算时间 |
| 14 | created_at | TIMESTAMPTZ | NO | 创建时间 |
| 15 | updated_at | TIMESTAMPTZ | NO | 更新时间 |
## 时间窗口
助教看板使用 6 个时间窗口：
| 枚举值 | 说明 |
|--------|------|
| THIS_MONTH | 本月（月初 ~ 今天） |
| THIS_QUARTER | 本季度（季度首月1日 ~ 今天） |
| LAST_MONTH | 上月（上月初 ~ 上月末） |
| LAST_3_MONTHS_EXCL_CURRENT | 前3个月不含本月 |
| LAST_QUARTER | 上季度 |
| LAST_6_MONTHS | 最近半年（不含本月） |
## 索引
| 索引名 | 字段 | 类型 | 说明 |
|--------|------|------|------|
| pk_dws_assistant_project_tag | id | 主键 | 自增主键 |
| uk_dws_assistant_project_tag | (site_id, assistant_id, time_window, category_code) | 唯一 | 业务唯一键 |
| idx_apt_site_window_tagged | (site_id, time_window) WHERE is_tagged=TRUE | 部分索引 | 加速看板查询 |
## 数据链路
```
dwd.dwd_assistant_service_log (income_seconds, site_table_id)
→ JOIN dwd.dim_table (site_table_id → table_id, scd2_is_current=1)
→ get_area_category(area_name, table_name) -- 通过 cfg_area_category 映射
→ 只保留 BILLIARD/SNOOKER/MAHJONG/KTV
→ 按 (assistant_id, category_code) 汇总 income_seconds
→ 计算占比 percentage = duration_seconds / total_seconds
→ ≥0.25 标记 is_tagged=TRUE
→ 写入 dws.dws_assistant_project_tag
```
### 关键规则
1. 数据链路走 `dim_table`（通过 `site_table_id` JOIN），不直接用事实表的 `site_table_area_name`
2. 只计算四大项目（BILLIARD/SNOOKER/MAHJONG/KTV），SPECIAL/OTHER 不参与
3. 标签阈值 25%（`TAG_THRESHOLD = 0.25`）
4. 全量删除重建策略：按 `site_id` 删除后重新插入所有时间窗口
5. `is_delete = 0` 过滤已删除的服务记录
## ETL 任务
| 属性 | 值 |
|------|-----|
| 任务代码 | DWS_ASSISTANT_PROJECT_TAG |
| Python 类 | AssistantProjectTagTask |
| 文件 | tasks/dws/assistant_project_tag_task.py |
| 依赖 | DWD_LOAD_FROM_ODS |
## 变更记录
| 日期 | 变更 | 说明 |
|------|------|------|
| 2026-03-07 | 新建表 | 支持助教看板按项目类型筛选 |
## 验证 SQL
```sql
-- 1. 确认表存在且有数据
SELECT COUNT(*) AS row_count,
COUNT(DISTINCT assistant_id) AS assistant_count,
COUNT(DISTINCT time_window) AS window_count
FROM dws.dws_assistant_project_tag;
-- 2. 确认 category_code 只有四大项目
SELECT DISTINCT category_code
FROM dws.dws_assistant_project_tag
ORDER BY category_code;
-- 期望：BILLIARD, KTV, MAHJONG, SNOOKER
-- 3. 确认占比计算正确（duration_seconds / total_seconds ≈ percentage）
SELECT site_id, assistant_id, time_window, category_code,
duration_seconds, total_seconds, percentage,
ROUND(duration_seconds::numeric / NULLIF(total_seconds, 0), 4) AS calc_pct,
is_tagged,
(percentage >= 0.25) AS should_be_tagged
FROM dws.dws_assistant_project_tag
WHERE percentage >= 0.25 AND is_tagged = FALSE
LIMIT 10;
-- 期望：0 行（所有 ≥25% 的都应标记为 TRUE）
-- 4. 确认唯一键无重复
SELECT site_id, assistant_id, time_window, category_code, COUNT(*)
FROM dws.dws_assistant_project_tag
GROUP BY site_id, assistant_id, time_window, category_code
HAVING COUNT(*) > 1;
-- 期望：0 行
```
## 回滚策略
```sql
-- 删除表（不影响其他表）
DROP TABLE IF EXISTS dws.dws_assistant_project_tag CASCADE;
-- 从 task_registry.py 移除 DWS_ASSISTANT_PROJECT_TAG 注册
-- 从 maintenance_task.py DEFAULT_RETENTION_TABLES 移除对应条目
```

55
apps/etl/connectors/feiqiu/docs/database/DWS/main/BD_manual_dws_finance_daily_summary.md
View File

@@ -22,7 +22,7 @@
| 2 | site_id | BIGINT | NO | 门店ID |
| 3 | tenant_id | BIGINT | NO | 租户ID |
| 4 | stat_date | DATE | NO | 统计日期 |
| 5 | gross_amount | NUMERIC(14,2) | NO | 发生额合计 |
| 5 | gross_amount | NUMERIC(14,2) | NO | 发生额合计（= 四项正价之和：table_fee + goods + assistant_pd + assistant_cx，不含 electricity_money，不使用 `consume_money`） |
| 6 | table_fee_amount | NUMERIC(14,2) | NO | 台费正价 |
| 7 | goods_amount | NUMERIC(14,2) | NO | 商品正价 |
| 8 | assistant_pd_amount | NUMERIC(14,2) | NO | 助教基础课正价（陪打） |
@@ -31,9 +31,9 @@
| 11 | discount_groupbuy | NUMERIC(14,2) | NO | 团购优惠 |
| 12 | discount_vip | NUMERIC(14,2) | NO | 会员折扣 |
| 13 | discount_gift_card | NUMERIC(14,2) | NO | 赠送卡抵扣（余额变动） |
| 14 | discount_manual | NUMERIC(14,2) | NO | 手动调整 |
| 14 | discount_manual | NUMERIC(14,2) | NO | 大客户优惠（从 adjust_amount 中按配置拆出） |
| 15 | discount_rounding | NUMERIC(14,2) | NO | 抹零 |
| 16 | discount_other | NUMERIC(14,2) | NO | 其他优惠 |
| 16 | discount_other | NUMERIC(14,2) | NO | 其他优惠（adjust_amount - 大客户优惠） |
| 17 | confirmed_income | NUMERIC(14,2) | NO | 确认收入 = 发生额 - 优惠 |
| 18 | cash_inflow_total | NUMERIC(14,2) | NO | 现金流入合计 |
| 19 | cash_pay_amount | NUMERIC(14,2) | NO | 收银实付 |
@@ -42,7 +42,7 @@
| 22 | platform_fee_amount | NUMERIC(14,2) | NO | 平台佣金+服务费（导入） |
| 23 | recharge_cash_inflow | NUMERIC(14,2) | NO | 充值现金流入 |
| 24 | card_consume_total | NUMERIC(14,2) | NO | 卡消费合计 |
| 25 | cash_card_consume | NUMERIC(14,2) | NO | 储值卡消费 |
| 25 | recharge_card_consume | NUMERIC(14,2) | NO | 现金充值卡消费（= `recharge_card_amount`，仅现金充值卡支付部分，不含赠送卡） |
| 26 | gift_card_consume | NUMERIC(14,2) | NO | 赠送卡消费 |
| 27 | cash_outflow_total | NUMERIC(14,2) | NO | 现金流出合计 |
| 28 | cash_balance_change | NUMERIC(14,2) | NO | 现金余额变动 |
@@ -63,7 +63,14 @@
## 数据来源
> ⚠️ **consume_money 口径警告**：飞球上游 `consume_money` 在不同时期存在三种口径（A/B/C），DWS 层不应直接使用。
> 应使用 `items_sum = table_charge_money + goods_money + assistant_pd_money + assistant_cx_money + electricity_money` 作为全时期一致的消费项目合计。
> 详见 [consume_money 口径详解](../../../../docs/reports/DWD-DOC/consume/consume-money-caliber.md)。
### 结账汇总：dwd_settlement_head
> ⚠️ 以下示例 SQL 使用 `DATE(pay_time)` 简化展示。实际代码使用 `biz_date_sql_expr(pay_time, cutoff_hour)` 进行营业日归属（跨日订单归前一天）。
```sql
SELECT
DATE(pay_time) AS stat_date,
@@ -82,6 +89,7 @@ SELECT
SUM(pl_coupon_sale_amount) AS pl_coupon_sale_amount
FROM dwd.dwd_settlement_head
WHERE site_id = :site_id
AND settle_type IN (1, 3) -- 仅台桌结账+商城订单，排除退货(6)/退款(7)
GROUP BY DATE(pay_time);
```
@@ -137,12 +145,49 @@ GROUP BY change_time::DATE;
**计算公式**
```
-- gross_amount 基于 items_sum 各分项（全时期一致），不使用 consume_money
gross_amount = table_fee_amount + goods_amount + assistant_pd_amount + assistant_cx_amount
discount_total = discount_groupbuy + discount_vip + discount_gift_card + discount_manual + discount_rounding + discount_other
confirmed_income = gross_amount - discount_total
cash_inflow_total = cash_pay_amount + groupbuy_pay_amount + platform_settlement_amount + recharge_cash_inflow
cash_inflow_total = cash_pay_amount + platform_inflow + recharge_cash_inflow
-- platform_inflow：优先取 platform_settlement_amount（平台回款），为 0 时取 groupbuy_pay_amount（团购支付）
-- 两者互斥，不可同时计入
```
> ⚠️ `discount_manual` 存储大客户优惠（从 adjust_amount 中按配置的会员ID/订单ID拆出），`discount_other` 存储其他手动调整（= adjust_amount - 大客户优惠）。两者互斥，之和 = adjust_amount。
**支付渠道恒等式**
```
-- 以下恒等式 100% 成立（DWD-DOC 校准确认）
balance_amount = recharge_card_amount + gift_card_amount -- 储值卡 = 充值卡 + 礼品卡
pay_amount = point_amount + cash_amount -- 实付 = 积分 + 现金（互斥）
```
> ⚠️ `balance_amount`（储值卡支付）是独立支付渠道，`recharge_card_amount` 和 `gift_card_amount` 是其分账明细，不可与 `balance_amount` 重复计算。
**团购券三层价格体系**
```
顾客支付价（PCR.sale_price）→ 平台结算价（SH.pl_coupon_sale_amount）→ 门店抵扣价（SH.coupon_amount）
门店补贴 = coupon_amount - pl_coupon_sale_amount
```
- `pl_coupon_sale_amount = SUM(GR.ledger_unit_price)` ✅ 100%
- `coupon_amount = SUM(GR.ledger_amount)` ✅ 100%
- P1 期间（2025-07~10）`pl_coupon_sale_amount` 恒为 0
**F2 收支平衡公式（三期差异）**
```
P1/P2（< 2026-01-15 12:45:59）:
consume = coupon + pay + balance - rounding + adjust + member_disc + prepay(ex)
B 类过渡期（2026-01-15 12:46~18:44，约 40 笔）:
consume = 2*coupon + pay + balance - rounding + adjust + member_disc + prepay(ex)
P3（≥ 2026-01-15 18:45，当前生效）:
consume = coupon + pl_coupon + pay + balance - rounding + adjust + member_disc + prepay(ex)
```
- 通过率：P1/P2 99.24% | B 95.00% | P3 99.87%
- 详见 [F2 收支平衡专项](../../../../docs/reports/DWD-DOC/05-f2-balance-audit.md)
**物化汇总层（可选）**
- L1~L4 物化视图：`mv_dws_finance_daily_summary_l1` / `l2` / `l3` / `l4`
- 刷新任务：`DWS_MV_REFRESH_FINANCE_DAILY`

15
apps/etl/connectors/feiqiu/docs/database/DWS/main/BD_manual_dws_finance_discount_detail.md
View File

@@ -37,16 +37,17 @@
|--------------------|--------------------|----------|
| GROUPBUY | 团购优惠 | dwd_settlement_head.coupon_amount - 团购实付 |
| VIP | 会员折扣 | dwd_settlement_head.member_discount_amount |
| GIFT_CARD_TABLE | 台费卡抵扣 | dwd_member_balance_change |
| GIFT_CARD_DRINK | 酒水卡抵扣 | dwd_member_balance_change |
| GIFT_CARD_COUPON | 活动抵用券抵扣 | dwd_member_balance_change |
| MANUAL | 手动调整 | dwd_settlement_head.adjust_amount |
| GIFT_CARD_TABLE | 台费卡抵扣 | dwd_member_balance_change（`card_type_id = 2791990152417157`） |
| GIFT_CARD_DRINK | 酒水卡抵扣 | dwd_member_balance_change（`card_type_id = 2794699703437125`） |
| GIFT_CARD_COUPON | 活动抵用券抵扣 | dwd_member_balance_change（`card_type_id = 2793266846533445`） |
| BIG_CUSTOMER | 大客户优惠 | dwd_settlement_head（big_customer_amount，从 adjust_amount 拆分） |
| OTHER | 其他优惠 | adjust_amount - big_customer_amount（其他无法归类的手动调整） |
| ROUNDING | 抹零 | dwd_settlement_head.rounding_amount |
| BIG_CUSTOMER | 大客户优惠 | dwd_settlement_head（特定会员优惠） |
| OTHER | 其他优惠 | 其他无法归类的优惠 |
## 数据来源
> ⚠️ 以下示例 SQL 使用 `pay_time::DATE` 简化展示。实际代码使用 `biz_date_sql_expr(pay_time, cutoff_hour)` 进行营业日归属（跨日订单归前一天），详见 ETL 配置 `app.business_day_start_hour`。
```sql
-- 从结账头表提取优惠汇总
SELECT
@@ -62,7 +63,7 @@ SELECT
COUNT(CASE WHEN rounding_amount != 0 THEN 1 END) AS rounding_order_count
FROM dwd.dwd_settlement_head
WHERE site_id = :site_id
AND settle_status = 1
AND settle_type IN (1, 3)
GROUP BY pay_time::DATE;
```

39
apps/etl/connectors/feiqiu/docs/database/DWS/main/BD_manual_dws_finance_income_structure.md
View File

@@ -10,7 +10,7 @@
| 表名 | dws_finance_income_structure |
| 主键 | id |
| 唯一键 | (site_id, stat_date, structure_type, category_code) |
| 数据来源 | dwd_table_fee_log + dwd_assistant_service_log + cfg_area_category |
| 数据来源 | dwd_settlement_head + dwd_table_fee_log + dwd_assistant_service_log + cfg_area_category |
| 更新频率 | 每日更新 |
| 说明 | 以"日期+区域/类型"为粒度，分析收入结构 |
@@ -35,23 +35,28 @@
## 分类代码说明
### 按区域分析 (structure_type = 'AREA')
| category_code | category_name | 来源 |
|---------------|---------------|------|
| BILLIARD | 台球散台 | A区/B区/C区/TV台 |
| BILLIARD_VIP | 台球VIP | VIP包厢 |
| SNOOKER | 斯诺克 | 斯诺克区 |
| MAHJONG | 麻将棋牌 | 麻将房/M7/M8/666/发财 |
| KTV | K歌娱乐 | K包/k包活动区/幸会158 |
| SPECIAL | 补时长 | 补时长 |
| OTHER | 其他 | 未映射区域 |
| category_code | category_name | display_name | 来源 |
|---------------|---------------|--------------|------|
| BILLIARD | 🎱 中式/追分 | 🎱 中式/追分 | A区/B区/C区/TV台/VIP包厢(V1-V4) |
| SNOOKER | 斯诺克 | 斯诺克 | 斯诺克区/VIP包厢(V5) |
| MAHJONG | 🀄 麻将/棋牌 | 🀄 麻将/棋牌 | 麻将房/M7/M8/666/发财 |
| KTV | 🎤 团建/K歌 | 🎤 团建/K歌 | K包/k包活动区/幸会158 |
| SPECIAL | 补时长 | 补时长 | 补时长 |
| OTHER | 其他 | 其他 | 未映射区域 |
> ⚠️ `BILLIARD_VIP` 已于 2026-03-07 废弃，VIP包厢按台桌级映射拆分至 BILLIARD(V1-V4) 和 SNOOKER(V5)。
### 按收入类型分析 (structure_type = 'INCOME_TYPE')
| category_code | category_name |
|---------------|---------------|
| TABLE_FEE | 台费收入 |
| GOODS | 商品收入 |
| ASSISTANT_BASE | 助教基础课收入 |
| ASSISTANT_BONUS | 助教附加课收入 |
| category_code | category_name | 数据来源字段 |
|---------------|---------------|-------------|
| TABLE_FEE | 台费收入 | `settlement_head.table_charge_money` |
| GOODS | 商品收入 | `settlement_head.goods_money` |
| ASSISTANT_PD | 助教陪打收入 | `settlement_head.assistant_pd_money` |
| ASSISTANT_CX | 助教超休收入 | `settlement_head.assistant_cx_money` |
> ⚠️ 历史版本曾使用 `ASSISTANT_BASE`/`ASSISTANT_BONUS`，已更正为 `ASSISTANT_PD`（陪打）/`ASSISTANT_CX`（超休），与 DWD 结算单字段对齐。
> 收入金额取自 `items_sum` 各分项（`table_charge_money + goods_money + assistant_pd_money + assistant_cx_money + electricity_money`），
> 不使用 `consume_money`（存在三种历史口径混合，详见 [consume_money 口径](../../../../docs/reports/DWD-DOC/consume/consume-money-caliber.md)）。
## 数据来源
@@ -85,4 +90,4 @@ income_ratio = income_amount / SUM(income_amount) OVER (PARTITION BY stat_date,
|------|------|
| 可回溯 | ✅ 完全可回溯 |
| 数据范围 | 2025-07-21 ~ 至今 |
| 依赖表 | dwd_table_fee_log, dwd_assistant_service_log, dim_table, cfg_area_category |
| 依赖表 | dwd_settlement_head, dwd_table_fee_log, dwd_assistant_service_log, dim_table, cfg_area_category |

12
apps/etl/connectors/feiqiu/docs/database/DWS/main/BD_manual_dws_finance_recharge_summary.md
View File

@@ -49,16 +49,16 @@
SELECT
DATE(pay_time) AS stat_date,
COUNT(*) AS recharge_count,
SUM(pay_money + gift_money) AS recharge_total,
SUM(pay_money) AS recharge_cash,
SUM(gift_money) AS recharge_gift,
SUM(pay_amount + point_amount) AS recharge_total,
SUM(pay_amount) AS recharge_cash,
SUM(point_amount) AS recharge_gift,
-- 首充
SUM(CASE WHEN is_first = 1 THEN 1 ELSE 0 END) AS first_recharge_count,
SUM(CASE WHEN is_first = 1 THEN pay_money ELSE 0 END) AS first_recharge_cash,
SUM(CASE WHEN is_first = 1 THEN gift_money ELSE 0 END) AS first_recharge_gift,
SUM(CASE WHEN is_first = 1 THEN pay_amount ELSE 0 END) AS first_recharge_cash,
SUM(CASE WHEN is_first = 1 THEN point_amount ELSE 0 END) AS first_recharge_gift,
-- 续充
SUM(CASE WHEN is_first != 1 OR is_first IS NULL THEN 1 ELSE 0 END) AS renewal_count,
SUM(CASE WHEN is_first != 1 OR is_first IS NULL THEN pay_money ELSE 0 END) AS renewal_cash,
SUM(CASE WHEN is_first != 1 OR is_first IS NULL THEN pay_amount ELSE 0 END) AS renewal_cash,
-- 会员数
COUNT(DISTINCT member_id) AS recharge_member_count
FROM dwd.dwd_recharge_order

68
apps/etl/connectors/feiqiu/docs/database/DWS/main/BD_manual_dws_member_consumption_summary.md
View File

@@ -30,40 +30,50 @@
| 10 | first_consume_date | DATE | YES | 首次消费日期 |
| 11 | last_consume_date | DATE | YES | 最近消费日期 |
| 12 | total_visit_count | INTEGER | NO | 累计到店次数 |
| 13 | total_consume_amount | NUMERIC(14,2) | NO | 累计消费金额 |
| 14 | total_recharge_amount | NUMERIC(14,2) | NO | 累计充值金额 |
| 15 | total_table_fee | NUMERIC(14,2) | NO | 累计台费 |
| 16 | total_goods_amount | NUMERIC(14,2) | NO | 累计商品消费 |
| 17 | total_assistant_amount | NUMERIC(14,2) | NO | 累计助教服务消费 |
| 13 | total_consume_amount | NUMERIC(14,2) | NO | 累计消费金额（基于 `items_sum` 口径，见下方说明） |
| 14 | total_recharge_amount | NUMERIC(14,2) | NO | 累计充值金额（来源：`dim_member.recharge_money_sum`，上游 API 同步值） |
| 15 | total_table_fee | NUMERIC(14,2) | NO | 累计台费（`table_charge_money`） |
| 16 | total_goods_amount | NUMERIC(14,2) | NO | 累计商品消费（`goods_money`） |
| 17 | total_assistant_amount | NUMERIC(14,2) | NO | 累计助教服务消费（= `assistant_pd_money` + `assistant_cx_money`） |
| 18-23 | visit_count_7d/10d/15d/30d/60d/90d | INTEGER | NO | 近N天到店次数 |
| 24-29 | consume_amount_7d/10d/15d/30d/60d/90d | NUMERIC(14,2) | NO | 近N天消费金额 |
| 30 | cash_card_balance | NUMERIC(14,2) | NO | 储值卡余额 |
| 31 | gift_card_balance | NUMERIC(14,2) | NO | 赠送卡余额 |
| 32 | total_card_balance | NUMERIC(14,2) | NO | 总卡余额 |
| 33 | days_since_last | INTEGER | YES | 距离最近消费的天数 |
| 34 | is_active_7d | BOOLEAN | NO | 近7天是否活跃 |
| 35 | is_active_30d | BOOLEAN | NO | 近30天是否活跃 |
| 36 | is_active_90d | BOOLEAN | NO | 近90天是否活跃 |
| 37 | customer_tier | VARCHAR(20) | YES | 客户分层（高价值/中等/低活跃/流失） |
| 38 | created_at | TIMESTAMPTZ | NO | 创建时间 |
| 39 | updated_at | TIMESTAMPTZ | NO | 更新时间 |
| 30-32 | recharge_count_30d/60d/90d | INTEGER | NO | 近N天充值笔数（来源：dwd_recharge_order） |
| 33-35 | recharge_amount_30d/60d/90d | NUMERIC(14,2) | NO | 近N天充值金额（仅 `pay_amount` 现金部分，不含 `point_amount` 赠送，来源：dwd_recharge_order） |
| 36 | avg_ticket_amount | NUMERIC(14,2) | NO | 次均消费（= total_consume_amount / MAX(total_visit_count, 1)） |
| 37 | cash_card_balance | NUMERIC(14,2) | NO | 储值卡余额 |
| 38 | gift_card_balance | NUMERIC(14,2) | NO | 赠送卡余额 |
| 39 | total_card_balance | NUMERIC(14,2) | NO | 总卡余额 |
| 40 | days_since_last | INTEGER | YES | 距离最近消费的天数 |
| 41 | is_active_7d | BOOLEAN | NO | 近7天是否活跃 |
| 42 | is_active_30d | BOOLEAN | NO | 近30天是否活跃 |
| 43 | is_active_90d | BOOLEAN | NO | 近90天是否活跃 |
| 44 | customer_tier | VARCHAR(20) | YES | 客户分层（高价值/中等/低活跃/流失） |
| 45 | created_at | TIMESTAMPTZ | NO | 创建时间 |
| 46 | updated_at | TIMESTAMPTZ | NO | 更新时间 |
## 数据来源
### 消费统计来源：dwd_settlement_head
> ⚠️ **consume_money 口径警告**：`consume_money` 在不同时期存在三种口径（A/B/C），DWS 层不应直接使用。
> 应使用 `items_sum = table_charge_money + goods_money + assistant_pd_money + assistant_cx_money + electricity_money` 作为全时期一致的消费项目合计。
> 详见 [consume_money 口径详解](../../../../docs/reports/DWD-DOC/consume/consume-money-caliber.md)。
```sql
SELECT
site_id,
member_id,
DATE(pay_time) AS consume_date,
COUNT(*) AS visit_count,
SUM(consume_money) AS consume_amount,
-- ✅ 使用 items_sum 口径（全时期一致），不使用 consume_money
SUM(table_charge_money + goods_money + assistant_pd_money
+ assistant_cx_money + electricity_money) AS consume_amount,
SUM(table_charge_money) AS table_fee,
SUM(goods_money) AS goods_amount,
SUM(assistant_pd_money + assistant_cx_money) AS assistant_amount
FROM dwd.dwd_settlement_head
WHERE member_id != 0 -- 排除散客
AND settle_type = 1 -- 已结账
AND settle_type IN (1, 3) -- 已结账订单（台桌结账 + 快捷结账）
GROUP BY site_id, member_id, DATE(pay_time);
```
@@ -84,19 +94,27 @@ GROUP BY tenant_member_id;
- member_id=0 的散客不进入此表统计
**客户分层规则**
```sql
customer_tier = CASE
WHEN consume_amount_30d >= 1000 THEN '高价值'
WHEN consume_amount_30d >= 300 THEN '中等'
WHEN is_active_30d THEN '低活跃'
ELSE '流失'
END
```python
# 基于 90 天消费次数+金额组合判断（代码实际逻辑）
if visit_count_90d >= 3 and consume_amount_90d >= 1000:
customer_tier = '高价值'
elif visit_count_30d > 0:
customer_tier = '中等'
elif visit_count_90d > 0:
customer_tier = '低活跃'
else:
customer_tier = '流失'
```
**金额口径说明**
- `total_consume_amount` 及各滚动窗口 `consume_amount_*d` 均基于 `items_sum` 口径
- `items_sum = table_charge_money + goods_money + assistant_pd_money + assistant_cx_money + electricity_money`
- `total_assistant_amount` = `assistant_pd_money`（陪打）+ `assistant_cx_money`（超休），不使用笼统的 `service_fee`
## 可回溯性
| 项目 | 说明 |
|------|------|
| 可回溯 | ✅ 完全可回溯 |
| 数据范围 | 2025-07-16 ~ 至今 |
| 依赖表 | dwd_settlement_head, dim_member, dim_member_card_account |
| 依赖表 | dwd_settlement_head, dwd_recharge_order, dim_member, dim_member_card_account |

133
apps/etl/connectors/feiqiu/docs/database/DWS/main/BD_manual_dws_member_project_tag.md Normal file
View File

@@ -0,0 +1,133 @@
# dws_member_project_tag 客户项目标签表
> 生成时间：2026-03-07
## 表信息
| 属性 | 值 |
|------|-----|
| Schema | dws |
| 表名 | dws_member_project_tag |
| 主键 | id |
| 唯一键 | (site_id, member_id, time_window, category_code) |
| 数据来源 | dwd_table_fee_log + dim_table + cfg_area_category |
| 更新频率 | 每日全量重建（按 site_id 删除后重新插入） |
| 说明 | 按时间窗口计算客户在四大项目的消费时长占比，≥25% 分配标签。散客不参与。 |
## 字段说明
| 序号 | 字段名 | 类型 | 可空 | 说明 |
|------|--------|------|------|------|
| 1 | id | BIGSERIAL | NO | 自增主键 |
| 2 | site_id | BIGINT | NO | 门店ID |
| 3 | tenant_id | BIGINT | NO | 租户ID |
| 4 | member_id | BIGINT | NO | 会员ID（散客不入此表） |
| 5 | time_window | VARCHAR(40) | NO | 时间窗口枚举值 |
| 6 | category_code | VARCHAR(30) | NO | 项目分类代码（BILLIARD/SNOOKER/MAHJONG/KTV） |
| 7 | category_name | VARCHAR(50) | NO | 项目显示名称（如 🎱 中式/追分） |
| 8 | short_name | VARCHAR(10) | NO | 项目简写（如 🎱） |
| 9 | duration_seconds | BIGINT | NO | 该项目总计费时长（秒，来源 ledger_count） |
| 10 | total_seconds | BIGINT | NO | 所有四大项目总时长（秒） |
| 11 | percentage | NUMERIC(5,4) | NO | 占比（0~1，四位小数） |
| 12 | is_tagged | BOOLEAN | NO | 占比≥0.25 时为 TRUE |
| 13 | computed_at | TIMESTAMPTZ | NO | 计算时间 |
| 14 | created_at | TIMESTAMPTZ | NO | 创建时间 |
| 15 | updated_at | TIMESTAMPTZ | NO | 更新时间 |
## 时间窗口
客户看板使用 2 个时间窗口：
| 枚举值 | 说明 |
|--------|------|
| LAST_30_DAYS | 近30天（含今天，base_date-29天 ~ base_date） |
| LAST_60_DAYS | 近60天（含今天，base_date-59天 ~ base_date） |
## 索引
| 索引名 | 字段 | 类型 | 说明 |
|--------|------|------|------|
| pk_dws_member_project_tag | id | 主键 | 自增主键 |
| uk_dws_member_project_tag | (site_id, member_id, time_window, category_code) | 唯一 | 业务唯一键 |
| idx_mpt_site_window_tagged | (site_id, time_window) WHERE is_tagged=TRUE | 部分索引 | 加速看板查询 |
## 数据链路
```
dwd.dwd_table_fee_log (ledger_count, site_table_id)
→ JOIN dwd.dim_table (site_table_id → table_id, scd2_is_current=1)
→ get_area_category(area_name, table_name) -- 通过 cfg_area_category 映射
→ 只保留 BILLIARD/SNOOKER/MAHJONG/KTV
→ 排除散客（member_id IS NULL 或 = 0）
→ 按 (member_id, category_code) 汇总 ledger_count
→ 计算占比 percentage = duration_seconds / total_seconds
→ ≥0.25 标记 is_tagged=TRUE
→ 写入 dws.dws_member_project_tag
```
### 关键规则
1. 数据链路走 `dim_table`（通过 `site_table_id` JOIN），不直接用事实表的 `site_table_area_name`
2. 客户时长使用 `ledger_count`（计费时长），不使用 `income_seconds`（那是助教工作时长）
3. 散客（member_id=0 或 NULL）不参与标签计算
4. 只计算四大项目（BILLIARD/SNOOKER/MAHJONG/KTV）
5. 标签阈值 25%（`TAG_THRESHOLD = 0.25`）
6. 全量删除重建策略：按 `site_id` 删除后重新插入所有时间窗口
7. `COALESCE(is_delete, 0) = 0` 过滤已删除的台费记录
## ETL 任务
| 属性 | 值 |
|------|-----|
| 任务代码 | DWS_MEMBER_PROJECT_TAG |
| Python 类 | MemberProjectTagTask |
| 文件 | tasks/dws/member_project_tag_task.py |
| 依赖 | DWD_LOAD_FROM_ODS |
## 变更记录
| 日期 | 变更 | 说明 |
|------|------|------|
| 2026-03-07 | 新建表 | 支持客户看板按项目类型筛选 |
## 验证 SQL
```sql
-- 1. 确认表存在且有数据
SELECT COUNT(*) AS row_count,
COUNT(DISTINCT member_id) AS member_count,
COUNT(DISTINCT time_window) AS window_count
FROM dws.dws_member_project_tag;
-- 2. 确认无散客数据
SELECT COUNT(*) FROM dws.dws_member_project_tag WHERE member_id = 0 OR member_id IS NULL;
-- 期望：0
-- 3. 确认占比计算正确
SELECT site_id, member_id, time_window, category_code,
duration_seconds, total_seconds, percentage,
ROUND(duration_seconds::numeric / NULLIF(total_seconds, 0), 4) AS calc_pct,
is_tagged,
(percentage >= 0.25) AS should_be_tagged
FROM dws.dws_member_project_tag
WHERE percentage >= 0.25 AND is_tagged = FALSE
LIMIT 10;
-- 期望：0 行
-- 4. 确认唯一键无重复
SELECT site_id, member_id, time_window, category_code, COUNT(*)
FROM dws.dws_member_project_tag
GROUP BY site_id, member_id, time_window, category_code
HAVING COUNT(*) > 1;
-- 期望：0 行
```
## 回滚策略
```sql
-- 删除表（不影响其他表）
DROP TABLE IF EXISTS dws.dws_member_project_tag CASCADE;
-- 从 task_registry.py 移除 DWS_MEMBER_PROJECT_TAG 注册
-- 从 maintenance_task.py DEFAULT_RETENTION_TABLES 移除对应条目
```

59
apps/etl/connectors/feiqiu/docs/database/DWS/main/BD_manual_dws_member_visit_detail.md
View File

@@ -35,13 +35,14 @@
| 15 | table_fee | NUMERIC(12,2) | NO | 台费 |
| 16 | goods_amount | NUMERIC(12,2) | NO | 商品金额 |
| 17 | assistant_amount | NUMERIC(12,2) | NO | 助教服务金额 |
| 18 | total_consume | NUMERIC(12,2) | NO | 消费总额（正价） |
| 18 | total_consume | NUMERIC(12,2) | NO | 消费总额（基于 `items_sum` 口径，= tc + goods + pd + cx + electricity） |
| 19 | total_discount | NUMERIC(12,2) | NO | 优惠总额 |
| 20 | actual_pay | NUMERIC(12,2) | NO | 实付金额 |
| 21 | cash_pay | NUMERIC(12,2) | NO | 现金/刷卡支付 |
| 22 | cash_card_pay | NUMERIC(12,2) | NO | 储值卡支付 |
| 21 | cash_pay | NUMERIC(12,2) | NO | 收银实付（= `pay_amount`，与 actual_pay 同值） |
| 22 | balance_pay | NUMERIC(12,2) | NO | 储值卡总支付（= recharge_card_pay + gift_card_pay） |
| 22a | recharge_card_pay | NUMERIC(12,2) | NO | 现金充值卡支付（balance_pay 的子项） |
| 23 | gift_card_pay | NUMERIC(12,2) | NO | 赠送卡支付 |
| 24 | groupbuy_pay | NUMERIC(12,2) | NO | 团购券支付 |
| 24 | groupbuy_pay | NUMERIC(12,2) | NO | 团购抵消台费金额（= `coupon_amount`） |
| 25 | table_duration_min | INTEGER | NO | 台桌使用时长（分钟，来自台费流水真实秒数） |
| 26 | assistant_duration_min | INTEGER | NO | 助教服务时长（分钟） |
| 27 | assistant_services | JSONB | YES | 助教服务列表 |
@@ -51,28 +52,36 @@
## 数据来源
### 主表来源：dwd_settlement_head
> ⚠️ `total_consume` 使用 `items_sum` 口径（全时期一致），不使用 `consume_money`（存在三种历史口径混合）。
```sql
SELECT
site_id,
tenant_id,
member_id,
order_settle_id,
DATE(pay_time) AS visit_date,
create_time AS visit_time,
member_name AS member_nickname,
member_phone AS member_mobile,
table_id,
table_charge_money AS table_fee,
goods_money AS goods_amount,
assistant_pd_money + assistant_cx_money AS assistant_amount,
consume_money AS total_consume,
member_discount_amount + adjust_amount + rounding_amount AS total_discount,
pay_amount AS actual_pay,
balance_amount AS cash_card_pay,
gift_card_amount AS gift_card_pay
FROM dwd.dwd_settlement_head
WHERE member_id != 0
AND settle_type = 1;
sh.site_id,
sh.tenant_id,
sh.member_id,
sh.order_settle_id,
DATE(sh.pay_time) AS visit_date,
sh.create_time AS visit_time,
-- ⚠️ member_nickname/member_mobile 实际从 dim_member 关联获取（nickname/mobile），非结算头表字段
dm.nickname AS member_nickname,
dm.mobile AS member_mobile,
sh.table_id,
sh.table_charge_money AS table_fee,
sh.goods_money AS goods_amount,
sh.assistant_pd_money + sh.assistant_cx_money AS assistant_amount,
-- ✅ 使用 items_sum 口径，不使用 consume_money
sh.table_charge_money + sh.goods_money + sh.assistant_pd_money
+ sh.assistant_cx_money + sh.electricity_money AS total_consume,
sh.member_discount_amount + sh.adjust_amount + sh.rounding_amount AS total_discount,
sh.pay_amount AS actual_pay,
sh.balance_amount AS balance_pay,
sh.recharge_card_amount AS recharge_card_pay,
sh.gift_card_amount AS gift_card_pay
FROM dwd.dwd_settlement_head sh
JOIN dwd.dim_member dm ON sh.tenant_member_id = dm.tenant_member_id AND dm.scd2_is_current = 1
WHERE sh.member_id IS NOT NULL AND sh.member_id != 0
AND sh.settle_type IN (1, 3); -- 仅台桌结账+商城订单，排除退货(6)/退款(7)
```
### 助教服务明细：dwd_assistant_service_log
@@ -127,4 +136,4 @@ area_category = COALESCE(
|------|------|
| 可回溯 | ✅ 完全可回溯 |
| 数据范围 | 2025-07-16 ~ 至今 |
| 依赖表 | dwd_settlement_head, dwd_assistant_service_log, dwd_table_fee_log, dim_table, dim_member |
| 依赖表 | dwd_settlement_head, dwd_assistant_service_log, dwd_table_fee_log, dim_table, dim_member, cfg_area_category |

30
apps/etl/connectors/feiqiu/docs/database/DWS/main/BD_manual_dws_order_summary.md
View File

@@ -23,20 +23,20 @@
| 4 | order_date | DATE | NO | 订单日期（优先 pay_time，其次 create_time） |
| 5 | tenant_id | BIGINT | NO | 租户ID |
| 6 | member_id | BIGINT | YES | 会员ID（NULL 或 0 为散客） |
| 7 | member_flag | BOOLEAN | NO | 是否会员订单 |
| 8 | recharge_order_flag | BOOLEAN | NO | 充值订单标记（消费金额=0 且实付>0） |
| 7 | member_flag | BOOLEAN | NO | 是否会员订单（来源：`is_bind_member`） |
| 8 | recharge_order_flag | BOOLEAN | NO | 充值订单标记（`consume_money = 0` 且实付>0，此处 consume_money 仅用于零值判断，不参与金额计算） |
| 9 | item_count | INTEGER | NO | 订单项数 |
| 10 | total_item_quantity | INTEGER | NO | 订单项总数量 |
| 11 | table_fee_amount | NUMERIC | NO | 台费金额 |
| 12 | assistant_service_amount | NUMERIC | NO | 助教服务金额 |
| 12 | assistant_service_amount | NUMERIC | NO | 助教服务金额（= `assistant_pd_money` + `assistant_cx_money`） |
| 13 | goods_amount | NUMERIC | NO | 商品金额 |
| 14 | group_amount | NUMERIC | NO | 团购金额 |
| 15 | total_coupon_deduction | NUMERIC | NO | 优惠券抵扣总额 |
| 16 | member_discount_amount | NUMERIC | NO | 会员折扣金额 |
| 17 | manual_discount_amount | NUMERIC | NO | 手动折扣金额 |
| 18 | order_original_amount | NUMERIC | NO | 原价估算（实付+优惠/抵扣） |
| 18 | order_original_amount | NUMERIC | NO | 原价估算（= `total_paid_amount + total_coupon_deduction + member_discount_amount + manual_discount_amount`） |
| 19 | order_final_amount | NUMERIC | NO | 最终应付金额 |
| 20 | stored_card_deduct | NUMERIC | NO | 储值卡抵扣金额 |
| 20 | stored_card_deduct | NUMERIC | NO | 储值卡抵扣金额（= `balance_amount`，即 `recharge_card_amount + gift_card_amount`） |
| 21 | external_paid_amount | NUMERIC | NO | 外部支付金额（实付-卡类抵扣） |
| 22 | total_paid_amount | NUMERIC | NO | 总实付金额 |
| 23 | book_table_flow | NUMERIC | NO | 台费流水 |
@@ -44,9 +44,9 @@
| 25 | book_goods_flow | NUMERIC | NO | 商品流水 |
| 26 | book_group_flow | NUMERIC | NO | 团购流水 |
| 27 | book_order_flow | NUMERIC | NO | 订单总流水（台费+助教+商品+团购） |
| 28 | order_effective_consume_cash | NUMERIC | NO | 有效消费现金 |
| 29 | order_effective_recharge_cash | NUMERIC | NO | 有效充值现金 |
| 30 | order_effective_flow | NUMERIC | NO | 有效流水 |
| 28 | order_effective_consume_cash | NUMERIC | NO | 有效消费现金（= `GREATEST(total_paid_amount - stored_card_deduct, 0)`，即外部支付金额） |
| 29 | order_effective_recharge_cash | NUMERIC | NO | 有效充值现金（当前硬编码为 0，占位字段，待后续实现） |
| 30 | order_effective_flow | NUMERIC | NO | 有效流水（当前 = `total_paid_amount`） |
| 31 | refund_amount | NUMERIC | NO | 退款金额 |
| 32 | net_income | NUMERIC | NO | 净收入（实付-退款） |
| 33 | created_at | TIMESTAMPTZ | NO | 创建时间 |
@@ -54,10 +54,18 @@
## 业务口径
> ⚠️ 本表金额字段基于 `items_sum` 各分项（`table_charge_money + goods_money + assistant_pd_money + assistant_cx_money + electricity_money`），
> 不使用 `consume_money`（存在三种历史口径混合）。
- order_date 优先取 pay_time，其次 create_time
- recharge_order_flag：消费金额=0 且实付>0 时标记为充值订单
- order_original_amount = 实付 + 优惠/抵扣
- recharge_order_flag：`consume_money = 0` 且实付>0 时标记为充值订单（此处 consume_money 仅用于零值判断，不参与金额计算）
- stored_card_deduct = `balance_amount`（恒等式：`balance_amount = recharge_card_amount + gift_card_amount`，三者不可相加）
- order_original_amount = `total_paid_amount + total_coupon_deduction + member_discount_amount + manual_discount_amount`（实付 + 团购抵扣 + 会员折扣 + 手动调整）
- external_paid_amount = total_paid_amount - stored_card_deduct（外部支付 = 实付 - 储值卡抵扣）
- book_order_flow = 台费 + 助教 + 商品 + 团购
- order_effective_recharge_cash：当前硬编码为 0，占位字段
- order_effective_consume_cash = `GREATEST(total_paid_amount - stored_card_deduct, 0)`（与 external_paid_amount 同值）
- order_effective_flow = `total_paid_amount`（当前实现）
- net_income = total_paid_amount - refund_amount
## 使用说明
@@ -81,4 +89,4 @@ ORDER BY order_date DESC;
| 项目 | 说明 |
|------|------|
| 可回溯 | ✅ 完全可回溯 |
| 依赖表 | dwd_settlement_head, dwd_table_fee_log, dwd_assistant_service_log, dwd_store_goods_sale, dwd_groupbuy_redemption, dwd_payment, dwd_refund |
| 依赖表 | dwd_settlement_head, dwd_table_fee_log, dwd_assistant_service_log, dwd_store_goods_sale, dwd_groupbuy_redemption, dwd_refund, dwd_refund_ex |

87
apps/etl/connectors/feiqiu/docs/database/ODS/main/BD_manual_group_buy_package_details.md Normal file
View File

@@ -0,0 +1,87 @@
# group_buy_package_details 团购套餐详情
> 生成时间：2026-03-05
## 表信息
| 属性 | 值 |
|------|-----|
| Schema | ods |
| 表名 | group_buy_package_details |
| 主键 | coupon_id |
| 数据来源 | `QueryPackageCouponInfo` 详情接口（二级拉取） |
| DDL 路径 | `db/etl_feiqiu/ods/group_buy_package_details.sql` |
| 说明 | 团购套餐详情 ODS 层，存储每个 couponId 的详情原始数据 |
## 数据获取方式
本表数据通过 `ODS_GROUP_PACKAGE` 任务的 **detail_endpoint 二级详情拉取** 子流程获取：
1. 主流程先从 `QueryPackageCouponList` 拉取团购列表写入 `ods.group_buy_packages`
2. 子流程遍历列表中每个 `id`，串行调用 `QueryPackageCouponInfo` 获取详情
3. 详情数据写入本表，采用全量快照模式（`SnapshotMode.FULL_TABLE`），UPSERT on `coupon_id`
## 字段说明
| 序号 | 字段名 | 类型 | 可空 | 说明 |
|------|--------|------|------|------|
| 1 | coupon_id | BIGINT | NO（PK） | 团购套餐 ID（= groupPurchasePackage.id） |
| 2 | package_name | TEXT | YES | 团购套餐名称 |
| 3 | duration | INTEGER | YES | 台费计时时长（秒） |
| 4 | start_time | TIMESTAMPTZ | YES | 可用日期开始 |
| 5 | end_time | TIMESTAMPTZ | YES | 可用日期结束 |
| 6 | add_start_clock | TEXT | YES | 可用时段开始（如 "00:00:00"） |
| 7 | add_end_clock | TEXT | YES | 可用时段结束（如 "1.00:00:00"） |
| 8 | is_enabled | INTEGER | YES | 是否启用（1=启用, 0=禁用） |
| 9 | is_delete | INTEGER | YES | 是否已删除（1=已删除, 0=正常） |
| 10 | site_id | BIGINT | YES | 店铺 ID |
| 11 | tenant_id | BIGINT | YES | 租户 ID |
| 12 | create_time | TIMESTAMPTZ | YES | 创建时间 |
| 13 | creator_name | TEXT | YES | 创建人 |
| 14 | table_area_ids | JSONB | YES | 可用台区 ID 列表（来自 groupPurchasePackage.tableAreaId） |
| 15 | table_area_names | JSONB | YES | 可用台区名称列表（来自 groupPurchasePackage.tableAreaNameList） |
| 16 | assistant_services | JSONB | YES | 助教服务关联数组（来自 packageCouponAssistants） |
| 17 | groupon_site_infos | JSONB | YES | 关联门店信息数组（来自 grouponSiteInfos） |
| 18 | package_services | JSONB | YES | 套餐服务数组（来自 packagePackageService，待调研） |
| 19 | coupon_details_list | JSONB | YES | 券明细数组（来自 packageCouponDetailsList，待调研） |
| 20 | content_hash | TEXT | YES | 业务字段内容哈希，用于变更检测 |
| 21 | payload | JSONB | YES | 详情接口完整原始 JSON 响应 |
| 22 | fetched_at | TIMESTAMPTZ | YES | ETL 拉取时间戳 |
## 与列表表的关系
```
ods.group_buy_packages (列表)
└── ods.group_buy_package_details (详情)
关联字段：group_buy_packages.id = group_buy_package_details.coupon_id
关系：1:1（每个列表记录对应一条详情）
```
## 下游消费
DWD 层 `dwd.dim_groupbuy_package_ex` 在加载时通过 LEFT JOIN 本表，将 `table_area_ids`、`table_area_names`、`assistant_services`、`groupon_site_infos` 四个 JSONB 字段合并到扩展表。
## 使用说明
```sql
-- 查询最新入库的详情记录
SELECT coupon_id, package_name, table_area_names, assistant_services
FROM ods.group_buy_package_details
ORDER BY fetched_at DESC
LIMIT 10;
```
```sql
-- 关联列表表查看完整信息
SELECT p.id, p.package_name, p.selling_price,
d.table_area_names, d.assistant_services, d.groupon_site_infos
FROM ods.group_buy_packages p
LEFT JOIN ods.group_buy_package_details d ON p.id = d.coupon_id
WHERE p.is_delete IS DISTINCT FROM 1;
```
## 可回溯性
| 项目 | 说明 |
|------|------|
| 可回溯 | ✅ 完全可回溯（保留 payload 原始 JSON） |
| 数据来源 | `PackageCoupon/QueryPackageCouponInfo` API |

89
apps/etl/connectors/feiqiu/docs/database/ODS/mappings/mapping_QueryPackageCouponInfo_group_buy_package_details.md Normal file
View File

@@ -0,0 +1,89 @@
# 团购套餐详情（QueryPackageCouponInfo） → group_buy_package_details 字段映射
> 生成时间：2026-03-05
## 端点信息
| 属性 | 值 |
|------|-----|
| 接口路径 | `PackageCoupon/QueryPackageCouponInfo` |
| 请求方法 | POST |
| 请求参数 | `{ "couponId": <id> }`（从 `ods.group_buy_packages.id` 获取） |
| ODS 对应表 | `ods.group_buy_package_details` |
| JSON 数据路径 | `data` |
| 调用方式 | 二级详情拉取（`ODS_GROUP_PACKAGE` 任务的 `detail_endpoint` 子流程） |
## 响应结构
```json
{
"data": {
"groupPurchasePackage": {
"id": 123,
"packageName": "...",
"duration": 3600,
"startTime": "...",
"endTime": "...",
"addStartClock": "00:00:00",
"addEndClock": "1.00:00:00",
"isEnabled": 1,
"isDelete": 0,
"siteId": 456,
"tenantId": 789,
"createTime": "...",
"creatorName": "...",
"tableAreaId": [1, 2, 3],
"tableAreaNameList": ["A区", "B区"]
},
"packageCouponAssistants": [...],
"grouponSiteInfos": [...],
"packagePackageService": [...],
"packageCouponDetailsList": [...]
}
}
```
## 字段映射
### 结构化字段（来自 data.groupPurchasePackage）
| JSON 路径 | ODS 列名 | 类型转换 | 说明 |
|-----------|----------|----------|------|
| data.groupPurchasePackage.id | coupon_id | int→BIGINT | 团购套餐 ID，主键 |
| data.groupPurchasePackage.packageName | package_name | string→TEXT | 套餐名称 |
| data.groupPurchasePackage.duration | duration | int→INTEGER | 台费计时时长（秒） |
| data.groupPurchasePackage.startTime | start_time | string→TIMESTAMPTZ | 可用日期开始 |
| data.groupPurchasePackage.endTime | end_time | string→TIMESTAMPTZ | 可用日期结束 |
| data.groupPurchasePackage.addStartClock | add_start_clock | string→TEXT | 可用时段开始 |
| data.groupPurchasePackage.addEndClock | add_end_clock | string→TEXT | 可用时段结束 |
| data.groupPurchasePackage.isEnabled | is_enabled | int→INTEGER | 是否启用 |
| data.groupPurchasePackage.isDelete | is_delete | int→INTEGER | 是否已删除 |
| data.groupPurchasePackage.siteId | site_id | int→BIGINT | 店铺 ID |
| data.groupPurchasePackage.tenantId | tenant_id | int→BIGINT | 租户 ID |
| data.groupPurchasePackage.createTime | create_time | string→TIMESTAMPTZ | 创建时间 |
| data.groupPurchasePackage.creatorName | creator_name | string→TEXT | 创建人 |
### JSONB 数组字段
| JSON 路径 | ODS 列名 | 类型转换 | 说明 |
|-----------|----------|----------|------|
| data.groupPurchasePackage.tableAreaId | table_area_ids | array→JSONB | 可用台区 ID 列表 |
| data.groupPurchasePackage.tableAreaNameList | table_area_names | array→JSONB | 可用台区名称列表 |
| data.packageCouponAssistants | assistant_services | array→JSONB | 助教服务关联（含 skillId/assistantLevel/assistantDuration） |
| data.grouponSiteInfos | groupon_site_infos | array→JSONB | 关联门店信息（含 siteId/siteName） |
| data.packagePackageService | package_services | array→JSONB | 套餐服务数组（待调研，可能为空） |
| data.packageCouponDetailsList | coupon_details_list | array→JSONB | 券明细数组（待调研，可能为空） |
## ETL 补充字段
| ODS 列名 | 生成逻辑 |
|-----------|----------|
| content_hash | 基于原始 payload + is_delete 计算 SHA-256 |
| payload | 完整原始 JSON 响应（`data` 节点） |
| fetched_at | ETL 拉取时间戳（`DEFAULT now()`） |
## 写入策略
- 全量快照模式（`SnapshotMode.FULL_TABLE`）
- UPSERT on `coupon_id`，每次运行覆盖全部记录
- 通过 `content_hash` 去重，内容未变则跳过写入

5
apps/etl/connectors/feiqiu/docs/etl_tasks/README.md
View File

@@ -51,7 +51,7 @@ graph LR
| 文档 | 说明 |
|------|------|
| [BaseTask 公共机制](base_task_mechanism.md) | 任务基类模板方法、TaskContext、时间窗口、注册表、Flow 执行 |
| [ODS 层任务](ods_tasks.md) | 23 个通用 ODS 任务的架构、配置结构、API 端点、目标表 |
| [ODS 层任务](ods_tasks.md) | 22 个通用 ODS 任务的架构、配置结构、API 端点、目标表 |
| [DWD 层任务](dwd_tasks.md) | DWD_LOAD_FROM_ODS 核心装载、SCD2 处理、质量校验 |
| [DWS 层任务](dws_tasks.md) | 助教业绩、会员分析、财务统计、库存汇总、运维任务共 17 个 DWS 任务 |
| [INDEX 层任务](index_tasks.md) | WBI/NCI/RS/SPI 指数算法 + ML 手动台账导入 |
@@ -69,10 +69,9 @@ graph LR
|----------|-----------|--------|----------|------|
| `ODS_ASSISTANT_ACCOUNT` | `OdsAssistantAccountsTask` | `ods.assistant_accounts_master` | 助教账号档案 | [查看](ods_tasks.md) |
| `ODS_ASSISTANT_LEDGER` | `OdsAssistantLedgerTask` | `ods.assistant_service_records` | 助教服务流水 | [查看](ods_tasks.md) |
| `ODS_ASSISTANT_ABOLISH` | `OdsAssistantAbolishTask` | `ods.assistant_cancellation_records` | 助教废除记录 | [查看](ods_tasks.md) |
| `ODS_INVENTORY_CHANGE` | `OdsInventoryChangeTask` | `ods.goods_stock_movements` | 库存变化记录 | [查看](ods_tasks.md) |
| `ODS_INVENTORY_STOCK` | `OdsInventoryStockTask` | `ods.goods_stock_summary` | 库存汇总 | [查看](ods_tasks.md) |
| `ODS_GROUP_PACKAGE` | `OdsPackageTask` | `ods.group_buy_packages` | 团购套餐定义 | [查看](ods_tasks.md) |
| `ODS_GROUP_PACKAGE` | `OdsPackageTask` | `ods.group_buy_packages` | 团购套餐定义 + 详情子流程（通过 `detail_endpoint` 串行调用 `QueryPackageCouponInfo` 获取每个团购的详情数据，写入 `ods.group_buy_package_details`） | [查看](ods_tasks.md) |
| `ODS_GROUP_BUY_REDEMPTION` | `OdsGroupBuyRedemptionTask` | `ods.group_buy_redemption_records` | 团购套餐核销 | [查看](ods_tasks.md) |
| `ODS_MEMBER` | `OdsMemberTask` | `ods.member_profiles` | 会员档案 | [查看](ods_tasks.md) |
| `ODS_MEMBER_BALANCE` | `OdsMemberBalanceTask` | `ods.member_balance_changes` | 会员余额变动 | [查看](ods_tasks.md) |

50
apps/etl/connectors/feiqiu/docs/etl_tasks/dwd_tasks.md
View File

@@ -77,13 +77,15 @@ load(extracted, context) → 遍历 TABLE_MAP
| `dwd.dim_goods_category` | `ods.stock_goods_category_tree` | 商品分类维度（含子类展开） |
| `dwd.dim_groupbuy_package` | `ods.group_buy_packages` | 团购套餐维度 |
| `dwd.dim_groupbuy_package_ex` | `ods.group_buy_packages` | 团购套餐扩展 |
| `dwd.dim_staff` | `ods.staff_info_master` | 员工维度 |
| `dwd.dim_staff_ex` | `ods.staff_info_master` | 员工扩展 |
#### 事实表映射
| DWD 表 | ODS 源表 | 说明 |
|--------|----------|------|
| `dwd.dwd_settlement_head` | `ods.settlement_records` | 结算头（订单结算主记录） |
| `dwd.dwd_settlement_head` | `ods.settlement_records` | 结算头（订单结算主记录）— 详见下方「结算头关键字段口径」 |
| `dwd.dwd_settlement_head_ex` | `ods.settlement_records` | 结算头扩展（支付方式、撤单、促销等） |
| `dwd.dwd_table_fee_log` | `ods.table_fee_transactions` | 台费流水 |
| `dwd.dwd_table_fee_log_ex` | `ods.table_fee_transactions` | 台费流水扩展（销售员、消费类型等） |
@@ -93,8 +95,8 @@ load(extracted, context) → 遍历 TABLE_MAP
| `dwd.dwd_store_goods_sale_ex` | `ods.store_goods_sales_records` | 商品销售扩展 |
| `dwd.dwd_assistant_service_log` | `ods.assistant_service_records` | 助教服务记录 |
| `dwd.dwd_assistant_service_log_ex` | `ods.assistant_service_records` | 助教服务扩展 |
| `dwd.dwd_assistant_trash_event` | `ods.assistant_cancellation_records` | 助教取消/废单事件 |
| `dwd.dwd_assistant_trash_event_ex` | `ods.assistant_cancellation_records` | 助教取消扩展 |
| ~~`dwd.dwd_assistant_trash_event`~~ | ~~`ods.assistant_cancellation_records`~~ | ~~助教取消/废单事件（2026-02-22 DROP，2026-03-01 清理残留）~~ |
| ~~`dwd.dwd_assistant_trash_event_ex`~~ | ~~`ods.assistant_cancellation_records`~~ | ~~助教取消扩展（2026-02-22 DROP，2026-03-01 清理残留）~~ |
| `dwd.dwd_member_balance_change` | `ods.member_balance_changes` | 会员余额变动 |
| `dwd.dwd_member_balance_change_ex` | `ods.member_balance_changes` | 会员余额变动扩展 |
| `dwd.dwd_groupbuy_redemption` | `ods.group_buy_redemption_records` | 团购核销记录 |
@@ -106,8 +108,48 @@ load(extracted, context) → 遍历 TABLE_MAP
| `dwd.dwd_payment` | `ods.payment_transactions` | 支付记录 |
| `dwd.dwd_refund` | `ods.refund_transactions` | 退款记录 |
| `dwd.dwd_refund_ex` | `ods.refund_transactions` | 退款扩展 |
| `dwd.dwd_goods_stock_summary` | `ods.goods_stock_summary` | 库存汇总 |
| `dwd.dwd_goods_stock_movement` | `ods.goods_stock_movements` | 库存变动 |
> 共计 **17 对维度映射**（含 `_ex`）+ **23 对事实映射**（含 `_ex`）= **40 对**映射。
> 共计 **19 对维度映射**（含 `_ex`）+ **23 对事实映射**（含 `_ex`，已排除 2026-02-22 DROP 的 assistant_trash_event）= **42 对**有效映射。
---
### 结算头关键字段口径
`dwd_settlement_head` 是核心交易事实表，以下字段在下游消费时需特别注意：
#### settle_type 枚举
| 值 | 含义 | 说明 |
|----|------|------|
| 1 | 台桌结账 | 正常台桌消费结账 |
| 3 | 商城订单 | 商品零售订单 |
| 6 | 退货订单 | 商品退货 |
| 7 | 退款订单 | 金额退款 |
> DWS 层计算发生额、收入等指标时，通常只取 `settle_type IN (1, 3)`（正向交易），排除退货/退款。
> 本表无 `is_delete` 字段，不可用 `is_delete` 过滤。
#### consume_money 口径警告
`consume_money` 存在三种历史口径（A/B/C），**DWS 层不应直接使用**。
应使用 `items_sum` 口径：
```
items_sum = table_charge_money + goods_money + assistant_pd_money + assistant_cx_money + electricity_money
```
> 详见 [consume_money 口径校准文档](../../../../docs/reports/DWD-DOC/consume/consume-money-caliber.md)
> 及 [BD 手册 dwd_settlement_head](../database/DWD/main/BD_manual_dwd_settlement_head.md)
#### 支付渠道恒等式
```
balance_amount = recharge_card_amount + gift_card_amount -- 储值卡 = 充值卡 + 礼品卡
```
> `balance_amount` 是独立支付渠道，`recharge_card_amount` / `gift_card_amount` 是其分账明细，三者不可重复计算。
---

192
apps/etl/connectors/feiqiu/docs/etl_tasks/dws_tasks.md
View File

@@ -8,7 +8,7 @@
## 概述
DWS 层共有 17 个已注册任务（含 DWS_MAINTENANCE），按业务域分为五组：
DWS 层共有 19 个已注册任务（含 DWS_MAINTENANCE），按业务域分为六组：
### 助教业绩域（6 个）
@@ -28,6 +28,13 @@ DWS 层共有 17 个已注册任务（含 DWS_MAINTENANCE），按业务域分
| `DWS_MEMBER_CONSUMPTION` | `MemberConsumptionTask` | `dws_member_consumption_summary` | 日期+会员 | delete-before-insert |
| `DWS_MEMBER_VISIT` | `MemberVisitTask` | `dws_member_visit_detail` | 日期+会员+结账单 | delete-before-insert |
### 项目标签域（2 个）
| 任务代码 | Python 类 | 目标表 | 粒度 | 更新策略 |
|----------|-----------|--------|------|----------|
| `DWS_ASSISTANT_PROJECT_TAG` | `AssistantProjectTagTask` | `dws_assistant_project_tag` | 助教+时间窗口+项目 | 全量删除重建（按 site_id） |
| `DWS_MEMBER_PROJECT_TAG` | `MemberProjectTagTask` | `dws_member_project_tag` | 会员+时间窗口+项目 | 全量删除重建（按 site_id） |
### 财务统计域（4 个）
| 任务代码 | Python 类 | 目标表 | 粒度 | 更新策略 |
@@ -373,7 +380,7 @@ DWS 汇总计算涉及历史月份时，不能直接使用维度表的"当前版
```
dwd_assistant_service_log ──┬──► DWS_ASSISTANT_DAILY（日度明细）
dwd_assistant_trash_event ──┘ │
dwd_assistant_service_log_ex ┘ │
▼
DWS_ASSISTANT_MONTHLY（月度汇总+档位+排名）
│
@@ -448,7 +455,7 @@ dwd_assistant_service_log ────► DWS_ASSISTANT_CUSTOMER（客户关系
| 来源表 | Schema | 用途 |
|--------|--------|------|
| `dwd_assistant_service_log` | `dwd` | 助教服务流水（主数据源） |
| `dwd_assistant_trash_event` | `dwd` | 废除记录（排除无效业绩） |
| `dwd_assistant_service_log_ex` | `dwd` | 扩展表（`is_trash` 标记废除记录） |
| `dim_assistant` | `dwd` | 助教维度（SCD2，获取当日等级） |
| `cfg_skill_type` | `dws` | 技能 → 课程类型映射 |
@@ -459,21 +466,23 @@ dwd_assistant_service_log ────► DWS_ASSISTANT_CUSTOMER（客户关系
| 字段分组 | 字段 | 说明 |
|----------|------|------|
| 标识 | `site_id`, `tenant_id`, `assistant_id`, `assistant_nickname`, `stat_date` | 门店、助教、日期 |
| 等级 | `assistant_level_code`, `assistant_level_name` | SCD2 as-of 取值，取统计日当日生效的等级 |
| 等级 | `assistant_level_code`, `assistant_level_name` | SCD2 as-of 取值（`level_code`），`level_name` 由 code 静态映射得出 |
| 服务次数 | `total_service_count`, `base_service_count`, `bonus_service_count`, `room_service_count` | 总/基础课/附加课/包厢课 |
| 计费秒数 | `total_seconds`, `base_seconds`, `bonus_seconds`, `room_seconds` | 原始秒数 |
| 计费小时 | `total_hours`, `base_hours`, `bonus_hours`, `room_hours` | 秒数 ÷ 3600，`Decimal` 精度 |
| 计费金额 | `total_ledger_amount`, `base_ledger_amount`, `bonus_ledger_amount`, `room_ledger_amount` | 台账金额 |
| 去重统计 | `unique_customers`, `unique_tables` | 去重客户数（排除散客）、去重台桌数 |
| 废除统计 | `trashed_seconds`, `trashed_count` | 被废除的秒数和次数 |
| 惩罚检测 | `penalty_minutes`, `penalty_reason`, `is_exempt`, `per_hour_contribution` | 惩罚分钟数（公式：`actual_minutes × (1 - per_hour_contribution / 24)`）、惩罚原因、是否豁免、每小时贡献金额（= `base_ledger_amount / base_hours / overlap_count`） |
#### 核心业务逻辑
1. **课程类型分类**：通过 `skill_id` 查询 `cfg_skill_type` 映射，分为 `BASE`（基础课）、`BONUS`（附加课）、`ROOM`（包厢课），未匹配默认 `BASE`
2. **废除记录排除**：以 `assistant_service_id` 为键构建废除索引，被废除的服务记录不计入有效业绩（服务次数、时长、金额），但单独统计 `trashed_seconds` 和 `trashed_count`
2. **废除记录排除**：通过 JOIN `dwd_assistant_service_log_ex` 的 `is_trash = 1` 标记识别废除记录（`dwd_assistant_trash_event` 已于 2026-02-22 废弃），被废除的服务记录不计入有效业绩（服务次数、时长、金额），但单独统计 `trashed_seconds` 和 `trashed_count`
3. **助教等级 SCD2 取值**：调用 `get_assistant_level_asof(assistant_id, service_date)` 获取统计日当日生效的等级版本，而非当前最新版本
4. **散客过滤**：`unique_customers` 统计时排除 `member_id` 为 0 或 None 的散客
5. **客户/台桌去重**：无论服务记录是否被废除，客户和台桌均参与去重统计
6. **定档折算惩罚检测**：聚合完成后，检测同一台桌多名助教重叠挂台的违规情况（规则2）。计算 `per_hour_contribution = base_ledger_amount / base_hours / overlap_count`，若低于阈值（默认 24 元/小时）则按比例扣减 `penalty_minutes`。豁免助教（`is_exempt = True`）不参与惩罚计算。
---
@@ -813,6 +822,9 @@ dim_table ────────────────────┘
| 全量累计 | `first_consume_date`, `last_consume_date`, `total_visit_count`, `total_consume_amount`, `total_recharge_amount`, `total_table_fee`, `total_goods_amount`, `total_assistant_amount` | 首次/最近消费日期、累计到店次数、累计消费金额、累计充值金额、累计台费、累计商品金额、累计助教费用 |
| 滚动窗口（次数） | `visit_count_7d`, `visit_count_10d`, `visit_count_15d`, `visit_count_30d`, `visit_count_60d`, `visit_count_90d` | 各窗口到店次数 |
| 滚动窗口（金额） | `consume_amount_7d`, `consume_amount_10d`, `consume_amount_15d`, `consume_amount_30d`, `consume_amount_60d`, `consume_amount_90d` | 各窗口消费金额 |
| 充值窗口（笔数） | `recharge_count_30d`, `recharge_count_60d`, `recharge_count_90d` | 近 30/60/90 天充值笔数（来源：dwd_recharge_order） |
| 充值窗口（金额） | `recharge_amount_30d`, `recharge_amount_60d`, `recharge_amount_90d` | 近 30/60/90 天充值金额（仅 `pay_amount` 现金部分，不含 `point_amount` 赠送） |
| 次均消费 | `avg_ticket_amount` | total_consume_amount / MAX(total_visit_count, 1) |
| 卡余额 | `cash_card_balance`, `gift_card_balance`, `total_card_balance` | 储值卡（现金卡）余额、赠送卡余额、总余额 |
| 活跃度 | `days_since_last`, `is_active_7d`, `is_active_30d`, `is_active_90d` | 距最近消费天数、近 7/30/90 天是否活跃 |
| 客户分层 | `customer_tier` | 分层标签（高价值/中等/低活跃/流失） |
@@ -821,15 +833,17 @@ dim_table ────────────────────┘
**1. 散客排除**
`member_id` 为 0 或 None 的散客不进入此表统计。SQL 层面和 transform 阶段均做过滤。
`member_id` 为 0 或 None 的散客不进入此表统计。SQL 层面和 transform 阶段均做过滤，同时通过 `settle_type IN (1, 3)` 仅保留台桌结账和商城订单（排除退货/退款）。
**2. 消费统计来源**
从 `dwd_settlement_head` 按 `member_id` 聚合，消费金额拆分为：
- `consume_money`：总消费金额
从 `dwd_settlement_head` 按 `member_id` 聚合，消费金额使用 `items_sum` 口径拆分为：
- `items_sum`：消费项目合计（= `table_charge_money + goods_money + assistant_pd_money + assistant_cx_money + electricity_money`）
- `table_charge_money`：台费
- `goods_money`：商品金额
- `assistant_pd_money + assistant_cx_money`：助教费用（专业课 + 陪练课合计）
- `assistant_pd_money + assistant_cx_money`：助教费用（陪打 + 超休合计）
> ⚠️ 不使用 `consume_money`（三种历史口径混合），详见 `docs/reports/DWD-DOC/consume/consume-money-caliber.md`
**3. 滚动窗口**
@@ -909,7 +923,7 @@ dim_table ────────────────────┘
| 会员信息 | `member_nickname`, `member_mobile`, `member_birthday` | 昵称、脱敏手机号、生日 |
| 台桌信息 | `table_id`, `table_name`, `area_name`, `area_category` | 台桌 ID、台桌名称、区域名称、区域分类 |
| 消费金额 | `table_fee`, `goods_amount`, `assistant_amount`, `total_consume`, `total_discount`, `actual_pay` | 台费、商品金额、助教费用、总消费、总优惠、实付金额 |
| 支付方式 | `cash_pay`, `cash_card_pay`, `gift_card_pay`, `groupbuy_pay` | 现金/在线支付、储值卡支付、赠送卡支付、团购券支付 |
| 支付方式 | `cash_pay`, `balance_pay`, `recharge_card_pay`, `gift_card_pay`, `groupbuy_pay` | 现金/在线支付、储值卡总支付、现金充值卡支付、赠送卡支付、团购券支付 |
| 时长 | `table_duration_min`, `assistant_duration_min` | 台桌使用时长（分钟）、助教服务时长（分钟） |
| 助教服务 | `assistant_services` | JSON 格式的助教服务明细 |
@@ -917,15 +931,15 @@ dim_table ────────────────────┘
**1. 散客排除**
SQL 层面通过 `member_id IS NOT NULL AND member_id != 0` 过滤，transform 阶段通过 `is_guest()` 二次过滤。
SQL 层面通过 `member_id IS NOT NULL AND member_id != 0` 过滤，同时通过 `settle_type IN (1, 3)` 仅保留台桌结账和商城订单（排除退货/退款），transform 阶段通过 `is_guest()` 二次过滤。
**2. 消费金额拆分**
从 `dwd_settlement_head` 直接读取各金额字段：
- `table_fee`：`table_charge_money`（台费）
- `goods_amount`：`goods_money`（商品金额）
- `assistant_amount`：`assistant_pd_money + assistant_cx_money`（专业课 + 陪练课助教费用合计）
- `total_consume`：`consume_money`（总消费金额）
- `assistant_amount`：`assistant_pd_money + assistant_cx_money`（陪打 + 超休助教费用合计）
- `total_consume`：`items_sum`（= `table_charge_money + goods_money + assistant_pd_money + assistant_cx_money + electricity_money`，不使用 `consume_money`）
- `actual_pay`：`pay_amount`（实付金额）
**3. 总优惠计算**
@@ -943,7 +957,8 @@ total_discount = adjust_amount + member_discount_amount + rounding_amount
| 字段 | 来源字段 | 说明 |
|------|----------|------|
| `cash_pay` | `pay_amount` | 现金/在线支付 |
| `cash_card_pay` | `balance_amount` | 储值卡（现金卡）支付 |
| `balance_pay` | `balance_amount` | 储值卡总支付（= recharge_card_pay + gift_card_pay） |
| `recharge_card_pay` | `recharge_card_amount` | 现金充值卡支付（balance_pay 的子项） |
| `gift_card_pay` | `gift_card_amount` | 赠送卡支付 |
| `groupbuy_pay` | `coupon_amount` | 团购券支付 |
@@ -1046,12 +1061,12 @@ dwd_member_balance_change ────┘
|----------|------|------|
| 标识 | `site_id`, `tenant_id`, `stat_date` | 门店、统计日期 |
| 发生额 | `gross_amount`, `table_fee_amount`, `goods_amount`, `assistant_pd_amount`, `assistant_cx_amount` | 正价总额及按类型拆分（台费/商品/专业课/陪练课） |
| 优惠 | `discount_total`, `discount_groupbuy`, `discount_vip`, `discount_gift_card`, `discount_manual`, `discount_rounding`, `discount_other` | 优惠合计及按类型拆分 |
| 优惠 | `discount_total`, `discount_groupbuy`, `discount_vip`, `discount_gift_card`, `discount_manual`, `discount_rounding`, `discount_other` | 优惠合计及按类型拆分（discount_manual=大客户优惠，discount_other=其他手动调整，两者互斥） |
| 确认收入 | `confirmed_income` | 发生额 - 优惠合计 |
| 现金流入 | `cash_inflow_total`, `cash_pay_amount`, `groupbuy_pay_amount`, `platform_settlement_amount`, `recharge_cash_inflow` | 现金流入合计及来源拆分 |
| 现金流出 | `cash_outflow_total`, `platform_fee_amount` | 现金流出合计（支出 + 平台费用） |
| 现金净变动 | `cash_balance_change` | 流入 - 流出 |
| 卡消费 | `card_consume_total`, `cash_card_consume`, `gift_card_consume` | 储值卡消费 + 赠送卡消费 |
| 卡消费 | `card_consume_total`, `recharge_card_consume`, `gift_card_consume` | 现金充值卡消费（= `recharge_card_amount`）+ 赠送卡消费 |
| 充值统计 | `recharge_count`, `recharge_total`, `recharge_cash`, `recharge_gift`, `first_recharge_count`, `first_recharge_amount`, `renewal_count`, `renewal_amount` | 充值笔数/金额、首充/续充拆分 |
| 订单统计 | `order_count`, `member_order_count`, `guest_order_count`, `avg_order_amount` | 总订单数、会员/散客订单数、客单价 |
@@ -1063,7 +1078,9 @@ dwd_member_balance_change ────┘
gross_amount = table_charge_money + goods_money + assistant_pd_money + assistant_cx_money
```
从 `dwd_settlement_head` 按 `DATE(pay_time)` 聚合，分别统计台费、商品、专业课（PD）、陪练课（CX）四类收入。
> 注意：`gross_amount` 为发生额（正价四项），不含 `electricity_money`。完整消费项目合计（`items_sum`）还需加上 `electricity_money`。
从 `dwd_settlement_head` 按 `biz_date(pay_time)` 聚合，通过 `settle_type IN (1, 3)` 仅保留台桌结账和商城订单（排除退货/退款），分别统计台费、商品、陪打（PD）、超休（CX）四类收入。
**2. 团购优惠计算**
@@ -1096,10 +1113,12 @@ discount_other = adjust_amount - big_customer_amount （负值置 0）
**5. 优惠合计与确认收入**
```
discount_total = discount_groupbuy + discount_vip + discount_gift_card + discount_manual + discount_rounding
discount_total = discount_groupbuy + discount_vip + discount_gift_card + discount_manual + discount_rounding + discount_other
confirmed_income = gross_amount - discount_total
```
> `discount_manual` = 大客户优惠，`discount_other` = 其他手动调整，两者互斥，之和 = adjust_amount。
**6. 现金流计算**
```
@@ -1123,11 +1142,13 @@ daily_expense = expense_amount / days_in_month
**8. 卡消费统计**
```
cash_card_consume = recharge_card_amount + balance_amount （储值卡支付）
recharge_card_consume = recharge_card_amount （现金充值卡支付部分）
gift_card_consume = 赠送卡消费总额 （来自余额变动）
card_consume_total = cash_card_consume + gift_card_consume
card_consume_total = recharge_card_consume + gift_card_consume
```
> 注意：`balance_amount = recharge_card_amount + gift_card_amount`（恒等式），因此 `recharge_card_consume` 只取 `recharge_card_amount`，不可再加 `balance_amount`，否则重复计算。
---
### DWS_FINANCE_RECHARGE — 充值统计
@@ -1170,7 +1191,7 @@ card_consume_total = cash_card_consume + gift_card_consume
每笔充值金额拆分为：
```
充值总额 = pay_money（现金部分）+ gift_money（赠送部分）
充值总额 = pay_amount（现金部分）+ point_amount（赠送部分）
```
**2. 会员去重统计**
@@ -1236,14 +1257,14 @@ total_card_balance = cash_card_balance + gift_card_balance
**维度 1：按收入类型（`structure_type = 'INCOME_TYPE'`）**
从 `dwd_settlement_head` 按 `pay_time::DATE` 聚合，仅统计已结账订单（`settle_status = 1`），每日展开为 4 条记录：
从 `dwd_settlement_head` 按 `pay_time::DATE` 聚合，仅统计已结账订单（`settle_type IN (1, 3)`），每日展开为 4 条记录：
| category_code | category_name | 来源字段 | 说明 |
|---------------|---------------|----------|------|
| `TABLE_FEE` | 台费收入 | `table_charge_money` | 台桌使用费 |
| `GOODS` | 商品收入 | `goods_money` | 商品销售 |
| `ASSISTANT_BASE` | 助教基础课 | `assistant_pd_money` | 专业课（PD=陪打） |
| `ASSISTANT_BONUS` | 助教附加课 | `assistant_cx_money` | 附加课（CX=超休/促销） |
| `ASSISTANT_PD` | 助教陪打 | `assistant_pd_money` | 陪打收入 |
| `ASSISTANT_CX` | 助教超休 | `assistant_cx_money` | 超休收入 |
占比计算：`income_ratio = 该类型金额 / 当日四类收入总和`
@@ -1327,7 +1348,7 @@ total_card_balance = cash_card_balance + gift_card_balance
团购优惠 = coupon_amount - 团购实付
```
仅统计 `coupon_amount > 0` 的已结账订单（`settle_status = 1`）。
仅统计 `coupon_amount > 0` 的已结账订单（`settle_type IN (1, 3)`）。
**2. 赠送卡消费拆分**
@@ -1406,10 +1427,10 @@ dws_*（所有 DWS 汇总表）──────► DWS_MAINTENANCE（统一维
| 商品 | `item_count`, `total_item_quantity` | 商品种类数、商品总数量 |
| 费用明细 | `table_fee_amount`, `assistant_service_amount`, `goods_amount`, `group_amount` | 台费、助教费、商品金额、团购金额 |
| 优惠 | `total_coupon_deduction`, `member_discount_amount`, `manual_discount_amount` | 团购抵扣、会员折扣、手动调整 |
| 金额汇总 | `order_original_amount`, `order_final_amount` | 订单原价、实付金额 |
| 支付方式 | `stored_card_deduct`, `external_paid_amount`, `total_paid_amount` | 储值卡抵扣、外部支付、总支付 |
| 金额汇总 | `order_original_amount`, `order_final_amount` | 订单原价（= `total_paid_amount + total_coupon_deduction + member_discount_amount + manual_discount_amount`）、实付金额 |
| 支付方式 | `stored_card_deduct`, `external_paid_amount`, `total_paid_amount` | 储值卡抵扣（= `balance_amount`）、外部支付、总支付 |
| 台账流水 | `book_table_flow`, `book_assistant_flow`, `book_goods_flow`, `book_group_flow`, `book_order_flow` | 台费/助教/商品/团购/订单台账流水 |
| 有效消费 | `order_effective_consume_cash`, `order_effective_recharge_cash`, `order_effective_flow` | 有效消费现金、有效充值现金、有效流水 |
| 有效消费 | `order_effective_consume_cash`, `order_effective_recharge_cash`, `order_effective_flow` | 有效消费现金、有效充值现金（当前硬编码为 0，占位）、有效流水 |
| 退款 | `refund_amount`, `net_income` | 退款金额、净收入 |
#### 核心业务逻辑
@@ -1463,7 +1484,7 @@ net_income = total_paid_amount - refund_amount
recharge_order_flag = (consume_money = 0 AND pay_amount > 0)
```
消费金额为 0 但有支付金额的订单标记为充值订单。
消费金额为 0 但有支付金额的订单标记为充值订单。此处 `consume_money` 仅用于零值判断（三种口径在 =0 时等价），不涉及金额聚合。
#### 配置参数
@@ -1638,3 +1659,116 @@ dwd_goods_stock_summary ──┬──► DWS_GOODS_STOCK_DAILY（日度汇总
- `range_start_stock` 取该月第一条记录的值（期初快照）
- `range_end_stock` / `current_stock` 取该月最后一条记录的值（期末快照）
- `stat_period = 'monthly'`
---
## 项目标签域
项目标签域包含 2 个任务，按时间窗口计算助教和客户在四大项目类型（BILLIARD/SNOOKER/MAHJONG/KTV）的时长占比，占比≥25% 则分配标签。数据流向为：
```
dwd_assistant_service_log (income_seconds) ──┐
├──► dim_table (site_table_id JOIN)
dwd_table_fee_log (ledger_count) ────────────┘ │
▼
cfg_area_category (get_area_category)
│
┌──────────────────┴──────────────────┐
▼ ▼
DWS_ASSISTANT_PROJECT_TAG DWS_MEMBER_PROJECT_TAG
（助教项目标签，6 个时间窗口） （客户项目标签，2 个时间窗口）
```
### 公共逻辑
1. 数据链路走 `dim_table`（通过 `site_table_id` JOIN，`scd2_is_current=1`），获取 `area_name` 和 `table_name`
2. 通过 `get_area_category(area_name, table_name)` 映射到 `category_code`
3. 只保留四大项目（BILLIARD/SNOOKER/MAHJONG/KTV），排除 SPECIAL/OTHER
4. 标签阈值：`TAG_THRESHOLD = 0.25`（25%）
5. 更新策略：全量删除重建（按 `site_id` 删除后重新插入所有时间窗口）
---
### DWS_ASSISTANT_PROJECT_TAG — 助教项目标签
| 属性 | 值 |
|------|-----|
| 任务代码 | `DWS_ASSISTANT_PROJECT_TAG` |
| Python 类 | `AssistantProjectTagTask`（`tasks/dws/assistant_project_tag_task.py`） |
| 目标表 | `dws.dws_assistant_project_tag` |
| 主键 | `site_id`, `assistant_id`, `time_window`, `category_code` |
| 粒度 | 助教 + 时间窗口 + 项目类型 |
| 更新策略 | 全量删除重建（按 site_id） |
| 更新频率 | 每日更新 |
| 依赖 | `DWD_LOAD_FROM_ODS` |
#### 数据来源
| 来源表 | Schema | 用途 |
|--------|--------|------|
| `dwd_assistant_service_log` | `dwd` | 助教服务流水（`income_seconds` 工作时长） |
| `dim_table` | `dwd` | 台桌维度（SCD2 当前版本，`area_name` + `table_name`） |
| `cfg_area_category` | `dws` | 区域分类映射（通过 ConfigCache 加载） |
#### 时间窗口
| 枚举值 | 说明 |
|--------|------|
| `THIS_MONTH` | 本月（月初 ~ 今天） |
| `THIS_QUARTER` | 本季度（季度首月1日 ~ 今天） |
| `LAST_MONTH` | 上月（上月初 ~ 上月末） |
| `LAST_3_MONTHS_EXCL_CURRENT` | 前3个月不含本月 |
| `LAST_QUARTER` | 上季度 |
| `LAST_6_MONTHS` | 最近半年（不含本月） |
#### 核心业务逻辑
1. 从 `dwd_assistant_service_log` 按 `(site_assistant_id, site_table_id)` 聚合 `income_seconds`
2. 通过 `dim_table` JOIN 获取台桌的 `area_name` 和 `table_name`
3. 调用 `get_area_category(area_name, table_name)` 映射到 `category_code`
4. 按 `(assistant_id, category_code)` 汇总各项目时长
5. 计算占比：`percentage = duration_seconds / total_seconds`（四位小数）
6. 占比 ≥ 0.25 标记 `is_tagged = TRUE`
7. 过滤条件：`is_delete = 0`，营业日切点通过 `biz_date_sql_expr` 处理
---
### DWS_MEMBER_PROJECT_TAG — 客户项目标签
| 属性 | 值 |
|------|-----|
| 任务代码 | `DWS_MEMBER_PROJECT_TAG` |
| Python 类 | `MemberProjectTagTask`（`tasks/dws/member_project_tag_task.py`） |
| 目标表 | `dws.dws_member_project_tag` |
| 主键 | `site_id`, `member_id`, `time_window`, `category_code` |
| 粒度 | 会员 + 时间窗口 + 项目类型 |
| 更新策略 | 全量删除重建（按 site_id） |
| 更新频率 | 每日更新 |
| 依赖 | `DWD_LOAD_FROM_ODS` |
#### 数据来源
| 来源表 | Schema | 用途 |
|--------|--------|------|
| `dwd_table_fee_log` | `dwd` | 台费流水（`ledger_count` 计费时长） |
| `dim_table` | `dwd` | 台桌维度（SCD2 当前版本，`area_name` + `table_name`） |
| `cfg_area_category` | `dws` | 区域分类映射（通过 ConfigCache 加载） |
#### 时间窗口
| 枚举值 | 说明 |
|--------|------|
| `LAST_30_DAYS` | 近30天（含今天，base_date-29天 ~ base_date） |
| `LAST_60_DAYS` | 近60天（含今天，base_date-59天 ~ base_date） |
#### 核心业务逻辑
1. 从 `dwd_table_fee_log` 按 `(member_id, site_table_id)` 聚合 `ledger_count`
2. 散客排除：`member_id IS NOT NULL AND member_id != 0`
3. 通过 `dim_table` JOIN 获取台桌的 `area_name` 和 `table_name`
4. 调用 `get_area_category(area_name, table_name)` 映射到 `category_code`
5. 按 `(member_id, category_code)` 汇总各项目时长
6. 计算占比：`percentage = duration_seconds / total_seconds`（四位小数）
7. 占比 ≥ 0.25 标记 `is_tagged = TRUE`
8. 过滤条件：`COALESCE(is_delete, 0) = 0`，营业日切点通过 `biz_date_sql_expr` 处理

3
apps/etl/connectors/feiqiu/docs/etl_tasks/ods_task_params_matrix.md
View File

@@ -78,7 +78,6 @@ API 返回的 JSON 响应通过两级路径定位数据：先按 `data_path`（
| `ODS_SETTLEMENT_RECORDS` | ✅ | `(rangeStartTime, rangeEndTime)` | ❌ | ✅ | ✅ | `NONE` | — |
| `ODS_TABLE_USE` | ❌ | 默认 | ❌ | ✅ | ✅ | `WINDOW` | `create_time` |
| `ODS_ASSISTANT_LEDGER` | ✅ | 默认 | ❌ | ✅ | ✅ | `WINDOW` | `create_time` |
| `ODS_ASSISTANT_ABOLISH` | ✅ | 默认 | ❌ | ✅ | ✅ | `NONE` | — |
| `ODS_STORE_GOODS_SALES` | ❌ | 默认 | ❌ | ✅ | ✅ | `WINDOW` | `create_time` |
| `ODS_PAYMENT` | ❌ | 默认 | ❌ | ✅ | ✅ | `NONE` | — |
| `ODS_REFUND` | ❌ | 默认 | ❌ | ✅ | ✅ | `WINDOW` | `pay_time` |
@@ -88,6 +87,8 @@ API 返回的 JSON 响应通过两级路径定位数据：先按 `data_path`（
| `ODS_MEMBER_BALANCE` | ❌ | 默认 | ❌ | ✅ | ✅ | `WINDOW` | `create_time` |
| `ODS_RECHARGE_SETTLE` | ✅ | `(rangeStartTime, rangeEndTime)` | ✅ | ❌ | ✅ | `NONE` | — |
| `ODS_GROUP_PACKAGE` | ❌ | 默认 | ❌ | ✅ | ✅ | `FULL_TABLE` | — |
> `ODS_GROUP_PACKAGE` 额外配置了 `detail_endpoint`，在主流程完成后串行调用 `QueryPackageCouponInfo` 获取每个团购的详情数据，写入 `ods.group_buy_package_details`。
| `ODS_GROUP_BUY_REDEMPTION` | ❌ | 默认 | ❌ | ✅ | ✅ | `WINDOW` | `create_time` |
| `ODS_INVENTORY_STOCK` | ❌ | 默认 | ❌ | ✅ | ✅ | `NONE` | — |
| `ODS_INVENTORY_CHANGE` | ✅ | 默认 | ❌ | ✅ | ✅ | `NONE` | — |

34
apps/etl/connectors/feiqiu/docs/etl_tasks/ods_tasks.md
View File

@@ -7,7 +7,7 @@
## 概述
ODS 层采用**声明式配置**驱动的通用任务模式：由 `BaseOdsTask` + `OdsTaskSpec` 配置驱动，通过 `ODS_TASK_CLASSES` 字典动态注册，共 23 个任务。
ODS 层采用**声明式配置**驱动的通用任务模式：由 `BaseOdsTask` + `OdsTaskSpec` 配置驱动，通过 `ODS_TASK_CLASSES` 字典动态注册，共 22 个任务。
所有 ODS 任务写入 `ods.*` 表，原始 API 响应以 JSON 格式存入 `payload` 列，元数据列（`fetched_at`、`source_file`、`content_hash` 等）自动填充。
@@ -22,7 +22,6 @@ ODS 层采用**声明式配置**驱动的通用任务模式：由 `BaseOdsTask`
| `ODS_SETTLEMENT_RECORDS` | `OdsOrderSettleTask` | `/Site/GetAllOrderSettleList` | `settlement_records` | 结账记录 |
| `ODS_TABLE_USE` | `OdsTableUseTask` | `/Site/GetSiteTableOrderDetails` | `table_fee_transactions` | 台费计费流水 |
| `ODS_ASSISTANT_LEDGER` | `OdsAssistantLedgerTask` | `/AssistantPerformance/GetOrderAssistantDetails` | `assistant_service_records` | 助教服务流水 |
| `ODS_ASSISTANT_ABOLISH` | `OdsAssistantAbolishTask` | `/AssistantPerformance/GetAbolitionAssistant` | `assistant_cancellation_records` | 助教废除记录 |
| `ODS_STORE_GOODS_SALES` | `OdsGoodsLedgerTask` | `/TenantGoods/GetGoodsSalesList` | `store_goods_sales_records` | 门店商品销售流水 |
| `ODS_PAYMENT` | `OdsPaymentTask` | `/PayLog/GetPayLogListPage` | `payment_transactions` | 支付流水 |
| `ODS_REFUND` | `OdsRefundTask` | `/Order/GetRefundPayLogList` | `refund_transactions` | 退款流水 |
@@ -31,7 +30,7 @@ ODS 层采用**声明式配置**驱动的通用任务模式：由 `BaseOdsTask`
| `ODS_MEMBER_CARD` | `OdsMemberCardTask` | `/MemberProfile/GetTenantMemberCardList` | `member_stored_value_cards` | 会员储值卡 |
| `ODS_MEMBER_BALANCE` | `OdsMemberBalanceTask` | `/MemberProfile/GetMemberCardBalanceChange` | `member_balance_changes` | 会员余额变动 |
| `ODS_RECHARGE_SETTLE` | `OdsRechargeSettleTask` | `/Site/GetRechargeSettleList` | `recharge_settlements` | 充值结算 |
| `ODS_GROUP_PACKAGE` | `OdsPackageTask` | `/PackageCoupon/QueryPackageCouponList` | `group_buy_packages` | 团购套餐定义 |
| `ODS_GROUP_PACKAGE` | `OdsPackageTask` | `/PackageCoupon/QueryPackageCouponList` | `group_buy_packages` | 团购套餐定义（含详情子流程，见下方说明） |
| `ODS_GROUP_BUY_REDEMPTION` | `OdsGroupBuyRedemptionTask` | `/Site/GetSiteTableUseDetails` | `group_buy_redemption_records` | 团购套餐核销 |
| `ODS_INVENTORY_STOCK` | `OdsInventoryStockTask` | `/TenantGoods/GetGoodsStockReport` | `goods_stock_summary` | 库存汇总 |
| `ODS_INVENTORY_CHANGE` | `OdsInventoryChangeTask` | `/GoodsStockManage/QueryGoodsOutboundReceipt` | `goods_stock_movements` | 库存变化记录 |
@@ -44,6 +43,26 @@ ODS 层采用**声明式配置**驱动的通用任务模式：由 `BaseOdsTask`
> 所有目标表均位于 `ods` schema 下。
### ODS_GROUP_PACKAGE 详情子流程
`ODS_GROUP_PACKAGE` 任务通过 `detail_endpoint` 配置启用了二级详情拉取：
| 配置项 | 值 |
|--------|-----|
| `detail_endpoint` | `/PackageCoupon/QueryPackageCouponInfo` |
| `detail_target_table` | `ods.group_buy_package_details` |
| `detail_param_builder` | `lambda rec: {"couponId": rec["id"]}` |
| `detail_data_path` | `("data",)` |
| `detail_id_column` | `id` |
执行流程：
1. 主流程从 `QueryPackageCouponList` 拉取团购列表 → 写入 `ods.group_buy_packages`
2. 子流程从 `ods.group_buy_packages` 提取所有 `id`
3. 串行调用 `QueryPackageCouponInfo`（通过 `UnifiedPipeline` + `RateLimiter`），获取每个团购的详情
4. 详情数据经字段提取后写入 `ods.group_buy_package_details`（全量快照，UPSERT on `coupon_id`）
详情表字段映射见 `docs/database/ODS/mappings/mapping_QueryPackageCouponInfo_group_buy_package_details.md`。
---
## 通用 ODS 任务架构（BaseOdsTask + OdsTaskSpec 模式）
@@ -228,7 +247,7 @@ execute(cursor_data)
### content_hash 去重机制
`content_hash` 是通用 ODS 任务的核心去重手段，所有 23 个任务默认开启（`skip_unchanged=True`）。
`content_hash` 是通用 ODS 任务的核心去重手段，所有 22 个任务默认开启（`skip_unchanged=True`）。
#### 计算方式
@@ -277,8 +296,7 @@ ORDER BY id, fetched_at DESC;
| `ODS_SETTLEMENT_RECORDS` | 是 | `NONE` | — | 结账记录，按时间窗口增量抓取 |
| `ODS_TABLE_USE` | 否 | `WINDOW` | `create_time` | 台费计费流水 |
| `ODS_ASSISTANT_LEDGER` | 是 | `WINDOW` | `create_time` | 助教服务流水 |
| `ODS_ASSISTANT_ABOLISH` | 是 | `NONE` | — | 助教废除记录 |
| `ODS_STORE_GOODS_SALES` | 否 | `WINDOW` | `create_time` | 门店商品销售流水 |
| `ODS_STORE_GOODS_SALES` | 是 | `WINDOW` | `create_time` | 门店商品销售流水（2026-03-01 修复：`requires_window` 从 `False` 改为 `True`，新增 `time_fields=("startTime", "endTime")`） |
| `ODS_PAYMENT` | 否 | `NONE` | — | 支付流水 |
| `ODS_REFUND` | 否 | `WINDOW` | `pay_time` | 退款流水 |
| `ODS_PLATFORM_COUPON` | 否 | `WINDOW` | `consume_time` | 平台/团购券核销 |
@@ -286,7 +304,7 @@ ORDER BY id, fetched_at DESC;
| `ODS_MEMBER_CARD` | 否 | `FULL_TABLE` | — | 会员储值卡 |
| `ODS_MEMBER_BALANCE` | 否 | `WINDOW` | `create_time` | 会员余额变动 |
| `ODS_RECHARGE_SETTLE` | 是 | `NONE` | — | 充值结算 |
| `ODS_GROUP_PACKAGE` | 否 | `FULL_TABLE` | — | 团购套餐定义 |
| `ODS_GROUP_PACKAGE` | 否 | `FULL_TABLE` | — | 团购套餐定义 + 详情子流程（`detail_endpoint`） |
| `ODS_GROUP_BUY_REDEMPTION` | 否 | `WINDOW` | `create_time` | 团购套餐核销 |
| `ODS_INVENTORY_STOCK` | 否 | `NONE` | — | 库存汇总 |
| `ODS_INVENTORY_CHANGE` | 是 | `NONE` | — | 库存变化记录 |
@@ -297,4 +315,4 @@ ORDER BY id, fetched_at DESC;
| `ODS_TENANT_GOODS` | 否 | `FULL_TABLE` | — | 租户商品档案 |
| `ODS_STAFF_INFO` | 否 | `FULL_TABLE` | — | 员工档案，全量快照 |
> 所有 23 个任务默认 `skip_unchanged=True`（去重开启）。
> 所有 22 个任务默认 `skip_unchanged=True`（去重开启）。

1
apps/etl/connectors/feiqiu/docs/etl_tasks/utility_tasks.md
View File

@@ -283,7 +283,6 @@ execute()
| `member_stored_value_cards` | `ods.member_stored_value_cards` |
| `recharge_settlements` | `ods.recharge_settlements` |
| `settlement_records` | `ods.settlement_records` |
| `assistant_cancellation_records` | `ods.assistant_cancellation_records` |
| `assistant_accounts_master` | `ods.assistant_accounts_master` |
| `assistant_service_records` | `ods.assistant_service_records` |
| `site_tables_master` | `ods.site_tables_master` |

9
apps/etl/connectors/feiqiu/orchestration/flow_runner.py
View File

@@ -247,6 +247,7 @@ class FlowRunner:
"""ETL 完成后运行数据一致性检查，输出黑盒测试报告。
返回报告文件路径，失败时返回 None（不阻断主流程）。
CHANGE 2026-02-26 | 改用 FETCH_ROOT 读取实际抓取数据，替代 API_SAMPLE_CACHE_ROOT
"""
try:
from quality.consistency_checker import (
@@ -259,13 +260,19 @@ class FlowRunner:
timer.start_step("CONSISTENCY_CHECK")
try:
# 优先使用 FETCH_ROOT（ETL 实际抓取的分页 JSON）
fetch_root_str = os.environ.get("FETCH_ROOT")
fetch_root = Path(fetch_root_str) if fetch_root_str else None
# 兼容保留：api_sample_dir 作为回退
api_sample_dir_str = os.environ.get("API_SAMPLE_CACHE_ROOT")
api_sample_dir = Path(api_sample_dir_str) if api_sample_dir_str else None
report = run_consistency_check(
self.db_conn,
fetch_root=fetch_root,
api_sample_dir=api_sample_dir,
include_api_vs_ods=bool(api_sample_dir),
include_api_vs_ods=bool(fetch_root or api_sample_dir),
include_ods_vs_dwd=True,
tz=self.tz,
)

7
apps/etl/connectors/feiqiu/orchestration/task_executor.py
View File

@@ -26,6 +26,7 @@ from api.local_json_client import LocalJsonClient
from orchestration.cursor_manager import CursorManager
from orchestration.run_tracker import RunTracker
from orchestration.task_registry import TaskRegistry
from utils.task_log_buffer import TaskLogBuffer
class DataSource(str, Enum):
@@ -90,6 +91,8 @@ class TaskExecutor:
self.logger.info("开始运行任务: %s, run_uuid=%s", task_codes, run_uuid)
for task_code in task_codes:
# 为每个任务创建独立的日志缓冲区，避免多任务日志交叉
task_log_buf = TaskLogBuffer(task_code, self.logger)
try:
task_result = self.run_single_task(
task_code, run_uuid, store_id, data_source=data_source,
@@ -107,6 +110,7 @@ class TaskExecutor:
results.append(result_entry)
except Exception as exc: # noqa: BLE001
self.logger.error("任务 %s 失败: %s", task_code, exc, exc_info=True)
task_log_buf.error("任务失败: %s", exc)
# CHANGE 2026-02-24 | 任务失败后 rollback，防止 InFailedSqlTransaction 级联
try:
self.db.rollback()
@@ -119,6 +123,9 @@ class TaskExecutor:
"counts": {},
})
continue
finally:
# 任务完成（无论成功/失败），一次性输出该任务的缓冲日志
task_log_buf.flush()
self.logger.info("所有任务执行完成")
return results

6
apps/etl/connectors/feiqiu/orchestration/task_registry.py
View File

@@ -37,6 +37,8 @@ from tasks.dws import (
AssistantFinanceTask,
MemberConsumptionTask,
MemberVisitTask,
AssistantProjectTagTask,
MemberProjectTagTask,
FinanceDailyTask,
FinanceRechargeTask,
FinanceIncomeStructureTask,
@@ -156,6 +158,9 @@ default_registry.register("DWS_ASSISTANT_SALARY", AssistantSalaryTask, layer="DW
default_registry.register("DWS_ASSISTANT_FINANCE", AssistantFinanceTask, layer="DWS", depends_on=["DWS_ASSISTANT_SALARY"])
default_registry.register("DWS_MEMBER_CONSUMPTION", MemberConsumptionTask, layer="DWS")
default_registry.register("DWS_MEMBER_VISIT", MemberVisitTask, layer="DWS")
# CHANGE [2026-03-07] intent: 注册项目标签任务，依赖 DWD 装载完成
default_registry.register("DWS_ASSISTANT_PROJECT_TAG", AssistantProjectTagTask, layer="DWS", depends_on=["DWD_LOAD_FROM_ODS"])
default_registry.register("DWS_MEMBER_PROJECT_TAG", MemberProjectTagTask, layer="DWS", depends_on=["DWD_LOAD_FROM_ODS"])
default_registry.register("DWS_FINANCE_DAILY", FinanceDailyTask, layer="DWS")
default_registry.register("DWS_FINANCE_RECHARGE", FinanceRechargeTask, layer="DWS")
default_registry.register("DWS_FINANCE_INCOME_STRUCTURE", FinanceIncomeStructureTask, layer="DWS")
@@ -172,6 +177,7 @@ default_registry.register("DWS_MAINTENANCE", DwsMaintenanceTask, layer="DWS", de
"DWS_ASSISTANT_MONTHLY", "DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_SALARY", "DWS_ASSISTANT_FINANCE",
"DWS_MEMBER_CONSUMPTION", "DWS_MEMBER_VISIT",
"DWS_ASSISTANT_PROJECT_TAG", "DWS_MEMBER_PROJECT_TAG",
"DWS_FINANCE_DAILY", "DWS_FINANCE_RECHARGE",
"DWS_FINANCE_INCOME_STRUCTURE", "DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_BUILD_ORDER_SUMMARY",

Some files were not shown because too many files have changed in this diff Show More