在准备环境前提交次全部更改。

This commit is contained in:
Neo
2026-02-19 08:35:13 +08:00
parent ded6dfb9d8
commit 4eac07da47
1387 changed files with 6107191 additions and 33002 deletions

View File

@@ -0,0 +1,97 @@
"""
认证路由:登录与令牌刷新。
- POST /api/auth/login — 验证用户名密码,返回 JWT 令牌对
- POST /api/auth/refresh — 用刷新令牌换取新的访问令牌
"""
import logging
from fastapi import APIRouter, HTTPException, status
from jose import JWTError
from app.auth.jwt import (
create_access_token,
create_token_pair,
decode_refresh_token,
verify_password,
)
from app.database import get_connection
from app.schemas.auth import LoginRequest, RefreshRequest, TokenResponse
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/auth", tags=["认证"])
@router.post("/login", response_model=TokenResponse)
async def login(body: LoginRequest):
"""
用户登录。
查询 admin_users 表验证用户名密码,成功后返回 JWT 令牌对。
- 用户不存在或密码错误401
- 账号已禁用is_active=false401
"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"SELECT id, password_hash, site_id, is_active "
"FROM admin_users WHERE username = %s",
(body.username,),
)
row = cur.fetchone()
finally:
conn.close()
if row is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="用户名或密码错误",
)
user_id, password_hash, site_id, is_active = row
if not is_active:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="账号已被禁用",
)
if not verify_password(body.password, password_hash):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="用户名或密码错误",
)
tokens = create_token_pair(user_id, site_id)
return TokenResponse(**tokens)
@router.post("/refresh", response_model=TokenResponse)
async def refresh(body: RefreshRequest):
"""
刷新访问令牌。
验证 refresh_token 有效性,成功后仅返回新的 access_token
refresh_token 保持不变,由客户端继续持有)。
"""
try:
payload = decode_refresh_token(body.refresh_token)
except JWTError:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="无效的刷新令牌",
)
user_id = int(payload["sub"])
site_id = payload["site_id"]
# 生成新的 access_tokenrefresh_token 原样返回
new_access = create_access_token(user_id, site_id)
return TokenResponse(
access_token=new_access,
refresh_token=body.refresh_token,
token_type="bearer",
)

View File

@@ -0,0 +1,228 @@
# -*- coding: utf-8 -*-
"""数据库查看器 API
提供 4 个端点:
- GET /api/db/schemas — 返回 Schema 列表
- GET /api/db/schemas/{name}/tables — 返回表列表和行数
- GET /api/db/tables/{schema}/{table}/columns — 返回列定义
- POST /api/db/query — 只读 SQL 执行
所有端点需要 JWT 认证。
使用 get_etl_readonly_connection(site_id) 确保 RLS 隔离。
"""
from __future__ import annotations
import logging
import re
from fastapi import APIRouter, Depends, HTTPException, status
from psycopg2 import errors as pg_errors, OperationalError
from app.auth.dependencies import CurrentUser, get_current_user
from app.database import get_etl_readonly_connection
from app.schemas.db_viewer import (
ColumnInfo,
QueryRequest,
QueryResponse,
SchemaInfo,
TableInfo,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/db", tags=["数据库查看器"])
# 写操作关键词(不区分大小写)
_WRITE_KEYWORDS = re.compile(
r"\b(INSERT|UPDATE|DELETE|DROP|TRUNCATE)\b",
re.IGNORECASE,
)
# 查询结果行数上限
_MAX_ROWS = 1000
# 查询超时(秒)
_QUERY_TIMEOUT_SEC = 30
# ── GET /api/db/schemas ──────────────────────────────────────
@router.get("/schemas", response_model=list[SchemaInfo])
async def list_schemas(
user: CurrentUser = Depends(get_current_user),
) -> list[SchemaInfo]:
"""返回 ETL 数据库中的 Schema 列表。"""
conn = get_etl_readonly_connection(user.site_id)
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT schema_name
FROM information_schema.schemata
WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
ORDER BY schema_name
"""
)
rows = cur.fetchall()
return [SchemaInfo(name=row[0]) for row in rows]
finally:
conn.close()
# ── GET /api/db/schemas/{name}/tables ────────────────────────
@router.get("/schemas/{name}/tables", response_model=list[TableInfo])
async def list_tables(
name: str,
user: CurrentUser = Depends(get_current_user),
) -> list[TableInfo]:
"""返回指定 Schema 下所有表的名称和行数统计。"""
conn = get_etl_readonly_connection(user.site_id)
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
t.table_name,
s.n_live_tup
FROM information_schema.tables t
LEFT JOIN pg_stat_user_tables s
ON s.schemaname = t.table_schema
AND s.relname = t.table_name
WHERE t.table_schema = %s
AND t.table_type = 'BASE TABLE'
ORDER BY t.table_name
""",
(name,),
)
rows = cur.fetchall()
return [
TableInfo(name=row[0], row_count=row[1])
for row in rows
]
finally:
conn.close()
# ── GET /api/db/tables/{schema}/{table}/columns ──────────────
@router.get(
"/tables/{schema}/{table}/columns",
response_model=list[ColumnInfo],
)
async def list_columns(
schema: str,
table: str,
user: CurrentUser = Depends(get_current_user),
) -> list[ColumnInfo]:
"""返回指定表的列定义。"""
conn = get_etl_readonly_connection(user.site_id)
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
column_name,
data_type,
is_nullable,
column_default
FROM information_schema.columns
WHERE table_schema = %s AND table_name = %s
ORDER BY ordinal_position
""",
(schema, table),
)
rows = cur.fetchall()
return [
ColumnInfo(
name=row[0],
data_type=row[1],
is_nullable=row[2] == "YES",
column_default=row[3],
)
for row in rows
]
finally:
conn.close()
# ── POST /api/db/query ───────────────────────────────────────
@router.post("/query", response_model=QueryResponse)
async def execute_query(
body: QueryRequest,
user: CurrentUser = Depends(get_current_user),
) -> QueryResponse:
"""只读 SQL 执行。
安全措施:
1. 拦截写操作关键词INSERT / UPDATE / DELETE / DROP / TRUNCATE
2. 限制返回行数上限 1000 行
3. 设置查询超时 30 秒
"""
sql = body.sql.strip()
if not sql:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="SQL 语句不能为空",
)
# 拦截写操作
if _WRITE_KEYWORDS.search(sql):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="只允许只读查询,禁止 INSERT / UPDATE / DELETE / DROP / TRUNCATE 操作",
)
conn = get_etl_readonly_connection(user.site_id)
try:
with conn.cursor() as cur:
# 设置查询超时
cur.execute(
"SET LOCAL statement_timeout = %s",
(f"{_QUERY_TIMEOUT_SEC}s",),
)
try:
cur.execute(sql)
except pg_errors.QueryCanceled:
raise HTTPException(
status_code=status.HTTP_408_REQUEST_TIMEOUT,
detail=f"查询超时(超过 {_QUERY_TIMEOUT_SEC} 秒)",
)
except Exception as exc:
# SQL 语法错误或其他执行错误
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"SQL 执行错误: {exc}",
)
# 提取列名
columns = (
[desc[0] for desc in cur.description]
if cur.description
else []
)
# 限制返回行数
rows = cur.fetchmany(_MAX_ROWS)
# 将元组转为列表,便于 JSON 序列化
rows_list = [list(row) for row in rows]
return QueryResponse(
columns=columns,
rows=rows_list,
row_count=len(rows_list),
)
except HTTPException:
raise
except OperationalError as exc:
# 连接级错误
logger.error("数据库查看器连接错误: %s", exc)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="数据库连接错误",
)
finally:
conn.close()

View File

@@ -0,0 +1,240 @@
# -*- coding: utf-8 -*-
"""环境配置 API
提供 3 个端点:
- GET /api/env-config — 读取 .env敏感值掩码
- PUT /api/env-config — 验证并写入 .env
- GET /api/env-config/export — 导出去敏感值的配置文件
所有端点需要 JWT 认证。
敏感键判定:键名中包含 PASSWORD、TOKEN、SECRET、DSN不区分大小写
"""
from __future__ import annotations
import logging
import re
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.responses import PlainTextResponse
from pydantic import BaseModel
from app.auth.dependencies import CurrentUser, get_current_user
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/env-config", tags=["环境配置"])
# .env 文件路径:项目根目录
_ENV_PATH = Path(__file__).resolve().parents[3] / ".env"
# 敏感键关键词(不区分大小写)
_SENSITIVE_KEYWORDS = ("PASSWORD", "TOKEN", "SECRET", "DSN")
_MASK = "****"
# ── Pydantic 模型 ────────────────────────────────────────────
class EnvEntry(BaseModel):
"""单条环境变量键值对。"""
key: str
value: str
class EnvConfigResponse(BaseModel):
"""GET 响应:键值对列表。"""
entries: list[EnvEntry]
class EnvConfigUpdateRequest(BaseModel):
"""PUT 请求体:键值对列表。"""
entries: list[EnvEntry]
# ── 工具函数 ─────────────────────────────────────────────────
def _is_sensitive(key: str) -> bool:
"""判断键名是否为敏感键。"""
upper = key.upper()
return any(kw in upper for kw in _SENSITIVE_KEYWORDS)
def _parse_env(content: str) -> list[dict]:
"""解析 .env 文件内容,返回行级结构。
每行分为三种类型:
- comment: 注释行或空行(原样保留)
- entry: 键值对
"""
lines: list[dict] = []
for raw_line in content.splitlines():
stripped = raw_line.strip()
if not stripped or stripped.startswith("#"):
lines.append({"type": "comment", "raw": raw_line})
else:
# 支持 KEY=VALUE 和 KEY="VALUE" 格式
match = re.match(r'^([A-Za-z_][A-Za-z0-9_]*)=(.*)', raw_line)
if match:
key = match.group(1)
value = match.group(2).strip()
# 去除引号包裹
if len(value) >= 2 and value[0] == value[-1] and value[0] in ('"', "'"):
value = value[1:-1]
lines.append({"type": "entry", "key": key, "value": value, "raw": raw_line})
else:
# 无法解析的行当作注释保留
lines.append({"type": "comment", "raw": raw_line})
return lines
def _read_env_file(path: Path) -> str:
"""读取 .env 文件内容。"""
if not path.exists():
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=".env 文件不存在",
)
return path.read_text(encoding="utf-8")
def _write_env_file(path: Path, content: str) -> None:
"""写入 .env 文件。"""
try:
path.write_text(content, encoding="utf-8")
except OSError as exc:
logger.error("写入 .env 文件失败: %s", exc)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="写入 .env 文件失败",
)
def _validate_entries(entries: list[EnvEntry]) -> None:
"""验证键值对格式。"""
for idx, entry in enumerate(entries):
if not entry.key:
raise HTTPException(
status_code=422,
detail=f"{idx + 1} 行:键名不能为空",
)
if not re.match(r'^[A-Za-z_][A-Za-z0-9_]*$', entry.key):
raise HTTPException(
status_code=422,
detail=f"{idx + 1} 行:键名 '{entry.key}' 格式无效(仅允许字母、数字、下划线,且不能以数字开头)",
)
# ── GET /api/env-config — 读取 ───────────────────────────────
@router.get("", response_model=EnvConfigResponse)
async def get_env_config(
user: CurrentUser = Depends(get_current_user),
) -> EnvConfigResponse:
"""读取 .env 文件,敏感值以掩码展示。"""
content = _read_env_file(_ENV_PATH)
parsed = _parse_env(content)
entries = []
for line in parsed:
if line["type"] == "entry":
value = _MASK if _is_sensitive(line["key"]) else line["value"]
entries.append(EnvEntry(key=line["key"], value=value))
return EnvConfigResponse(entries=entries)
# ── PUT /api/env-config — 写入 ───────────────────────────────
@router.put("", response_model=EnvConfigResponse)
async def update_env_config(
body: EnvConfigUpdateRequest,
user: CurrentUser = Depends(get_current_user),
) -> EnvConfigResponse:
"""验证并写入 .env 文件。
保留原文件中的注释行和空行。对于已有键,更新值;
对于新键,追加到文件末尾。掩码值(****)的键跳过更新,保留原值。
"""
_validate_entries(body.entries)
# 读取原文件(如果存在)
if _ENV_PATH.exists():
original_content = _ENV_PATH.read_text(encoding="utf-8")
parsed = _parse_env(original_content)
else:
parsed = []
# 构建新值映射(跳过掩码值)
new_values: dict[str, str] = {}
for entry in body.entries:
if entry.value != _MASK:
new_values[entry.key] = entry.value
# 更新已有行
seen_keys: set[str] = set()
output_lines: list[str] = []
for line in parsed:
if line["type"] == "comment":
output_lines.append(line["raw"])
elif line["type"] == "entry":
key = line["key"]
seen_keys.add(key)
if key in new_values:
output_lines.append(f"{key}={new_values[key]}")
else:
# 保留原值(包括掩码跳过的敏感键)
output_lines.append(line["raw"])
# 追加新键
for entry in body.entries:
if entry.key not in seen_keys and entry.value != _MASK:
output_lines.append(f"{entry.key}={entry.value}")
new_content = "\n".join(output_lines)
if output_lines:
new_content += "\n"
_write_env_file(_ENV_PATH, new_content)
# 返回更新后的配置(敏感值掩码)
result_parsed = _parse_env(new_content)
entries = []
for line in result_parsed:
if line["type"] == "entry":
value = _MASK if _is_sensitive(line["key"]) else line["value"]
entries.append(EnvEntry(key=line["key"], value=value))
return EnvConfigResponse(entries=entries)
# ── GET /api/env-config/export — 导出 ────────────────────────
@router.get("/export")
async def export_env_config(
user: CurrentUser = Depends(get_current_user),
) -> PlainTextResponse:
"""导出去除敏感值的配置文件(作为文件下载)。"""
content = _read_env_file(_ENV_PATH)
parsed = _parse_env(content)
output_lines: list[str] = []
for line in parsed:
if line["type"] == "comment":
output_lines.append(line["raw"])
elif line["type"] == "entry":
if _is_sensitive(line["key"]):
output_lines.append(f"{line['key']}={_MASK}")
else:
output_lines.append(line["raw"])
export_content = "\n".join(output_lines)
if output_lines:
export_content += "\n"
return PlainTextResponse(
content=export_content,
media_type="text/plain",
headers={"Content-Disposition": "attachment; filename=env-config.txt"},
)

View File

@@ -0,0 +1,134 @@
# -*- coding: utf-8 -*-
"""ETL 状态监控 API
提供 2 个端点:
- GET /api/etl-status/cursors — 返回各任务的数据游标(最后抓取时间、记录数)
- GET /api/etl-status/recent-runs — 返回最近 50 条任务执行记录
所有端点需要 JWT 认证。
游标端点查询 ETL 数据库meta.etl_cursor
执行记录端点查询 zqyy_app 数据库task_execution_log
"""
from __future__ import annotations
import logging
from fastapi import APIRouter, Depends, HTTPException, status
from psycopg2 import OperationalError
from app.auth.dependencies import CurrentUser, get_current_user
from app.database import get_connection, get_etl_readonly_connection
from app.schemas.etl_status import CursorInfo, RecentRun
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/etl-status", tags=["ETL 状态"])
# 最近执行记录条数上限
_RECENT_RUNS_LIMIT = 50
# ── GET /api/etl-status/cursors ──────────────────────────────
@router.get("/cursors", response_model=list[CursorInfo])
async def list_cursors(
user: CurrentUser = Depends(get_current_user),
) -> list[CursorInfo]:
"""返回各 ODS 表的最新数据游标。
查询 ETL 数据库中的 meta.etl_cursor 表。
如果该表不存在,返回空列表而非报错。
"""
conn = get_etl_readonly_connection(user.site_id)
try:
with conn.cursor() as cur:
# CHANGE 2026-02-15 | 对齐新库 etl_feiqiu 六层架构etl_admin → meta
cur.execute(
"""
SELECT EXISTS (
SELECT 1
FROM information_schema.tables
WHERE table_schema = 'meta'
AND table_name = 'etl_cursor'
)
"""
)
exists = cur.fetchone()[0]
if not exists:
return []
cur.execute(
"""
SELECT task_code, last_fetch_time, record_count
FROM meta.etl_cursor
ORDER BY task_code
"""
)
rows = cur.fetchall()
return [
CursorInfo(
task_code=row[0],
last_fetch_time=str(row[1]) if row[1] is not None else None,
record_count=row[2],
)
for row in rows
]
except OperationalError as exc:
logger.error("ETL 游标查询连接错误: %s", exc)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="ETL 数据库连接错误",
)
finally:
conn.close()
# ── GET /api/etl-status/recent-runs ──────────────────────────
@router.get("/recent-runs", response_model=list[RecentRun])
async def list_recent_runs(
user: CurrentUser = Depends(get_current_user),
) -> list[RecentRun]:
"""返回最近 50 条任务执行记录。
查询 zqyy_app 数据库中的 task_execution_log 表,
按 site_id 过滤,按 started_at DESC 排序。
"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, task_codes, status, started_at,
finished_at, duration_ms, exit_code
FROM task_execution_log
WHERE site_id = %s
ORDER BY started_at DESC
LIMIT %s
""",
(user.site_id, _RECENT_RUNS_LIMIT),
)
rows = cur.fetchall()
return [
RecentRun(
id=str(row[0]),
task_codes=list(row[1]) if row[1] else [],
status=row[2],
started_at=str(row[3]),
finished_at=str(row[4]) if row[4] is not None else None,
duration_ms=row[5],
exit_code=row[6],
)
for row in rows
]
except OperationalError as exc:
logger.error("执行记录查询连接错误: %s", exc)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="数据库连接错误",
)
finally:
conn.close()

View File

@@ -0,0 +1,281 @@
# -*- coding: utf-8 -*-
"""执行与队列 API
提供 8 个端点:
- POST /api/execution/run — 直接执行任务
- GET /api/execution/queue — 获取当前队列(按 site_id 过滤)
- POST /api/execution/queue — 添加到队列
- PUT /api/execution/queue/reorder — 重排队列
- DELETE /api/execution/queue/{id} — 删除队列任务
- POST /api/execution/{id}/cancel — 取消执行中的任务
- GET /api/execution/history — 执行历史(按 site_id 过滤)
- GET /api/execution/{id}/logs — 获取历史日志
所有端点需要 JWT 认证site_id 从 JWT 提取。
"""
from __future__ import annotations
import asyncio
import json
import logging
import uuid
from fastapi import APIRouter, Depends, HTTPException, Query, status
from app.auth.dependencies import CurrentUser, get_current_user
from app.database import get_connection
from app.schemas.execution import (
ExecutionHistoryItem,
ExecutionLogsResponse,
ExecutionRunResponse,
QueueTaskResponse,
ReorderRequest,
)
from app.schemas.tasks import TaskConfigSchema
from app.services.task_executor import task_executor
from app.services.task_queue import task_queue
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/execution", tags=["任务执行"])
# ── POST /api/execution/run — 直接执行任务 ────────────────────
@router.post("/run", response_model=ExecutionRunResponse)
async def run_task(
config: TaskConfigSchema,
user: CurrentUser = Depends(get_current_user),
) -> ExecutionRunResponse:
"""直接执行任务(不经过队列)。
从 JWT 注入 store_id创建 execution_id 后异步启动子进程。
"""
config = config.model_copy(update={"store_id": user.site_id})
execution_id = str(uuid.uuid4())
# 异步启动执行,不阻塞响应
asyncio.create_task(
task_executor.execute(
config=config,
execution_id=execution_id,
site_id=user.site_id,
)
)
return ExecutionRunResponse(
execution_id=execution_id,
message="任务已提交执行",
)
# ── GET /api/execution/queue — 获取当前队列 ───────────────────
@router.get("/queue", response_model=list[QueueTaskResponse])
async def get_queue(
user: CurrentUser = Depends(get_current_user),
) -> list[QueueTaskResponse]:
"""获取当前门店的待执行队列。"""
tasks = task_queue.list_pending(user.site_id)
return [
QueueTaskResponse(
id=t.id,
site_id=t.site_id,
config=t.config,
status=t.status,
position=t.position,
created_at=t.created_at,
started_at=t.started_at,
finished_at=t.finished_at,
exit_code=t.exit_code,
error_message=t.error_message,
)
for t in tasks
]
# ── POST /api/execution/queue — 添加到队列 ───────────────────
@router.post("/queue", response_model=QueueTaskResponse, status_code=status.HTTP_201_CREATED)
async def enqueue_task(
config: TaskConfigSchema,
user: CurrentUser = Depends(get_current_user),
) -> QueueTaskResponse:
"""将任务配置添加到执行队列。"""
config = config.model_copy(update={"store_id": user.site_id})
task_id = task_queue.enqueue(config, user.site_id)
# 查询刚创建的任务返回完整信息
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, site_id, config, status, position,
created_at, started_at, finished_at,
exit_code, error_message
FROM task_queue WHERE id = %s
""",
(task_id,),
)
row = cur.fetchone()
conn.commit()
finally:
conn.close()
if row is None:
raise HTTPException(status_code=500, detail="入队后查询失败")
config_data = row[2] if isinstance(row[2], dict) else json.loads(row[2])
return QueueTaskResponse(
id=str(row[0]),
site_id=row[1],
config=config_data,
status=row[3],
position=row[4],
created_at=row[5],
started_at=row[6],
finished_at=row[7],
exit_code=row[8],
error_message=row[9],
)
# ── PUT /api/execution/queue/reorder — 重排队列 ──────────────
@router.put("/queue/reorder")
async def reorder_queue(
body: ReorderRequest,
user: CurrentUser = Depends(get_current_user),
) -> dict:
"""调整队列中任务的执行顺序。"""
task_queue.reorder(body.task_id, body.new_position, user.site_id)
return {"message": "队列已重排"}
# ── DELETE /api/execution/queue/{id} — 删除队列任务 ──────────
@router.delete("/queue/{task_id}")
async def delete_queue_task(
task_id: str,
user: CurrentUser = Depends(get_current_user),
) -> dict:
"""从队列中删除待执行任务。仅允许删除 pending 状态的任务。"""
deleted = task_queue.delete(task_id, user.site_id)
if not deleted:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="任务不存在或非待执行状态,无法删除",
)
return {"message": "任务已从队列中删除"}
# ── POST /api/execution/{id}/cancel — 取消执行 ──────────────
@router.post("/{execution_id}/cancel")
async def cancel_execution(
execution_id: str,
user: CurrentUser = Depends(get_current_user),
) -> dict:
"""取消正在执行的任务。"""
cancelled = await task_executor.cancel(execution_id)
if not cancelled:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="执行任务不存在或已完成",
)
return {"message": "已发送取消信号"}
# ── GET /api/execution/history — 执行历史 ────────────────────
@router.get("/history", response_model=list[ExecutionHistoryItem])
async def get_execution_history(
limit: int = Query(default=50, ge=1, le=200),
user: CurrentUser = Depends(get_current_user),
) -> list[ExecutionHistoryItem]:
"""获取执行历史记录,按 started_at 降序排列。"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, site_id, task_codes, status, started_at,
finished_at, exit_code, duration_ms, command, summary
FROM task_execution_log
WHERE site_id = %s
ORDER BY started_at DESC
LIMIT %s
""",
(user.site_id, limit),
)
rows = cur.fetchall()
conn.commit()
finally:
conn.close()
return [
ExecutionHistoryItem(
id=str(row[0]),
site_id=row[1],
task_codes=row[2] or [],
status=row[3],
started_at=row[4],
finished_at=row[5],
exit_code=row[6],
duration_ms=row[7],
command=row[8],
summary=row[9],
)
for row in rows
]
# ── GET /api/execution/{id}/logs — 获取历史日志 ──────────────
@router.get("/{execution_id}/logs", response_model=ExecutionLogsResponse)
async def get_execution_logs(
execution_id: str,
user: CurrentUser = Depends(get_current_user),
) -> ExecutionLogsResponse:
"""获取指定执行的完整日志。
优先从内存缓冲区读取(执行中),否则从数据库读取(已完成)。
"""
# 先尝试内存缓冲区(执行中的任务)
if task_executor.is_running(execution_id):
lines = task_executor.get_logs(execution_id)
return ExecutionLogsResponse(
execution_id=execution_id,
output_log="\n".join(lines) if lines else None,
)
# 从数据库读取已完成任务的日志
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT output_log, error_log
FROM task_execution_log
WHERE id = %s AND site_id = %s
""",
(execution_id, user.site_id),
)
row = cur.fetchone()
conn.commit()
finally:
conn.close()
if row is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="执行记录不存在",
)
return ExecutionLogsResponse(
execution_id=execution_id,
output_log=row[0],
error_log=row[1],
)

View File

@@ -0,0 +1,293 @@
# -*- coding: utf-8 -*-
"""调度任务 CRUD API
提供 5 个端点:
- GET /api/schedules — 列表(按 site_id 过滤)
- POST /api/schedules — 创建
- PUT /api/schedules/{id} — 更新
- DELETE /api/schedules/{id} — 删除
- PATCH /api/schedules/{id}/toggle — 启用/禁用
所有端点需要 JWT 认证site_id 从 JWT 提取。
"""
from __future__ import annotations
import json
import logging
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, status
from app.auth.dependencies import CurrentUser, get_current_user
from app.database import get_connection
from app.schemas.schedules import (
CreateScheduleRequest,
ScheduleResponse,
UpdateScheduleRequest,
)
from app.services.scheduler import calculate_next_run
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/schedules", tags=["调度管理"])
def _row_to_response(row) -> ScheduleResponse:
"""将数据库行转换为 ScheduleResponse。"""
task_config = row[4] if isinstance(row[4], dict) else json.loads(row[4])
schedule_config = row[5] if isinstance(row[5], dict) else json.loads(row[5])
return ScheduleResponse(
id=str(row[0]),
site_id=row[1],
name=row[2],
task_codes=row[3] or [],
task_config=task_config,
schedule_config=schedule_config,
enabled=row[6],
last_run_at=row[7],
next_run_at=row[8],
run_count=row[9],
last_status=row[10],
created_at=row[11],
updated_at=row[12],
)
# 查询列列表,复用于多个端点
_SELECT_COLS = """
id, site_id, name, task_codes, task_config, schedule_config,
enabled, last_run_at, next_run_at, run_count, last_status,
created_at, updated_at
"""
# ── GET /api/schedules — 列表 ────────────────────────────────
@router.get("", response_model=list[ScheduleResponse])
async def list_schedules(
user: CurrentUser = Depends(get_current_user),
) -> list[ScheduleResponse]:
"""获取当前门店的所有调度任务。"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
f"SELECT {_SELECT_COLS} FROM scheduled_tasks WHERE site_id = %s ORDER BY created_at DESC",
(user.site_id,),
)
rows = cur.fetchall()
conn.commit()
finally:
conn.close()
return [_row_to_response(row) for row in rows]
# ── POST /api/schedules — 创建 ──────────────────────────────
@router.post("", response_model=ScheduleResponse, status_code=status.HTTP_201_CREATED)
async def create_schedule(
body: CreateScheduleRequest,
user: CurrentUser = Depends(get_current_user),
) -> ScheduleResponse:
"""创建调度任务,自动计算 next_run_at。"""
now = datetime.now(timezone.utc)
next_run = calculate_next_run(body.schedule_config, now)
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
f"""
INSERT INTO scheduled_tasks
(site_id, name, task_codes, task_config, schedule_config, enabled, next_run_at)
VALUES (%s, %s, %s, %s, %s, %s, %s)
RETURNING {_SELECT_COLS}
""",
(
user.site_id,
body.name,
body.task_codes,
json.dumps(body.task_config),
body.schedule_config.model_dump_json(),
body.schedule_config.enabled,
next_run,
),
)
row = cur.fetchone()
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
return _row_to_response(row)
# ── PUT /api/schedules/{id} — 更新 ──────────────────────────
@router.put("/{schedule_id}", response_model=ScheduleResponse)
async def update_schedule(
schedule_id: str,
body: UpdateScheduleRequest,
user: CurrentUser = Depends(get_current_user),
) -> ScheduleResponse:
"""更新调度任务,仅更新请求中提供的字段。"""
# 构建动态 SET 子句
set_parts: list[str] = []
params: list = []
if body.name is not None:
set_parts.append("name = %s")
params.append(body.name)
if body.task_codes is not None:
set_parts.append("task_codes = %s")
params.append(body.task_codes)
if body.task_config is not None:
set_parts.append("task_config = %s")
params.append(json.dumps(body.task_config))
if body.schedule_config is not None:
set_parts.append("schedule_config = %s")
params.append(body.schedule_config.model_dump_json())
# 更新调度配置时重新计算 next_run_at
now = datetime.now(timezone.utc)
next_run = calculate_next_run(body.schedule_config, now)
set_parts.append("next_run_at = %s")
params.append(next_run)
if not set_parts:
raise HTTPException(
status_code=422,
detail="至少需要提供一个更新字段",
)
set_parts.append("updated_at = NOW()")
set_clause = ", ".join(set_parts)
params.extend([schedule_id, user.site_id])
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
f"""
UPDATE scheduled_tasks
SET {set_clause}
WHERE id = %s AND site_id = %s
RETURNING {_SELECT_COLS}
""",
params,
)
row = cur.fetchone()
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
if row is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="调度任务不存在",
)
return _row_to_response(row)
# ── DELETE /api/schedules/{id} — 删除 ────────────────────────
@router.delete("/{schedule_id}")
async def delete_schedule(
schedule_id: str,
user: CurrentUser = Depends(get_current_user),
) -> dict:
"""删除调度任务。"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(
"DELETE FROM scheduled_tasks WHERE id = %s AND site_id = %s",
(schedule_id, user.site_id),
)
deleted = cur.rowcount
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
if deleted == 0:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="调度任务不存在",
)
return {"message": "调度任务已删除"}
# ── PATCH /api/schedules/{id}/toggle — 启用/禁用 ─────────────
@router.patch("/{schedule_id}/toggle", response_model=ScheduleResponse)
async def toggle_schedule(
schedule_id: str,
user: CurrentUser = Depends(get_current_user),
) -> ScheduleResponse:
"""切换调度任务的启用/禁用状态。
禁用时 next_run_at 置 NULL启用时重新计算 next_run_at。
"""
conn = get_connection()
try:
# 先查询当前状态和调度配置
with conn.cursor() as cur:
cur.execute(
"SELECT enabled, schedule_config FROM scheduled_tasks WHERE id = %s AND site_id = %s",
(schedule_id, user.site_id),
)
row = cur.fetchone()
if row is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="调度任务不存在",
)
current_enabled = row[0]
new_enabled = not current_enabled
if new_enabled:
# 启用:重新计算 next_run_at
schedule_config_raw = row[1] if isinstance(row[1], dict) else json.loads(row[1])
from app.schemas.schedules import ScheduleConfigSchema
schedule_cfg = ScheduleConfigSchema(**schedule_config_raw)
now = datetime.now(timezone.utc)
next_run = calculate_next_run(schedule_cfg, now)
else:
# 禁用next_run_at 置 NULL
next_run = None
with conn.cursor() as cur:
cur.execute(
f"""
UPDATE scheduled_tasks
SET enabled = %s, next_run_at = %s, updated_at = NOW()
WHERE id = %s AND site_id = %s
RETURNING {_SELECT_COLS}
""",
(new_enabled, next_run, schedule_id, user.site_id),
)
updated_row = cur.fetchone()
conn.commit()
except HTTPException:
raise
except Exception:
conn.rollback()
raise
finally:
conn.close()
return _row_to_response(updated_row)

View File

@@ -0,0 +1,209 @@
# -*- coding: utf-8 -*-
"""任务注册表 & 配置 API
提供 4 个端点:
- GET /api/tasks/registry — 按业务域分组的任务列表
- GET /api/tasks/dwd-tables — 按业务域分组的 DWD 表定义
- GET /api/tasks/flows — 7 种 Flow + 3 种处理模式
- POST /api/tasks/validate — 验证 TaskConfig 并返回 CLI 命令预览
所有端点需要 JWT 认证。validate 端点从 JWT 注入 store_id。
"""
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from app.auth.dependencies import CurrentUser, get_current_user
from app.config import ETL_PROJECT_PATH
from app.schemas.tasks import (
FlowDefinition,
ProcessingModeDefinition,
TaskConfigSchema,
)
from app.services.cli_builder import cli_builder
from app.services.task_registry import (
DWD_TABLES,
FLOW_LAYER_MAP,
get_dwd_tables_grouped_by_domain,
get_tasks_grouped_by_domain,
)
router = APIRouter(prefix="/api/tasks", tags=["任务配置"])
# ── 响应模型 ──────────────────────────────────────────────────
class TaskItem(BaseModel):
code: str
name: str
description: str
domain: str
layer: str
requires_window: bool
is_ods: bool
is_dimension: bool
default_enabled: bool
is_common: bool
class DwdTableItem(BaseModel):
table_name: str
display_name: str
domain: str
ods_source: str
is_dimension: bool
class TaskRegistryResponse(BaseModel):
"""按业务域分组的任务列表"""
groups: dict[str, list[TaskItem]]
class DwdTablesResponse(BaseModel):
"""按业务域分组的 DWD 表定义"""
groups: dict[str, list[DwdTableItem]]
class FlowsResponse(BaseModel):
"""Flow 定义 + 处理模式定义"""
flows: list[FlowDefinition]
processing_modes: list[ProcessingModeDefinition]
class ValidateRequest(BaseModel):
"""验证请求体 — 复用 TaskConfigSchema但 store_id 由后端注入"""
config: TaskConfigSchema
class ValidateResponse(BaseModel):
"""验证结果 + CLI 命令预览"""
valid: bool
command: str
command_args: list[str]
errors: list[str]
# ── Flow 定义(静态) ────────────────────────────────────────
FLOW_DEFINITIONS: list[FlowDefinition] = [
FlowDefinition(id="api_ods", name="API → ODS", layers=["ODS"]),
FlowDefinition(id="api_ods_dwd", name="API → ODS → DWD", layers=["ODS", "DWD"]),
FlowDefinition(id="api_full", name="API → ODS → DWD → DWS汇总 → DWS指数", layers=["ODS", "DWD", "DWS", "INDEX"]),
FlowDefinition(id="ods_dwd", name="ODS → DWD", layers=["DWD"]),
FlowDefinition(id="dwd_dws", name="DWD → DWS汇总", layers=["DWS"]),
FlowDefinition(id="dwd_dws_index", name="DWD → DWS汇总 → DWS指数", layers=["DWS", "INDEX"]),
FlowDefinition(id="dwd_index", name="DWD → DWS指数", layers=["INDEX"]),
]
PROCESSING_MODE_DEFINITIONS: list[ProcessingModeDefinition] = [
ProcessingModeDefinition(id="increment_only", name="仅增量处理", description="只处理新增和变更的数据"),
ProcessingModeDefinition(id="verify_only", name="仅校验修复", description="校验现有数据并修复不一致(可选'校验前从 API 获取'"),
ProcessingModeDefinition(id="increment_verify", name="增量 + 校验修复", description="先增量处理,再校验并修复"),
]
# ── 端点 ──────────────────────────────────────────────────────
@router.get("/registry", response_model=TaskRegistryResponse)
async def get_task_registry(
user: CurrentUser = Depends(get_current_user),
) -> TaskRegistryResponse:
"""返回按业务域分组的任务列表"""
grouped = get_tasks_grouped_by_domain()
return TaskRegistryResponse(
groups={
domain: [
TaskItem(
code=t.code,
name=t.name,
description=t.description,
domain=t.domain,
layer=t.layer,
requires_window=t.requires_window,
is_ods=t.is_ods,
is_dimension=t.is_dimension,
default_enabled=t.default_enabled,
is_common=t.is_common,
)
for t in tasks
]
for domain, tasks in grouped.items()
}
)
@router.get("/dwd-tables", response_model=DwdTablesResponse)
async def get_dwd_tables(
user: CurrentUser = Depends(get_current_user),
) -> DwdTablesResponse:
"""返回按业务域分组的 DWD 表定义"""
grouped = get_dwd_tables_grouped_by_domain()
return DwdTablesResponse(
groups={
domain: [
DwdTableItem(
table_name=t.table_name,
display_name=t.display_name,
domain=t.domain,
ods_source=t.ods_source,
is_dimension=t.is_dimension,
)
for t in tables
]
for domain, tables in grouped.items()
}
)
@router.get("/flows", response_model=FlowsResponse)
async def get_flows(
user: CurrentUser = Depends(get_current_user),
) -> FlowsResponse:
"""返回 7 种 Flow 定义和 3 种处理模式定义"""
return FlowsResponse(
flows=FLOW_DEFINITIONS,
processing_modes=PROCESSING_MODE_DEFINITIONS,
)
@router.post("/validate", response_model=ValidateResponse)
async def validate_task_config(
body: ValidateRequest,
user: CurrentUser = Depends(get_current_user),
) -> ValidateResponse:
"""验证 TaskConfig 并返回生成的 CLI 命令预览
从 JWT 注入 store_id前端无需传递。
"""
config = body.config.model_copy(update={"store_id": user.site_id})
errors: list[str] = []
# 验证 Flow ID
if config.pipeline not in FLOW_LAYER_MAP:
errors.append(f"无效的执行流程: {config.pipeline}")
# 验证任务列表非空
if not config.tasks:
errors.append("任务列表不能为空")
if errors:
return ValidateResponse(
valid=False,
command="",
command_args=[],
errors=errors,
)
cmd_args = cli_builder.build_command(config, ETL_PROJECT_PATH)
cmd_str = cli_builder.build_command_string(config, ETL_PROJECT_PATH)
return ValidateResponse(
valid=True,
command=cmd_str,
command_args=cmd_args,
errors=[],
)

View File

@@ -0,0 +1,104 @@
# AI_CHANGELOG
# - 2026-02-19 | Prompt: 配置微信消息推送 | 新增微信消息推送回调接口,支持 GET 验签 + POST 消息接收
"""
微信消息推送回调接口
处理两类请求:
1. GET — 微信服务器验证(配置时触发一次)
2. POST — 接收微信推送的消息/事件
安全模式下需要解密消息体,当前先用明文模式跑通,后续切安全模式。
"""
import hashlib
import logging
from fastapi import APIRouter, Query, Request, Response
from app.config import get
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/wx", tags=["微信回调"])
# Token 从环境变量读取,与微信后台填写的一致
# 放在 apps/backend/.env.local 中WX_CALLBACK_TOKEN=你自定义的token
WX_CALLBACK_TOKEN: str = get("WX_CALLBACK_TOKEN", "")
def _check_signature(signature: str, timestamp: str, nonce: str) -> bool:
"""
验证请求是否来自微信服务器。
将 Token、timestamp、nonce 字典序排序后拼接,做 SHA1
与 signature 比对。
"""
if not WX_CALLBACK_TOKEN:
logger.error("WX_CALLBACK_TOKEN 未配置")
return False
items = sorted([WX_CALLBACK_TOKEN, timestamp, nonce])
hash_str = hashlib.sha1("".join(items).encode("utf-8")).hexdigest()
return hash_str == signature
@router.get("/callback")
async def verify(
signature: str = Query(...),
timestamp: str = Query(...),
nonce: str = Query(...),
echostr: str = Query(...),
):
"""
微信服务器验证接口。
配置消息推送时微信会发 GET 请求,验签通过后原样返回 echostr。
"""
if _check_signature(signature, timestamp, nonce):
logger.info("微信回调验证通过")
# 必须原样返回 echostr纯文本不能包裹 JSON
return Response(content=echostr, media_type="text/plain")
else:
logger.warning("微信回调验签失败: signature=%s", signature)
return Response(content="signature mismatch", status_code=403)
@router.post("/callback")
async def receive_message(
request: Request,
signature: str = Query(""),
timestamp: str = Query(""),
nonce: str = Query(""),
):
"""
接收微信推送的消息/事件。
当前为明文模式,直接解析 JSON 包体。
后续切安全模式时需增加 AES 解密逻辑。
"""
# 验签POST 也带 signature 参数)
if not _check_signature(signature, timestamp, nonce):
logger.warning("消息推送验签失败")
return Response(content="signature mismatch", status_code=403)
# 解析消息体
body = await request.body()
content_type = request.headers.get("content-type", "")
if "json" in content_type:
import json
try:
data = json.loads(body)
except json.JSONDecodeError:
data = {"raw": body.decode("utf-8", errors="replace")}
else:
# XML 格式暂不解析,记录原文
data = {"raw_xml": body.decode("utf-8", errors="replace")}
logger.info("收到微信推送: MsgType=%s, Event=%s",
data.get("MsgType", "?"), data.get("Event", "?"))
# TODO: 根据 MsgType/Event 分发处理(客服消息、订阅事件等)
# 当前统一返回 success
return Response(content="success", media_type="text/plain")

View File

@@ -0,0 +1,37 @@
# AI_CHANGELOG
# - 2026-02-19 | Prompt: 小程序 MVP 全链路验证 | 新增 /api/xcx-test 接口,从 test."xcx-test" 表读取 ti 列第一行
"""
小程序 MVP 验证接口
从 test_zqyy_app 库的 test."xcx-test" 表读取数据,
用于验证小程序 → 后端 → 数据库全链路连通性。
"""
from fastapi import APIRouter, HTTPException
from app.database import get_connection
router = APIRouter(prefix="/api/xcx-test", tags=["小程序MVP"])
@router.get("")
async def get_xcx_test():
"""
读取 test."xcx-test" 表 ti 列第一行。
用于小程序 MVP 全链路验证:小程序 → API → DB → 返回数据。
"""
conn = get_connection()
try:
with conn.cursor() as cur:
# CHANGE 2026-02-19 | 读取 test schema 下的 xcx-test 表
# 表名含连字符,必须用双引号包裹
cur.execute('SELECT ti FROM test."xcx-test" LIMIT 1')
row = cur.fetchone()
finally:
conn.close()
if row is None:
raise HTTPException(status_code=404, detail="无数据")
return {"ti": row[0]}