在准备环境前提交次全部更改。
This commit is contained in:
97
apps/backend/app/routers/auth.py
Normal file
97
apps/backend/app/routers/auth.py
Normal file
@@ -0,0 +1,97 @@
|
||||
"""
|
||||
认证路由:登录与令牌刷新。
|
||||
|
||||
- POST /api/auth/login — 验证用户名密码,返回 JWT 令牌对
|
||||
- POST /api/auth/refresh — 用刷新令牌换取新的访问令牌
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
from jose import JWTError
|
||||
|
||||
from app.auth.jwt import (
|
||||
create_access_token,
|
||||
create_token_pair,
|
||||
decode_refresh_token,
|
||||
verify_password,
|
||||
)
|
||||
from app.database import get_connection
|
||||
from app.schemas.auth import LoginRequest, RefreshRequest, TokenResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/auth", tags=["认证"])
|
||||
|
||||
|
||||
@router.post("/login", response_model=TokenResponse)
|
||||
async def login(body: LoginRequest):
|
||||
"""
|
||||
用户登录。
|
||||
|
||||
查询 admin_users 表验证用户名密码,成功后返回 JWT 令牌对。
|
||||
- 用户不存在或密码错误:401
|
||||
- 账号已禁用(is_active=false):401
|
||||
"""
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, password_hash, site_id, is_active "
|
||||
"FROM admin_users WHERE username = %s",
|
||||
(body.username,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if row is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="用户名或密码错误",
|
||||
)
|
||||
|
||||
user_id, password_hash, site_id, is_active = row
|
||||
|
||||
if not is_active:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="账号已被禁用",
|
||||
)
|
||||
|
||||
if not verify_password(body.password, password_hash):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="用户名或密码错误",
|
||||
)
|
||||
|
||||
tokens = create_token_pair(user_id, site_id)
|
||||
return TokenResponse(**tokens)
|
||||
|
||||
|
||||
@router.post("/refresh", response_model=TokenResponse)
|
||||
async def refresh(body: RefreshRequest):
|
||||
"""
|
||||
刷新访问令牌。
|
||||
|
||||
验证 refresh_token 有效性,成功后仅返回新的 access_token
|
||||
(refresh_token 保持不变,由客户端继续持有)。
|
||||
"""
|
||||
try:
|
||||
payload = decode_refresh_token(body.refresh_token)
|
||||
except JWTError:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="无效的刷新令牌",
|
||||
)
|
||||
|
||||
user_id = int(payload["sub"])
|
||||
site_id = payload["site_id"]
|
||||
|
||||
# 生成新的 access_token,refresh_token 原样返回
|
||||
new_access = create_access_token(user_id, site_id)
|
||||
return TokenResponse(
|
||||
access_token=new_access,
|
||||
refresh_token=body.refresh_token,
|
||||
token_type="bearer",
|
||||
)
|
||||
228
apps/backend/app/routers/db_viewer.py
Normal file
228
apps/backend/app/routers/db_viewer.py
Normal file
@@ -0,0 +1,228 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""数据库查看器 API
|
||||
|
||||
提供 4 个端点:
|
||||
- GET /api/db/schemas — 返回 Schema 列表
|
||||
- GET /api/db/schemas/{name}/tables — 返回表列表和行数
|
||||
- GET /api/db/tables/{schema}/{table}/columns — 返回列定义
|
||||
- POST /api/db/query — 只读 SQL 执行
|
||||
|
||||
所有端点需要 JWT 认证。
|
||||
使用 get_etl_readonly_connection(site_id) 确保 RLS 隔离。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from psycopg2 import errors as pg_errors, OperationalError
|
||||
|
||||
from app.auth.dependencies import CurrentUser, get_current_user
|
||||
from app.database import get_etl_readonly_connection
|
||||
from app.schemas.db_viewer import (
|
||||
ColumnInfo,
|
||||
QueryRequest,
|
||||
QueryResponse,
|
||||
SchemaInfo,
|
||||
TableInfo,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/db", tags=["数据库查看器"])
|
||||
|
||||
# 写操作关键词(不区分大小写)
|
||||
_WRITE_KEYWORDS = re.compile(
|
||||
r"\b(INSERT|UPDATE|DELETE|DROP|TRUNCATE)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# 查询结果行数上限
|
||||
_MAX_ROWS = 1000
|
||||
|
||||
# 查询超时(秒)
|
||||
_QUERY_TIMEOUT_SEC = 30
|
||||
|
||||
|
||||
# ── GET /api/db/schemas ──────────────────────────────────────
|
||||
|
||||
@router.get("/schemas", response_model=list[SchemaInfo])
|
||||
async def list_schemas(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> list[SchemaInfo]:
|
||||
"""返回 ETL 数据库中的 Schema 列表。"""
|
||||
conn = get_etl_readonly_connection(user.site_id)
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
|
||||
ORDER BY schema_name
|
||||
"""
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return [SchemaInfo(name=row[0]) for row in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ── GET /api/db/schemas/{name}/tables ────────────────────────
|
||||
|
||||
@router.get("/schemas/{name}/tables", response_model=list[TableInfo])
|
||||
async def list_tables(
|
||||
name: str,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> list[TableInfo]:
|
||||
"""返回指定 Schema 下所有表的名称和行数统计。"""
|
||||
conn = get_etl_readonly_connection(user.site_id)
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
t.table_name,
|
||||
s.n_live_tup
|
||||
FROM information_schema.tables t
|
||||
LEFT JOIN pg_stat_user_tables s
|
||||
ON s.schemaname = t.table_schema
|
||||
AND s.relname = t.table_name
|
||||
WHERE t.table_schema = %s
|
||||
AND t.table_type = 'BASE TABLE'
|
||||
ORDER BY t.table_name
|
||||
""",
|
||||
(name,),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return [
|
||||
TableInfo(name=row[0], row_count=row[1])
|
||||
for row in rows
|
||||
]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ── GET /api/db/tables/{schema}/{table}/columns ──────────────
|
||||
|
||||
@router.get(
|
||||
"/tables/{schema}/{table}/columns",
|
||||
response_model=list[ColumnInfo],
|
||||
)
|
||||
async def list_columns(
|
||||
schema: str,
|
||||
table: str,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> list[ColumnInfo]:
|
||||
"""返回指定表的列定义。"""
|
||||
conn = get_etl_readonly_connection(user.site_id)
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
column_name,
|
||||
data_type,
|
||||
is_nullable,
|
||||
column_default
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = %s AND table_name = %s
|
||||
ORDER BY ordinal_position
|
||||
""",
|
||||
(schema, table),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return [
|
||||
ColumnInfo(
|
||||
name=row[0],
|
||||
data_type=row[1],
|
||||
is_nullable=row[2] == "YES",
|
||||
column_default=row[3],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ── POST /api/db/query ───────────────────────────────────────
|
||||
|
||||
@router.post("/query", response_model=QueryResponse)
|
||||
async def execute_query(
|
||||
body: QueryRequest,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> QueryResponse:
|
||||
"""只读 SQL 执行。
|
||||
|
||||
安全措施:
|
||||
1. 拦截写操作关键词(INSERT / UPDATE / DELETE / DROP / TRUNCATE)
|
||||
2. 限制返回行数上限 1000 行
|
||||
3. 设置查询超时 30 秒
|
||||
"""
|
||||
sql = body.sql.strip()
|
||||
if not sql:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="SQL 语句不能为空",
|
||||
)
|
||||
|
||||
# 拦截写操作
|
||||
if _WRITE_KEYWORDS.search(sql):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="只允许只读查询,禁止 INSERT / UPDATE / DELETE / DROP / TRUNCATE 操作",
|
||||
)
|
||||
|
||||
conn = get_etl_readonly_connection(user.site_id)
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
# 设置查询超时
|
||||
cur.execute(
|
||||
"SET LOCAL statement_timeout = %s",
|
||||
(f"{_QUERY_TIMEOUT_SEC}s",),
|
||||
)
|
||||
|
||||
try:
|
||||
cur.execute(sql)
|
||||
except pg_errors.QueryCanceled:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_408_REQUEST_TIMEOUT,
|
||||
detail=f"查询超时(超过 {_QUERY_TIMEOUT_SEC} 秒)",
|
||||
)
|
||||
except Exception as exc:
|
||||
# SQL 语法错误或其他执行错误
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"SQL 执行错误: {exc}",
|
||||
)
|
||||
|
||||
# 提取列名
|
||||
columns = (
|
||||
[desc[0] for desc in cur.description]
|
||||
if cur.description
|
||||
else []
|
||||
)
|
||||
|
||||
# 限制返回行数
|
||||
rows = cur.fetchmany(_MAX_ROWS)
|
||||
# 将元组转为列表,便于 JSON 序列化
|
||||
rows_list = [list(row) for row in rows]
|
||||
|
||||
return QueryResponse(
|
||||
columns=columns,
|
||||
rows=rows_list,
|
||||
row_count=len(rows_list),
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except OperationalError as exc:
|
||||
# 连接级错误
|
||||
logger.error("数据库查看器连接错误: %s", exc)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="数据库连接错误",
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
240
apps/backend/app/routers/env_config.py
Normal file
240
apps/backend/app/routers/env_config.py
Normal file
@@ -0,0 +1,240 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""环境配置 API
|
||||
|
||||
提供 3 个端点:
|
||||
- GET /api/env-config — 读取 .env,敏感值掩码
|
||||
- PUT /api/env-config — 验证并写入 .env
|
||||
- GET /api/env-config/export — 导出去敏感值的配置文件
|
||||
|
||||
所有端点需要 JWT 认证。
|
||||
敏感键判定:键名中包含 PASSWORD、TOKEN、SECRET、DSN(不区分大小写)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from fastapi.responses import PlainTextResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.auth.dependencies import CurrentUser, get_current_user
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/env-config", tags=["环境配置"])
|
||||
|
||||
# .env 文件路径:项目根目录
|
||||
_ENV_PATH = Path(__file__).resolve().parents[3] / ".env"
|
||||
|
||||
# 敏感键关键词(不区分大小写)
|
||||
_SENSITIVE_KEYWORDS = ("PASSWORD", "TOKEN", "SECRET", "DSN")
|
||||
|
||||
_MASK = "****"
|
||||
|
||||
|
||||
# ── Pydantic 模型 ────────────────────────────────────────────
|
||||
|
||||
class EnvEntry(BaseModel):
|
||||
"""单条环境变量键值对。"""
|
||||
key: str
|
||||
value: str
|
||||
|
||||
|
||||
class EnvConfigResponse(BaseModel):
|
||||
"""GET 响应:键值对列表。"""
|
||||
entries: list[EnvEntry]
|
||||
|
||||
|
||||
class EnvConfigUpdateRequest(BaseModel):
|
||||
"""PUT 请求体:键值对列表。"""
|
||||
entries: list[EnvEntry]
|
||||
|
||||
|
||||
# ── 工具函数 ─────────────────────────────────────────────────
|
||||
|
||||
def _is_sensitive(key: str) -> bool:
|
||||
"""判断键名是否为敏感键。"""
|
||||
upper = key.upper()
|
||||
return any(kw in upper for kw in _SENSITIVE_KEYWORDS)
|
||||
|
||||
|
||||
def _parse_env(content: str) -> list[dict]:
|
||||
"""解析 .env 文件内容,返回行级结构。
|
||||
|
||||
每行分为三种类型:
|
||||
- comment: 注释行或空行(原样保留)
|
||||
- entry: 键值对
|
||||
"""
|
||||
lines: list[dict] = []
|
||||
for raw_line in content.splitlines():
|
||||
stripped = raw_line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
lines.append({"type": "comment", "raw": raw_line})
|
||||
else:
|
||||
# 支持 KEY=VALUE 和 KEY="VALUE" 格式
|
||||
match = re.match(r'^([A-Za-z_][A-Za-z0-9_]*)=(.*)', raw_line)
|
||||
if match:
|
||||
key = match.group(1)
|
||||
value = match.group(2).strip()
|
||||
# 去除引号包裹
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in ('"', "'"):
|
||||
value = value[1:-1]
|
||||
lines.append({"type": "entry", "key": key, "value": value, "raw": raw_line})
|
||||
else:
|
||||
# 无法解析的行当作注释保留
|
||||
lines.append({"type": "comment", "raw": raw_line})
|
||||
return lines
|
||||
|
||||
|
||||
def _read_env_file(path: Path) -> str:
|
||||
"""读取 .env 文件内容。"""
|
||||
if not path.exists():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=".env 文件不存在",
|
||||
)
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _write_env_file(path: Path, content: str) -> None:
|
||||
"""写入 .env 文件。"""
|
||||
try:
|
||||
path.write_text(content, encoding="utf-8")
|
||||
except OSError as exc:
|
||||
logger.error("写入 .env 文件失败: %s", exc)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="写入 .env 文件失败",
|
||||
)
|
||||
|
||||
|
||||
def _validate_entries(entries: list[EnvEntry]) -> None:
|
||||
"""验证键值对格式。"""
|
||||
for idx, entry in enumerate(entries):
|
||||
if not entry.key:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"第 {idx + 1} 行:键名不能为空",
|
||||
)
|
||||
if not re.match(r'^[A-Za-z_][A-Za-z0-9_]*$', entry.key):
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"第 {idx + 1} 行:键名 '{entry.key}' 格式无效(仅允许字母、数字、下划线,且不能以数字开头)",
|
||||
)
|
||||
|
||||
|
||||
# ── GET /api/env-config — 读取 ───────────────────────────────
|
||||
|
||||
@router.get("", response_model=EnvConfigResponse)
|
||||
async def get_env_config(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> EnvConfigResponse:
|
||||
"""读取 .env 文件,敏感值以掩码展示。"""
|
||||
content = _read_env_file(_ENV_PATH)
|
||||
parsed = _parse_env(content)
|
||||
|
||||
entries = []
|
||||
for line in parsed:
|
||||
if line["type"] == "entry":
|
||||
value = _MASK if _is_sensitive(line["key"]) else line["value"]
|
||||
entries.append(EnvEntry(key=line["key"], value=value))
|
||||
|
||||
return EnvConfigResponse(entries=entries)
|
||||
|
||||
|
||||
# ── PUT /api/env-config — 写入 ───────────────────────────────
|
||||
|
||||
@router.put("", response_model=EnvConfigResponse)
|
||||
async def update_env_config(
|
||||
body: EnvConfigUpdateRequest,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> EnvConfigResponse:
|
||||
"""验证并写入 .env 文件。
|
||||
|
||||
保留原文件中的注释行和空行。对于已有键,更新值;
|
||||
对于新键,追加到文件末尾。掩码值(****)的键跳过更新,保留原值。
|
||||
"""
|
||||
_validate_entries(body.entries)
|
||||
|
||||
# 读取原文件(如果存在)
|
||||
if _ENV_PATH.exists():
|
||||
original_content = _ENV_PATH.read_text(encoding="utf-8")
|
||||
parsed = _parse_env(original_content)
|
||||
else:
|
||||
parsed = []
|
||||
|
||||
# 构建新值映射(跳过掩码值)
|
||||
new_values: dict[str, str] = {}
|
||||
for entry in body.entries:
|
||||
if entry.value != _MASK:
|
||||
new_values[entry.key] = entry.value
|
||||
|
||||
# 更新已有行
|
||||
seen_keys: set[str] = set()
|
||||
output_lines: list[str] = []
|
||||
for line in parsed:
|
||||
if line["type"] == "comment":
|
||||
output_lines.append(line["raw"])
|
||||
elif line["type"] == "entry":
|
||||
key = line["key"]
|
||||
seen_keys.add(key)
|
||||
if key in new_values:
|
||||
output_lines.append(f"{key}={new_values[key]}")
|
||||
else:
|
||||
# 保留原值(包括掩码跳过的敏感键)
|
||||
output_lines.append(line["raw"])
|
||||
|
||||
# 追加新键
|
||||
for entry in body.entries:
|
||||
if entry.key not in seen_keys and entry.value != _MASK:
|
||||
output_lines.append(f"{entry.key}={entry.value}")
|
||||
|
||||
new_content = "\n".join(output_lines)
|
||||
if output_lines:
|
||||
new_content += "\n"
|
||||
|
||||
_write_env_file(_ENV_PATH, new_content)
|
||||
|
||||
# 返回更新后的配置(敏感值掩码)
|
||||
result_parsed = _parse_env(new_content)
|
||||
entries = []
|
||||
for line in result_parsed:
|
||||
if line["type"] == "entry":
|
||||
value = _MASK if _is_sensitive(line["key"]) else line["value"]
|
||||
entries.append(EnvEntry(key=line["key"], value=value))
|
||||
|
||||
return EnvConfigResponse(entries=entries)
|
||||
|
||||
|
||||
# ── GET /api/env-config/export — 导出 ────────────────────────
|
||||
|
||||
@router.get("/export")
|
||||
async def export_env_config(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> PlainTextResponse:
|
||||
"""导出去除敏感值的配置文件(作为文件下载)。"""
|
||||
content = _read_env_file(_ENV_PATH)
|
||||
parsed = _parse_env(content)
|
||||
|
||||
output_lines: list[str] = []
|
||||
for line in parsed:
|
||||
if line["type"] == "comment":
|
||||
output_lines.append(line["raw"])
|
||||
elif line["type"] == "entry":
|
||||
if _is_sensitive(line["key"]):
|
||||
output_lines.append(f"{line['key']}={_MASK}")
|
||||
else:
|
||||
output_lines.append(line["raw"])
|
||||
|
||||
export_content = "\n".join(output_lines)
|
||||
if output_lines:
|
||||
export_content += "\n"
|
||||
|
||||
return PlainTextResponse(
|
||||
content=export_content,
|
||||
media_type="text/plain",
|
||||
headers={"Content-Disposition": "attachment; filename=env-config.txt"},
|
||||
)
|
||||
134
apps/backend/app/routers/etl_status.py
Normal file
134
apps/backend/app/routers/etl_status.py
Normal file
@@ -0,0 +1,134 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""ETL 状态监控 API
|
||||
|
||||
提供 2 个端点:
|
||||
- GET /api/etl-status/cursors — 返回各任务的数据游标(最后抓取时间、记录数)
|
||||
- GET /api/etl-status/recent-runs — 返回最近 50 条任务执行记录
|
||||
|
||||
所有端点需要 JWT 认证。
|
||||
游标端点查询 ETL 数据库(meta.etl_cursor),
|
||||
执行记录端点查询 zqyy_app 数据库(task_execution_log)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from psycopg2 import OperationalError
|
||||
|
||||
from app.auth.dependencies import CurrentUser, get_current_user
|
||||
from app.database import get_connection, get_etl_readonly_connection
|
||||
from app.schemas.etl_status import CursorInfo, RecentRun
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/etl-status", tags=["ETL 状态"])
|
||||
|
||||
# 最近执行记录条数上限
|
||||
_RECENT_RUNS_LIMIT = 50
|
||||
|
||||
|
||||
# ── GET /api/etl-status/cursors ──────────────────────────────
|
||||
|
||||
@router.get("/cursors", response_model=list[CursorInfo])
|
||||
async def list_cursors(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> list[CursorInfo]:
|
||||
"""返回各 ODS 表的最新数据游标。
|
||||
|
||||
查询 ETL 数据库中的 meta.etl_cursor 表。
|
||||
如果该表不存在,返回空列表而非报错。
|
||||
"""
|
||||
conn = get_etl_readonly_connection(user.site_id)
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
# CHANGE 2026-02-15 | 对齐新库 etl_feiqiu 六层架构:etl_admin → meta
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'meta'
|
||||
AND table_name = 'etl_cursor'
|
||||
)
|
||||
"""
|
||||
)
|
||||
exists = cur.fetchone()[0]
|
||||
if not exists:
|
||||
return []
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT task_code, last_fetch_time, record_count
|
||||
FROM meta.etl_cursor
|
||||
ORDER BY task_code
|
||||
"""
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
|
||||
return [
|
||||
CursorInfo(
|
||||
task_code=row[0],
|
||||
last_fetch_time=str(row[1]) if row[1] is not None else None,
|
||||
record_count=row[2],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
except OperationalError as exc:
|
||||
logger.error("ETL 游标查询连接错误: %s", exc)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="ETL 数据库连接错误",
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ── GET /api/etl-status/recent-runs ──────────────────────────
|
||||
|
||||
@router.get("/recent-runs", response_model=list[RecentRun])
|
||||
async def list_recent_runs(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> list[RecentRun]:
|
||||
"""返回最近 50 条任务执行记录。
|
||||
|
||||
查询 zqyy_app 数据库中的 task_execution_log 表,
|
||||
按 site_id 过滤,按 started_at DESC 排序。
|
||||
"""
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, task_codes, status, started_at,
|
||||
finished_at, duration_ms, exit_code
|
||||
FROM task_execution_log
|
||||
WHERE site_id = %s
|
||||
ORDER BY started_at DESC
|
||||
LIMIT %s
|
||||
""",
|
||||
(user.site_id, _RECENT_RUNS_LIMIT),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
|
||||
return [
|
||||
RecentRun(
|
||||
id=str(row[0]),
|
||||
task_codes=list(row[1]) if row[1] else [],
|
||||
status=row[2],
|
||||
started_at=str(row[3]),
|
||||
finished_at=str(row[4]) if row[4] is not None else None,
|
||||
duration_ms=row[5],
|
||||
exit_code=row[6],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
except OperationalError as exc:
|
||||
logger.error("执行记录查询连接错误: %s", exc)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="数据库连接错误",
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
281
apps/backend/app/routers/execution.py
Normal file
281
apps/backend/app/routers/execution.py
Normal file
@@ -0,0 +1,281 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""执行与队列 API
|
||||
|
||||
提供 8 个端点:
|
||||
- POST /api/execution/run — 直接执行任务
|
||||
- GET /api/execution/queue — 获取当前队列(按 site_id 过滤)
|
||||
- POST /api/execution/queue — 添加到队列
|
||||
- PUT /api/execution/queue/reorder — 重排队列
|
||||
- DELETE /api/execution/queue/{id} — 删除队列任务
|
||||
- POST /api/execution/{id}/cancel — 取消执行中的任务
|
||||
- GET /api/execution/history — 执行历史(按 site_id 过滤)
|
||||
- GET /api/execution/{id}/logs — 获取历史日志
|
||||
|
||||
所有端点需要 JWT 认证,site_id 从 JWT 提取。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
|
||||
from app.auth.dependencies import CurrentUser, get_current_user
|
||||
from app.database import get_connection
|
||||
from app.schemas.execution import (
|
||||
ExecutionHistoryItem,
|
||||
ExecutionLogsResponse,
|
||||
ExecutionRunResponse,
|
||||
QueueTaskResponse,
|
||||
ReorderRequest,
|
||||
)
|
||||
from app.schemas.tasks import TaskConfigSchema
|
||||
from app.services.task_executor import task_executor
|
||||
from app.services.task_queue import task_queue
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/execution", tags=["任务执行"])
|
||||
|
||||
|
||||
# ── POST /api/execution/run — 直接执行任务 ────────────────────
|
||||
|
||||
@router.post("/run", response_model=ExecutionRunResponse)
|
||||
async def run_task(
|
||||
config: TaskConfigSchema,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> ExecutionRunResponse:
|
||||
"""直接执行任务(不经过队列)。
|
||||
|
||||
从 JWT 注入 store_id,创建 execution_id 后异步启动子进程。
|
||||
"""
|
||||
config = config.model_copy(update={"store_id": user.site_id})
|
||||
execution_id = str(uuid.uuid4())
|
||||
|
||||
# 异步启动执行,不阻塞响应
|
||||
asyncio.create_task(
|
||||
task_executor.execute(
|
||||
config=config,
|
||||
execution_id=execution_id,
|
||||
site_id=user.site_id,
|
||||
)
|
||||
)
|
||||
|
||||
return ExecutionRunResponse(
|
||||
execution_id=execution_id,
|
||||
message="任务已提交执行",
|
||||
)
|
||||
|
||||
|
||||
# ── GET /api/execution/queue — 获取当前队列 ───────────────────
|
||||
|
||||
@router.get("/queue", response_model=list[QueueTaskResponse])
|
||||
async def get_queue(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> list[QueueTaskResponse]:
|
||||
"""获取当前门店的待执行队列。"""
|
||||
tasks = task_queue.list_pending(user.site_id)
|
||||
return [
|
||||
QueueTaskResponse(
|
||||
id=t.id,
|
||||
site_id=t.site_id,
|
||||
config=t.config,
|
||||
status=t.status,
|
||||
position=t.position,
|
||||
created_at=t.created_at,
|
||||
started_at=t.started_at,
|
||||
finished_at=t.finished_at,
|
||||
exit_code=t.exit_code,
|
||||
error_message=t.error_message,
|
||||
)
|
||||
for t in tasks
|
||||
]
|
||||
|
||||
|
||||
# ── POST /api/execution/queue — 添加到队列 ───────────────────
|
||||
|
||||
@router.post("/queue", response_model=QueueTaskResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def enqueue_task(
|
||||
config: TaskConfigSchema,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> QueueTaskResponse:
|
||||
"""将任务配置添加到执行队列。"""
|
||||
config = config.model_copy(update={"store_id": user.site_id})
|
||||
task_id = task_queue.enqueue(config, user.site_id)
|
||||
|
||||
# 查询刚创建的任务返回完整信息
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, site_id, config, status, position,
|
||||
created_at, started_at, finished_at,
|
||||
exit_code, error_message
|
||||
FROM task_queue WHERE id = %s
|
||||
""",
|
||||
(task_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if row is None:
|
||||
raise HTTPException(status_code=500, detail="入队后查询失败")
|
||||
|
||||
config_data = row[2] if isinstance(row[2], dict) else json.loads(row[2])
|
||||
return QueueTaskResponse(
|
||||
id=str(row[0]),
|
||||
site_id=row[1],
|
||||
config=config_data,
|
||||
status=row[3],
|
||||
position=row[4],
|
||||
created_at=row[5],
|
||||
started_at=row[6],
|
||||
finished_at=row[7],
|
||||
exit_code=row[8],
|
||||
error_message=row[9],
|
||||
)
|
||||
|
||||
|
||||
# ── PUT /api/execution/queue/reorder — 重排队列 ──────────────
|
||||
|
||||
@router.put("/queue/reorder")
|
||||
async def reorder_queue(
|
||||
body: ReorderRequest,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> dict:
|
||||
"""调整队列中任务的执行顺序。"""
|
||||
task_queue.reorder(body.task_id, body.new_position, user.site_id)
|
||||
return {"message": "队列已重排"}
|
||||
|
||||
|
||||
# ── DELETE /api/execution/queue/{id} — 删除队列任务 ──────────
|
||||
|
||||
@router.delete("/queue/{task_id}")
|
||||
async def delete_queue_task(
|
||||
task_id: str,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> dict:
|
||||
"""从队列中删除待执行任务。仅允许删除 pending 状态的任务。"""
|
||||
deleted = task_queue.delete(task_id, user.site_id)
|
||||
if not deleted:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="任务不存在或非待执行状态,无法删除",
|
||||
)
|
||||
return {"message": "任务已从队列中删除"}
|
||||
|
||||
|
||||
# ── POST /api/execution/{id}/cancel — 取消执行 ──────────────
|
||||
|
||||
@router.post("/{execution_id}/cancel")
|
||||
async def cancel_execution(
|
||||
execution_id: str,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> dict:
|
||||
"""取消正在执行的任务。"""
|
||||
cancelled = await task_executor.cancel(execution_id)
|
||||
if not cancelled:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="执行任务不存在或已完成",
|
||||
)
|
||||
return {"message": "已发送取消信号"}
|
||||
|
||||
|
||||
# ── GET /api/execution/history — 执行历史 ────────────────────
|
||||
|
||||
@router.get("/history", response_model=list[ExecutionHistoryItem])
|
||||
async def get_execution_history(
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> list[ExecutionHistoryItem]:
|
||||
"""获取执行历史记录,按 started_at 降序排列。"""
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, site_id, task_codes, status, started_at,
|
||||
finished_at, exit_code, duration_ms, command, summary
|
||||
FROM task_execution_log
|
||||
WHERE site_id = %s
|
||||
ORDER BY started_at DESC
|
||||
LIMIT %s
|
||||
""",
|
||||
(user.site_id, limit),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return [
|
||||
ExecutionHistoryItem(
|
||||
id=str(row[0]),
|
||||
site_id=row[1],
|
||||
task_codes=row[2] or [],
|
||||
status=row[3],
|
||||
started_at=row[4],
|
||||
finished_at=row[5],
|
||||
exit_code=row[6],
|
||||
duration_ms=row[7],
|
||||
command=row[8],
|
||||
summary=row[9],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
# ── GET /api/execution/{id}/logs — 获取历史日志 ──────────────
|
||||
|
||||
@router.get("/{execution_id}/logs", response_model=ExecutionLogsResponse)
|
||||
async def get_execution_logs(
|
||||
execution_id: str,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> ExecutionLogsResponse:
|
||||
"""获取指定执行的完整日志。
|
||||
|
||||
优先从内存缓冲区读取(执行中),否则从数据库读取(已完成)。
|
||||
"""
|
||||
# 先尝试内存缓冲区(执行中的任务)
|
||||
if task_executor.is_running(execution_id):
|
||||
lines = task_executor.get_logs(execution_id)
|
||||
return ExecutionLogsResponse(
|
||||
execution_id=execution_id,
|
||||
output_log="\n".join(lines) if lines else None,
|
||||
)
|
||||
|
||||
# 从数据库读取已完成任务的日志
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT output_log, error_log
|
||||
FROM task_execution_log
|
||||
WHERE id = %s AND site_id = %s
|
||||
""",
|
||||
(execution_id, user.site_id),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if row is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="执行记录不存在",
|
||||
)
|
||||
|
||||
return ExecutionLogsResponse(
|
||||
execution_id=execution_id,
|
||||
output_log=row[0],
|
||||
error_log=row[1],
|
||||
)
|
||||
293
apps/backend/app/routers/schedules.py
Normal file
293
apps/backend/app/routers/schedules.py
Normal file
@@ -0,0 +1,293 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""调度任务 CRUD API
|
||||
|
||||
提供 5 个端点:
|
||||
- GET /api/schedules — 列表(按 site_id 过滤)
|
||||
- POST /api/schedules — 创建
|
||||
- PUT /api/schedules/{id} — 更新
|
||||
- DELETE /api/schedules/{id} — 删除
|
||||
- PATCH /api/schedules/{id}/toggle — 启用/禁用
|
||||
|
||||
所有端点需要 JWT 认证,site_id 从 JWT 提取。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
from app.auth.dependencies import CurrentUser, get_current_user
|
||||
from app.database import get_connection
|
||||
from app.schemas.schedules import (
|
||||
CreateScheduleRequest,
|
||||
ScheduleResponse,
|
||||
UpdateScheduleRequest,
|
||||
)
|
||||
from app.services.scheduler import calculate_next_run
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/schedules", tags=["调度管理"])
|
||||
|
||||
|
||||
|
||||
def _row_to_response(row) -> ScheduleResponse:
|
||||
"""将数据库行转换为 ScheduleResponse。"""
|
||||
task_config = row[4] if isinstance(row[4], dict) else json.loads(row[4])
|
||||
schedule_config = row[5] if isinstance(row[5], dict) else json.loads(row[5])
|
||||
return ScheduleResponse(
|
||||
id=str(row[0]),
|
||||
site_id=row[1],
|
||||
name=row[2],
|
||||
task_codes=row[3] or [],
|
||||
task_config=task_config,
|
||||
schedule_config=schedule_config,
|
||||
enabled=row[6],
|
||||
last_run_at=row[7],
|
||||
next_run_at=row[8],
|
||||
run_count=row[9],
|
||||
last_status=row[10],
|
||||
created_at=row[11],
|
||||
updated_at=row[12],
|
||||
)
|
||||
|
||||
|
||||
# 查询列列表,复用于多个端点
|
||||
_SELECT_COLS = """
|
||||
id, site_id, name, task_codes, task_config, schedule_config,
|
||||
enabled, last_run_at, next_run_at, run_count, last_status,
|
||||
created_at, updated_at
|
||||
"""
|
||||
|
||||
|
||||
# ── GET /api/schedules — 列表 ────────────────────────────────
|
||||
|
||||
@router.get("", response_model=list[ScheduleResponse])
|
||||
async def list_schedules(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> list[ScheduleResponse]:
|
||||
"""获取当前门店的所有调度任务。"""
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"SELECT {_SELECT_COLS} FROM scheduled_tasks WHERE site_id = %s ORDER BY created_at DESC",
|
||||
(user.site_id,),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return [_row_to_response(row) for row in rows]
|
||||
|
||||
|
||||
# ── POST /api/schedules — 创建 ──────────────────────────────
|
||||
|
||||
@router.post("", response_model=ScheduleResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_schedule(
|
||||
body: CreateScheduleRequest,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> ScheduleResponse:
|
||||
"""创建调度任务,自动计算 next_run_at。"""
|
||||
now = datetime.now(timezone.utc)
|
||||
next_run = calculate_next_run(body.schedule_config, now)
|
||||
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""
|
||||
INSERT INTO scheduled_tasks
|
||||
(site_id, name, task_codes, task_config, schedule_config, enabled, next_run_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING {_SELECT_COLS}
|
||||
""",
|
||||
(
|
||||
user.site_id,
|
||||
body.name,
|
||||
body.task_codes,
|
||||
json.dumps(body.task_config),
|
||||
body.schedule_config.model_dump_json(),
|
||||
body.schedule_config.enabled,
|
||||
next_run,
|
||||
),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return _row_to_response(row)
|
||||
|
||||
|
||||
# ── PUT /api/schedules/{id} — 更新 ──────────────────────────
|
||||
|
||||
@router.put("/{schedule_id}", response_model=ScheduleResponse)
|
||||
async def update_schedule(
|
||||
schedule_id: str,
|
||||
body: UpdateScheduleRequest,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> ScheduleResponse:
|
||||
"""更新调度任务,仅更新请求中提供的字段。"""
|
||||
# 构建动态 SET 子句
|
||||
set_parts: list[str] = []
|
||||
params: list = []
|
||||
|
||||
if body.name is not None:
|
||||
set_parts.append("name = %s")
|
||||
params.append(body.name)
|
||||
if body.task_codes is not None:
|
||||
set_parts.append("task_codes = %s")
|
||||
params.append(body.task_codes)
|
||||
if body.task_config is not None:
|
||||
set_parts.append("task_config = %s")
|
||||
params.append(json.dumps(body.task_config))
|
||||
if body.schedule_config is not None:
|
||||
set_parts.append("schedule_config = %s")
|
||||
params.append(body.schedule_config.model_dump_json())
|
||||
# 更新调度配置时重新计算 next_run_at
|
||||
now = datetime.now(timezone.utc)
|
||||
next_run = calculate_next_run(body.schedule_config, now)
|
||||
set_parts.append("next_run_at = %s")
|
||||
params.append(next_run)
|
||||
|
||||
if not set_parts:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail="至少需要提供一个更新字段",
|
||||
)
|
||||
|
||||
set_parts.append("updated_at = NOW()")
|
||||
set_clause = ", ".join(set_parts)
|
||||
params.extend([schedule_id, user.site_id])
|
||||
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE scheduled_tasks
|
||||
SET {set_clause}
|
||||
WHERE id = %s AND site_id = %s
|
||||
RETURNING {_SELECT_COLS}
|
||||
""",
|
||||
params,
|
||||
)
|
||||
row = cur.fetchone()
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if row is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="调度任务不存在",
|
||||
)
|
||||
|
||||
return _row_to_response(row)
|
||||
|
||||
|
||||
# ── DELETE /api/schedules/{id} — 删除 ────────────────────────
|
||||
|
||||
@router.delete("/{schedule_id}")
|
||||
async def delete_schedule(
|
||||
schedule_id: str,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> dict:
|
||||
"""删除调度任务。"""
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"DELETE FROM scheduled_tasks WHERE id = %s AND site_id = %s",
|
||||
(schedule_id, user.site_id),
|
||||
)
|
||||
deleted = cur.rowcount
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if deleted == 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="调度任务不存在",
|
||||
)
|
||||
|
||||
return {"message": "调度任务已删除"}
|
||||
|
||||
|
||||
# ── PATCH /api/schedules/{id}/toggle — 启用/禁用 ─────────────
|
||||
|
||||
@router.patch("/{schedule_id}/toggle", response_model=ScheduleResponse)
|
||||
async def toggle_schedule(
|
||||
schedule_id: str,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> ScheduleResponse:
|
||||
"""切换调度任务的启用/禁用状态。
|
||||
|
||||
禁用时 next_run_at 置 NULL;启用时重新计算 next_run_at。
|
||||
"""
|
||||
conn = get_connection()
|
||||
try:
|
||||
# 先查询当前状态和调度配置
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT enabled, schedule_config FROM scheduled_tasks WHERE id = %s AND site_id = %s",
|
||||
(schedule_id, user.site_id),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
|
||||
if row is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="调度任务不存在",
|
||||
)
|
||||
|
||||
current_enabled = row[0]
|
||||
new_enabled = not current_enabled
|
||||
|
||||
if new_enabled:
|
||||
# 启用:重新计算 next_run_at
|
||||
schedule_config_raw = row[1] if isinstance(row[1], dict) else json.loads(row[1])
|
||||
from app.schemas.schedules import ScheduleConfigSchema
|
||||
schedule_cfg = ScheduleConfigSchema(**schedule_config_raw)
|
||||
now = datetime.now(timezone.utc)
|
||||
next_run = calculate_next_run(schedule_cfg, now)
|
||||
else:
|
||||
# 禁用:next_run_at 置 NULL
|
||||
next_run = None
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE scheduled_tasks
|
||||
SET enabled = %s, next_run_at = %s, updated_at = NOW()
|
||||
WHERE id = %s AND site_id = %s
|
||||
RETURNING {_SELECT_COLS}
|
||||
""",
|
||||
(new_enabled, next_run, schedule_id, user.site_id),
|
||||
)
|
||||
updated_row = cur.fetchone()
|
||||
conn.commit()
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return _row_to_response(updated_row)
|
||||
209
apps/backend/app/routers/tasks.py
Normal file
209
apps/backend/app/routers/tasks.py
Normal file
@@ -0,0 +1,209 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""任务注册表 & 配置 API
|
||||
|
||||
提供 4 个端点:
|
||||
- GET /api/tasks/registry — 按业务域分组的任务列表
|
||||
- GET /api/tasks/dwd-tables — 按业务域分组的 DWD 表定义
|
||||
- GET /api/tasks/flows — 7 种 Flow + 3 种处理模式
|
||||
- POST /api/tasks/validate — 验证 TaskConfig 并返回 CLI 命令预览
|
||||
|
||||
所有端点需要 JWT 认证。validate 端点从 JWT 注入 store_id。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.auth.dependencies import CurrentUser, get_current_user
|
||||
from app.config import ETL_PROJECT_PATH
|
||||
from app.schemas.tasks import (
|
||||
FlowDefinition,
|
||||
ProcessingModeDefinition,
|
||||
TaskConfigSchema,
|
||||
)
|
||||
from app.services.cli_builder import cli_builder
|
||||
from app.services.task_registry import (
|
||||
DWD_TABLES,
|
||||
FLOW_LAYER_MAP,
|
||||
get_dwd_tables_grouped_by_domain,
|
||||
get_tasks_grouped_by_domain,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/tasks", tags=["任务配置"])
|
||||
|
||||
|
||||
# ── 响应模型 ──────────────────────────────────────────────────
|
||||
|
||||
class TaskItem(BaseModel):
|
||||
code: str
|
||||
name: str
|
||||
description: str
|
||||
domain: str
|
||||
layer: str
|
||||
requires_window: bool
|
||||
is_ods: bool
|
||||
is_dimension: bool
|
||||
default_enabled: bool
|
||||
is_common: bool
|
||||
|
||||
|
||||
class DwdTableItem(BaseModel):
|
||||
table_name: str
|
||||
display_name: str
|
||||
domain: str
|
||||
ods_source: str
|
||||
is_dimension: bool
|
||||
|
||||
|
||||
class TaskRegistryResponse(BaseModel):
|
||||
"""按业务域分组的任务列表"""
|
||||
groups: dict[str, list[TaskItem]]
|
||||
|
||||
|
||||
class DwdTablesResponse(BaseModel):
|
||||
"""按业务域分组的 DWD 表定义"""
|
||||
groups: dict[str, list[DwdTableItem]]
|
||||
|
||||
|
||||
class FlowsResponse(BaseModel):
|
||||
"""Flow 定义 + 处理模式定义"""
|
||||
flows: list[FlowDefinition]
|
||||
processing_modes: list[ProcessingModeDefinition]
|
||||
|
||||
|
||||
class ValidateRequest(BaseModel):
|
||||
"""验证请求体 — 复用 TaskConfigSchema,但 store_id 由后端注入"""
|
||||
config: TaskConfigSchema
|
||||
|
||||
|
||||
class ValidateResponse(BaseModel):
|
||||
"""验证结果 + CLI 命令预览"""
|
||||
valid: bool
|
||||
command: str
|
||||
command_args: list[str]
|
||||
errors: list[str]
|
||||
|
||||
|
||||
# ── Flow 定义(静态) ────────────────────────────────────────
|
||||
|
||||
FLOW_DEFINITIONS: list[FlowDefinition] = [
|
||||
FlowDefinition(id="api_ods", name="API → ODS", layers=["ODS"]),
|
||||
FlowDefinition(id="api_ods_dwd", name="API → ODS → DWD", layers=["ODS", "DWD"]),
|
||||
FlowDefinition(id="api_full", name="API → ODS → DWD → DWS汇总 → DWS指数", layers=["ODS", "DWD", "DWS", "INDEX"]),
|
||||
FlowDefinition(id="ods_dwd", name="ODS → DWD", layers=["DWD"]),
|
||||
FlowDefinition(id="dwd_dws", name="DWD → DWS汇总", layers=["DWS"]),
|
||||
FlowDefinition(id="dwd_dws_index", name="DWD → DWS汇总 → DWS指数", layers=["DWS", "INDEX"]),
|
||||
FlowDefinition(id="dwd_index", name="DWD → DWS指数", layers=["INDEX"]),
|
||||
]
|
||||
|
||||
PROCESSING_MODE_DEFINITIONS: list[ProcessingModeDefinition] = [
|
||||
ProcessingModeDefinition(id="increment_only", name="仅增量处理", description="只处理新增和变更的数据"),
|
||||
ProcessingModeDefinition(id="verify_only", name="仅校验修复", description="校验现有数据并修复不一致(可选'校验前从 API 获取')"),
|
||||
ProcessingModeDefinition(id="increment_verify", name="增量 + 校验修复", description="先增量处理,再校验并修复"),
|
||||
]
|
||||
|
||||
|
||||
# ── 端点 ──────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/registry", response_model=TaskRegistryResponse)
|
||||
async def get_task_registry(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> TaskRegistryResponse:
|
||||
"""返回按业务域分组的任务列表"""
|
||||
grouped = get_tasks_grouped_by_domain()
|
||||
return TaskRegistryResponse(
|
||||
groups={
|
||||
domain: [
|
||||
TaskItem(
|
||||
code=t.code,
|
||||
name=t.name,
|
||||
description=t.description,
|
||||
domain=t.domain,
|
||||
layer=t.layer,
|
||||
requires_window=t.requires_window,
|
||||
is_ods=t.is_ods,
|
||||
is_dimension=t.is_dimension,
|
||||
default_enabled=t.default_enabled,
|
||||
is_common=t.is_common,
|
||||
)
|
||||
for t in tasks
|
||||
]
|
||||
for domain, tasks in grouped.items()
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/dwd-tables", response_model=DwdTablesResponse)
|
||||
async def get_dwd_tables(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> DwdTablesResponse:
|
||||
"""返回按业务域分组的 DWD 表定义"""
|
||||
grouped = get_dwd_tables_grouped_by_domain()
|
||||
return DwdTablesResponse(
|
||||
groups={
|
||||
domain: [
|
||||
DwdTableItem(
|
||||
table_name=t.table_name,
|
||||
display_name=t.display_name,
|
||||
domain=t.domain,
|
||||
ods_source=t.ods_source,
|
||||
is_dimension=t.is_dimension,
|
||||
)
|
||||
for t in tables
|
||||
]
|
||||
for domain, tables in grouped.items()
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/flows", response_model=FlowsResponse)
|
||||
async def get_flows(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> FlowsResponse:
|
||||
"""返回 7 种 Flow 定义和 3 种处理模式定义"""
|
||||
return FlowsResponse(
|
||||
flows=FLOW_DEFINITIONS,
|
||||
processing_modes=PROCESSING_MODE_DEFINITIONS,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/validate", response_model=ValidateResponse)
|
||||
async def validate_task_config(
|
||||
body: ValidateRequest,
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> ValidateResponse:
|
||||
"""验证 TaskConfig 并返回生成的 CLI 命令预览
|
||||
|
||||
从 JWT 注入 store_id,前端无需传递。
|
||||
"""
|
||||
config = body.config.model_copy(update={"store_id": user.site_id})
|
||||
errors: list[str] = []
|
||||
|
||||
# 验证 Flow ID
|
||||
if config.pipeline not in FLOW_LAYER_MAP:
|
||||
errors.append(f"无效的执行流程: {config.pipeline}")
|
||||
|
||||
# 验证任务列表非空
|
||||
if not config.tasks:
|
||||
errors.append("任务列表不能为空")
|
||||
|
||||
if errors:
|
||||
return ValidateResponse(
|
||||
valid=False,
|
||||
command="",
|
||||
command_args=[],
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
cmd_args = cli_builder.build_command(config, ETL_PROJECT_PATH)
|
||||
cmd_str = cli_builder.build_command_string(config, ETL_PROJECT_PATH)
|
||||
|
||||
return ValidateResponse(
|
||||
valid=True,
|
||||
command=cmd_str,
|
||||
command_args=cmd_args,
|
||||
errors=[],
|
||||
)
|
||||
104
apps/backend/app/routers/wx_callback.py
Normal file
104
apps/backend/app/routers/wx_callback.py
Normal file
@@ -0,0 +1,104 @@
|
||||
# AI_CHANGELOG
|
||||
# - 2026-02-19 | Prompt: 配置微信消息推送 | 新增微信消息推送回调接口,支持 GET 验签 + POST 消息接收
|
||||
|
||||
"""
|
||||
微信消息推送回调接口
|
||||
|
||||
处理两类请求:
|
||||
1. GET — 微信服务器验证(配置时触发一次)
|
||||
2. POST — 接收微信推送的消息/事件
|
||||
|
||||
安全模式下需要解密消息体,当前先用明文模式跑通,后续切安全模式。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Query, Request, Response
|
||||
|
||||
from app.config import get
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/wx", tags=["微信回调"])
|
||||
|
||||
# Token 从环境变量读取,与微信后台填写的一致
|
||||
# 放在 apps/backend/.env.local 中:WX_CALLBACK_TOKEN=你自定义的token
|
||||
WX_CALLBACK_TOKEN: str = get("WX_CALLBACK_TOKEN", "")
|
||||
|
||||
|
||||
def _check_signature(signature: str, timestamp: str, nonce: str) -> bool:
|
||||
"""
|
||||
验证请求是否来自微信服务器。
|
||||
|
||||
将 Token、timestamp、nonce 字典序排序后拼接,做 SHA1,
|
||||
与 signature 比对。
|
||||
"""
|
||||
if not WX_CALLBACK_TOKEN:
|
||||
logger.error("WX_CALLBACK_TOKEN 未配置")
|
||||
return False
|
||||
|
||||
items = sorted([WX_CALLBACK_TOKEN, timestamp, nonce])
|
||||
hash_str = hashlib.sha1("".join(items).encode("utf-8")).hexdigest()
|
||||
return hash_str == signature
|
||||
|
||||
|
||||
@router.get("/callback")
|
||||
async def verify(
|
||||
signature: str = Query(...),
|
||||
timestamp: str = Query(...),
|
||||
nonce: str = Query(...),
|
||||
echostr: str = Query(...),
|
||||
):
|
||||
"""
|
||||
微信服务器验证接口。
|
||||
|
||||
配置消息推送时微信会发 GET 请求,验签通过后原样返回 echostr。
|
||||
"""
|
||||
if _check_signature(signature, timestamp, nonce):
|
||||
logger.info("微信回调验证通过")
|
||||
# 必须原样返回 echostr(纯文本,不能包裹 JSON)
|
||||
return Response(content=echostr, media_type="text/plain")
|
||||
else:
|
||||
logger.warning("微信回调验签失败: signature=%s", signature)
|
||||
return Response(content="signature mismatch", status_code=403)
|
||||
|
||||
|
||||
@router.post("/callback")
|
||||
async def receive_message(
|
||||
request: Request,
|
||||
signature: str = Query(""),
|
||||
timestamp: str = Query(""),
|
||||
nonce: str = Query(""),
|
||||
):
|
||||
"""
|
||||
接收微信推送的消息/事件。
|
||||
|
||||
当前为明文模式,直接解析 JSON 包体。
|
||||
后续切安全模式时需增加 AES 解密逻辑。
|
||||
"""
|
||||
# 验签(POST 也带 signature 参数)
|
||||
if not _check_signature(signature, timestamp, nonce):
|
||||
logger.warning("消息推送验签失败")
|
||||
return Response(content="signature mismatch", status_code=403)
|
||||
|
||||
# 解析消息体
|
||||
body = await request.body()
|
||||
content_type = request.headers.get("content-type", "")
|
||||
|
||||
if "json" in content_type:
|
||||
import json
|
||||
try:
|
||||
data = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
data = {"raw": body.decode("utf-8", errors="replace")}
|
||||
else:
|
||||
# XML 格式暂不解析,记录原文
|
||||
data = {"raw_xml": body.decode("utf-8", errors="replace")}
|
||||
|
||||
logger.info("收到微信推送: MsgType=%s, Event=%s",
|
||||
data.get("MsgType", "?"), data.get("Event", "?"))
|
||||
|
||||
# TODO: 根据 MsgType/Event 分发处理(客服消息、订阅事件等)
|
||||
# 当前统一返回 success
|
||||
return Response(content="success", media_type="text/plain")
|
||||
37
apps/backend/app/routers/xcx_test.py
Normal file
37
apps/backend/app/routers/xcx_test.py
Normal file
@@ -0,0 +1,37 @@
|
||||
# AI_CHANGELOG
|
||||
# - 2026-02-19 | Prompt: 小程序 MVP 全链路验证 | 新增 /api/xcx-test 接口,从 test."xcx-test" 表读取 ti 列第一行
|
||||
|
||||
"""
|
||||
小程序 MVP 验证接口
|
||||
|
||||
从 test_zqyy_app 库的 test."xcx-test" 表读取数据,
|
||||
用于验证小程序 → 后端 → 数据库全链路连通性。
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from app.database import get_connection
|
||||
|
||||
router = APIRouter(prefix="/api/xcx-test", tags=["小程序MVP"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def get_xcx_test():
|
||||
"""
|
||||
读取 test."xcx-test" 表 ti 列第一行。
|
||||
|
||||
用于小程序 MVP 全链路验证:小程序 → API → DB → 返回数据。
|
||||
"""
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
# CHANGE 2026-02-19 | 读取 test schema 下的 xcx-test 表
|
||||
# 表名含连字符,必须用双引号包裹
|
||||
cur.execute('SELECT ti FROM test."xcx-test" LIMIT 1')
|
||||
row = cur.fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="无数据")
|
||||
|
||||
return {"ti": row[0]}
|
||||
Reference in New Issue
Block a user