Neo
6f8f12314f
包含多个会话的累积代码变更： - backend: AI 聊天服务、触发器调度、认证增强、WebSocket、调度器最小间隔 - admin-web: ETL 状态页、任务管理、调度配置、登录优化 - miniprogram: 看板页面、聊天集成、UI 组件、导航更新 - etl: DWS 新任务（finance_area_daily/board_cache）、连接器增强 - tenant-admin: 项目初始化 - db: 19 个迁移脚本（etl_feiqiu 11 + zqyy_app 8） - packages/shared: 枚举和工具函数更新 - tools: 数据库工具、报表生成、健康检查 - docs: PRD/架构/部署/合约文档更新 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
375 lines
13 KiB
Python
375 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
管理端 — 开发调试全链路日志路由。
|
||
|
||
端点清单（8 个，全部需要 JWT + admin 角色）：
|
||
- GET /api/admin/dev-trace/dates — 有日志数据的日期列表
|
||
- GET /api/admin/dev-trace/requests — 按条件分页查询请求列表
|
||
- GET /api/admin/dev-trace/request/{id} — 指定 request_id 的完整 trace
|
||
- POST /api/admin/dev-trace/cleanup — 按日期范围手动清理日志
|
||
- GET /api/admin/dev-trace/settings — 当前设置
|
||
- PUT /api/admin/dev-trace/settings — 更新运行时设置
|
||
- GET /api/admin/dev-trace/coverage — 最近一次覆盖率扫描结果
|
||
- POST /api/admin/dev-trace/coverage/scan — 手动触发覆盖率扫描
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any, Optional
|
||
|
||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||
from pydantic import BaseModel
|
||
|
||
from app.auth.dependencies import CurrentUser, get_current_user
|
||
from app.trace.cleanup import cleanup_date_range
|
||
from app.trace.config import get_trace_config
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
router = APIRouter(prefix="/api/admin/dev-trace", tags=["开发调试日志"])
|
||
|
||
# 日期目录名格式
|
||
_DATE_FORMAT = "%Y-%m-%d"
|
||
|
||
|
||
# ── 权限依赖 ──────────────────────────────────────────────
|
||
|
||
|
||
def _require_admin():
|
||
"""管理端依赖：仅要求 JWT 认证通过。
|
||
|
||
dev-trace 是开发调试工具，不涉及业务数据，无需检查业务角色
|
||
（site_admin / tenant_admin）。只要是 admin-web 的已认证用户即可访问。
|
||
"""
|
||
|
||
async def _dependency(
|
||
user: CurrentUser = Depends(get_current_user),
|
||
) -> CurrentUser:
|
||
return user
|
||
|
||
return _dependency
|
||
|
||
|
||
# ── Pydantic 请求/响应模型 ────────────────────────────────
|
||
|
||
|
||
class CleanupRequest(BaseModel):
|
||
"""手动清理请求体。"""
|
||
start_date: str
|
||
end_date: str
|
||
|
||
|
||
class SettingsUpdate(BaseModel):
|
||
"""运行时设置更新请求体（所有字段可选）。"""
|
||
enabled: Optional[bool] = None
|
||
retention_days: Optional[int] = None
|
||
log_sql: Optional[bool] = None
|
||
log_params: Optional[bool] = None
|
||
|
||
|
||
# ── 辅助函数 ──────────────────────────────────────────────
|
||
|
||
|
||
def _get_base_dir() -> Path:
|
||
"""获取日志根目录 Path 对象。"""
|
||
return Path(get_trace_config().log_dir)
|
||
|
||
|
||
def _is_date_dir(name: str) -> bool:
|
||
"""判断目录名是否为 YYYY-MM-DD 格式。"""
|
||
try:
|
||
datetime.strptime(name, _DATE_FORMAT)
|
||
return True
|
||
except ValueError:
|
||
return False
|
||
|
||
|
||
def _read_jsonl_file(filepath: Path) -> list[dict[str, Any]]:
|
||
"""逐行读取 .jsonl 文件，跳过解析失败的行。"""
|
||
records: list[dict[str, Any]] = []
|
||
if not filepath.exists():
|
||
return records
|
||
with open(filepath, "r", encoding="utf-8") as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
records.append(json.loads(line))
|
||
except json.JSONDecodeError:
|
||
continue
|
||
return records
|
||
|
||
|
||
def _match_filter(record: dict[str, Any], **filters: Any) -> bool:
|
||
"""检查单条 trace 记录是否满足所有筛选条件。"""
|
||
# trace_type
|
||
if filters.get("trace_type") and record.get("trace_type") != filters["trace_type"]:
|
||
return False
|
||
|
||
# method
|
||
if filters.get("method") and record.get("method", "").upper() != filters["method"].upper():
|
||
return False
|
||
|
||
# path_contains
|
||
if filters.get("path_contains") and filters["path_contains"].lower() not in (record.get("path") or "").lower():
|
||
return False
|
||
|
||
# status_code
|
||
if filters.get("status_code") is not None and record.get("status_code") != filters["status_code"]:
|
||
return False
|
||
|
||
# min_duration
|
||
if filters.get("min_duration") is not None and (record.get("total_duration_ms") or 0) < filters["min_duration"]:
|
||
return False
|
||
|
||
# has_error
|
||
if filters.get("has_error") is not None:
|
||
has_err = record.get("error") is not None
|
||
if filters["has_error"] != has_err:
|
||
return False
|
||
|
||
# span_type — 检查 spans 列表中是否包含指定类型
|
||
if filters.get("span_type"):
|
||
span_types = {s.get("span_type") for s in record.get("spans", [])}
|
||
if filters["span_type"] not in span_types:
|
||
return False
|
||
|
||
# start_time / end_time — 基于 record.timestamp
|
||
ts_str = record.get("timestamp", "")
|
||
if ts_str and (filters.get("start_time") or filters.get("end_time")):
|
||
try:
|
||
rec_dt = datetime.fromisoformat(ts_str)
|
||
rec_time = rec_dt.time()
|
||
if filters.get("start_time") and rec_time < filters["start_time"]:
|
||
return False
|
||
if filters.get("end_time") and rec_time > filters["end_time"]:
|
||
return False
|
||
except (ValueError, TypeError):
|
||
pass
|
||
|
||
return True
|
||
|
||
|
||
# ── 1. GET /dates — 有日志数据的日期列表 ─────────────────
|
||
|
||
|
||
@router.get("/dates")
|
||
async def list_dates(
|
||
user: CurrentUser = Depends(_require_admin()),
|
||
) -> dict[str, list[str]]:
|
||
"""返回有日志数据的日期列表（降序排列）。"""
|
||
base = _get_base_dir()
|
||
if not base.exists():
|
||
return {"dates": []}
|
||
|
||
dates = sorted(
|
||
[d.name for d in base.iterdir() if d.is_dir() and _is_date_dir(d.name)],
|
||
reverse=True,
|
||
)
|
||
return {"dates": dates}
|
||
|
||
|
||
# ── 2. GET /requests — 按条件分页查询请求列表 ─────────────
|
||
|
||
|
||
@router.get("/requests")
|
||
async def list_requests(
|
||
date: str = Query(..., description="日期，格式 YYYY-MM-DD"),
|
||
start_time: Optional[str] = Query(None, description="起始时间 HH:MM:SS"),
|
||
end_time: Optional[str] = Query(None, description="结束时间 HH:MM:SS"),
|
||
trace_type: Optional[str] = Query(None, description="trace 类型：http/sse/ws/job"),
|
||
method: Optional[str] = Query(None, description="HTTP 方法：GET/POST/PUT/DELETE"),
|
||
path_contains: Optional[str] = Query(None, description="路径关键词"),
|
||
status_code: Optional[int] = Query(None, description="HTTP 状态码"),
|
||
min_duration: Optional[float] = Query(None, description="最小耗时（ms）"),
|
||
has_error: Optional[bool] = Query(None, description="是否有错误"),
|
||
span_type: Optional[str] = Query(None, description="包含的 span 类型"),
|
||
page: int = Query(1, ge=1, description="页码"),
|
||
page_size: int = Query(50, ge=1, le=200, description="每页条数"),
|
||
user: CurrentUser = Depends(_require_admin()),
|
||
) -> dict[str, Any]:
|
||
"""按条件分页查询指定日期的请求列表。"""
|
||
# 解析时间参数
|
||
parsed_start = None
|
||
parsed_end = None
|
||
if start_time:
|
||
try:
|
||
parsed_start = datetime.strptime(start_time, "%H:%M:%S").time()
|
||
except ValueError:
|
||
raise HTTPException(status_code=400, detail="start_time 格式无效，应为 HH:MM:SS")
|
||
if end_time:
|
||
try:
|
||
parsed_end = datetime.strptime(end_time, "%H:%M:%S").time()
|
||
except ValueError:
|
||
raise HTTPException(status_code=400, detail="end_time 格式无效，应为 HH:MM:SS")
|
||
|
||
# 读取指定日期目录下所有 .jsonl 文件
|
||
date_dir = _get_base_dir() / date
|
||
if not date_dir.exists() or not date_dir.is_dir():
|
||
return {"items": [], "total": 0, "page": page, "page_size": page_size}
|
||
|
||
all_records: list[dict[str, Any]] = []
|
||
for f in sorted(date_dir.glob("*.jsonl")):
|
||
all_records.extend(_read_jsonl_file(f))
|
||
|
||
# 过滤
|
||
filtered = [
|
||
r for r in all_records
|
||
if _match_filter(
|
||
r,
|
||
trace_type=trace_type,
|
||
method=method,
|
||
path_contains=path_contains,
|
||
status_code=status_code,
|
||
min_duration=min_duration,
|
||
has_error=has_error,
|
||
span_type=span_type,
|
||
start_time=parsed_start,
|
||
end_time=parsed_end,
|
||
)
|
||
]
|
||
|
||
# 按时间降序排列
|
||
filtered.sort(key=lambda r: r.get("timestamp", ""), reverse=True)
|
||
|
||
total = len(filtered)
|
||
start_idx = (page - 1) * page_size
|
||
items = filtered[start_idx : start_idx + page_size]
|
||
|
||
# 列表项不返回完整 spans，只返回摘要字段
|
||
summary_items = []
|
||
for item in items:
|
||
summary_items.append({
|
||
"request_id": item.get("request_id"),
|
||
"trace_type": item.get("trace_type"),
|
||
"timestamp": item.get("timestamp"),
|
||
"method": item.get("method"),
|
||
"path": item.get("path"),
|
||
"status_code": item.get("status_code"),
|
||
"total_duration_ms": item.get("total_duration_ms"),
|
||
"user_id": item.get("user_id"),
|
||
"site_id": item.get("site_id"),
|
||
"db_query_count": item.get("db_query_count"),
|
||
"db_total_ms": item.get("db_total_ms"),
|
||
"error": item.get("error"),
|
||
"span_count": len(item.get("spans", [])),
|
||
})
|
||
|
||
return {"items": summary_items, "total": total, "page": page, "page_size": page_size}
|
||
|
||
|
||
# ── 3. GET /request/{request_id} — 完整 trace 记录 ──────
|
||
|
||
|
||
@router.get("/request/{request_id}")
|
||
async def get_request_detail(
|
||
request_id: str,
|
||
user: CurrentUser = Depends(_require_admin()),
|
||
) -> dict[str, Any]:
|
||
"""返回指定 request_id 的完整 trace 记录（含所有 spans）。"""
|
||
base = _get_base_dir()
|
||
if not base.exists():
|
||
raise HTTPException(status_code=404, detail="未找到该 request_id 的 trace 记录")
|
||
|
||
# 搜索所有日期目录下的 .jsonl 文件
|
||
for date_dir in sorted(base.iterdir(), reverse=True):
|
||
if not date_dir.is_dir() or not _is_date_dir(date_dir.name):
|
||
continue
|
||
for f in date_dir.glob("*.jsonl"):
|
||
for record in _read_jsonl_file(f):
|
||
if record.get("request_id") == request_id:
|
||
return record
|
||
|
||
raise HTTPException(status_code=404, detail="未找到该 request_id 的 trace 记录")
|
||
|
||
|
||
# ── 4. POST /cleanup — 按日期范围手动清理 ────────────────
|
||
|
||
|
||
@router.post("/cleanup")
|
||
async def cleanup_logs(
|
||
body: CleanupRequest,
|
||
user: CurrentUser = Depends(_require_admin()),
|
||
) -> dict[str, Any]:
|
||
"""按日期范围手动清理日志目录。"""
|
||
# 校验日期格式
|
||
try:
|
||
datetime.strptime(body.start_date, _DATE_FORMAT)
|
||
datetime.strptime(body.end_date, _DATE_FORMAT)
|
||
except ValueError:
|
||
raise HTTPException(status_code=400, detail="日期格式无效，应为 YYYY-MM-DD")
|
||
|
||
if body.start_date > body.end_date:
|
||
raise HTTPException(status_code=400, detail="start_date 不能晚于 end_date")
|
||
|
||
result = cleanup_date_range(body.start_date, body.end_date)
|
||
return {
|
||
"deleted_dates": result["deleted_dirs"],
|
||
"deleted_files": result["deleted_count"],
|
||
}
|
||
|
||
|
||
# ── 5. GET /settings — 当前设置 ──────────────────────────
|
||
|
||
|
||
@router.get("/settings")
|
||
async def get_settings(
|
||
user: CurrentUser = Depends(_require_admin()),
|
||
) -> dict[str, Any]:
|
||
"""返回当前 trace 运行时设置。"""
|
||
return get_trace_config().get_settings()
|
||
|
||
|
||
# ── 6. PUT /settings — 更新运行时设置 ────────────────────
|
||
|
||
|
||
@router.put("/settings")
|
||
async def update_settings(
|
||
body: SettingsUpdate,
|
||
user: CurrentUser = Depends(_require_admin()),
|
||
) -> dict[str, Any]:
|
||
"""更新 trace 运行时设置（不需重启，重启后回退到 .env 值）。"""
|
||
cfg = get_trace_config()
|
||
cfg.update_settings(
|
||
enabled=body.enabled,
|
||
retention_days=body.retention_days,
|
||
log_sql=body.log_sql,
|
||
log_params=body.log_params,
|
||
)
|
||
return cfg.get_settings()
|
||
|
||
|
||
# ── 7. GET /coverage — 最近一次覆盖率扫描结果 ────────────
|
||
|
||
|
||
@router.get("/coverage")
|
||
async def get_coverage(
|
||
user: CurrentUser = Depends(_require_admin()),
|
||
) -> dict[str, Any]:
|
||
"""返回最近一次覆盖率扫描结果（缓存）。"""
|
||
from app.trace.coverage import get_cached_coverage, run_coverage_scan
|
||
|
||
result = get_cached_coverage()
|
||
if result is None:
|
||
# 首次访问时自动扫描一次
|
||
result = run_coverage_scan()
|
||
return result
|
||
|
||
|
||
# ── 8. POST /coverage/scan — 手动触发覆盖率扫描 ──────────
|
||
|
||
|
||
@router.post("/coverage/scan")
|
||
async def trigger_coverage_scan(
|
||
user: CurrentUser = Depends(_require_admin()),
|
||
) -> dict[str, Any]:
|
||
"""手动触发覆盖率扫描，返回最新结果。"""
|
||
from app.trace.coverage import run_coverage_scan
|
||
|
||
return run_coverage_scan()
|