在前后端开发联调前 的提交20260223

This commit is contained in:
Neo
2026-02-23 23:02:20 +08:00
parent 254ccb1e77
commit fafc95e64c
1142 changed files with 10366960 additions and 36957 deletions

View File

@@ -12,7 +12,9 @@ from fastapi.middleware.cors import CORSMiddleware
from app import config
# CHANGE 2026-02-19 | 新增 xcx_test 路由MVP 验证)+ wx_callback 路由(微信消息推送)
from app.routers import auth, execution, schedules, tasks, env_config, db_viewer, etl_status, xcx_test, wx_callback
# CHANGE 2026-02-22 | 新增 member_birthday 路由(助教手动补录会员生日)
# CHANGE 2026-02-23 | 新增 ops_panel 路由(运维控制面板)
from app.routers import auth, execution, schedules, tasks, env_config, db_viewer, etl_status, xcx_test, wx_callback, member_birthday, ops_panel
from app.services.scheduler import scheduler
from app.services.task_queue import task_queue
from app.ws.logs import ws_router
@@ -60,6 +62,8 @@ app.include_router(etl_status.router)
app.include_router(ws_router)
app.include_router(xcx_test.router)
app.include_router(wx_callback.router)
app.include_router(member_birthday.router)
app.include_router(ops_panel.router)
@app.get("/health", tags=["系统"])

View File

@@ -0,0 +1,57 @@
"""
会员生日手动补录路由。
- POST /api/member-birthday — 助教提交会员生日UPSERT
"""
import logging
from fastapi import APIRouter, HTTPException, status
from app.database import get_connection
from app.schemas.member_birthday import MemberBirthdaySubmit
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api", tags=["会员生日"])
@router.post("/member-birthday")
async def submit_member_birthday(body: MemberBirthdaySubmit):
"""
助教提交会员生日UPSERT
同一 (member_id, assistant_id) 组合重复提交时,
更新 birthday_value 和 recorded_at保留其他助教的记录。
"""
sql = """
INSERT INTO member_birthday_manual
(member_id, birthday_value, recorded_by_assistant_id, recorded_by_name, site_id)
VALUES (%s, %s, %s, %s, %s)
ON CONFLICT (member_id, recorded_by_assistant_id)
DO UPDATE SET
birthday_value = EXCLUDED.birthday_value,
recorded_at = NOW()
"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(sql, (
body.member_id,
body.birthday_value,
body.assistant_id,
body.assistant_name,
body.site_id,
))
conn.commit()
except Exception:
conn.rollback()
logger.exception("会员生日 UPSERT 失败: member_id=%s", body.member_id)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="生日提交失败,请稍后重试",
)
finally:
conn.close()
return {"status": "ok"}

View File

@@ -0,0 +1,372 @@
"""
运维控制面板 API
提供服务器各环境的服务状态查看、启停控制、Git 操作和配置管理。
面向管理后台的运维面板页面。
"""
import asyncio
import os
import subprocess
import platform
from datetime import datetime
from pathlib import Path
from typing import Any
import psutil
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
router = APIRouter(prefix="/api/ops", tags=["运维面板"])
# ---- 环境定义 ----
# 服务器上的两套环境;开发机上回退到本机路径(方便调试)
_SERVER_BASE = Path("D:/NeoZQYY")
ENVIRONMENTS: dict[str, dict[str, Any]] = {
"test": {
"label": "测试环境",
"repo_path": str(_SERVER_BASE / "test" / "repo"),
"branch": "test",
"port": 8001,
"bat_script": str(_SERVER_BASE / "scripts" / "start-test-api.bat"),
"window_title": "NeoZQYY Test API",
},
"prod": {
"label": "正式环境",
"repo_path": str(_SERVER_BASE / "prod" / "repo"),
"branch": "master",
"port": 8000,
"bat_script": str(_SERVER_BASE / "scripts" / "start-prod-api.bat"),
"window_title": "NeoZQYY Prod API",
},
}
# ---- 数据模型 ----
class ServiceStatus(BaseModel):
env: str
label: str
running: bool
pid: int | None = None
port: int
uptime_seconds: float | None = None
memory_mb: float | None = None
cpu_percent: float | None = None
class GitInfo(BaseModel):
env: str
branch: str
last_commit_hash: str
last_commit_message: str
last_commit_time: str
has_local_changes: bool
class SystemInfo(BaseModel):
cpu_percent: float
memory_total_gb: float
memory_used_gb: float
memory_percent: float
disk_total_gb: float
disk_used_gb: float
disk_percent: float
boot_time: str
class EnvFileContent(BaseModel):
content: str
class GitPullResult(BaseModel):
env: str
success: bool
output: str
class ServiceActionResult(BaseModel):
env: str
action: str
success: bool
message: str
# ---- 辅助函数 ----
def _find_uvicorn_process(port: int) -> psutil.Process | None:
"""查找监听指定端口的 uvicorn 进程。"""
for proc in psutil.process_iter(["pid", "name", "cmdline"]):
try:
cmdline = proc.info.get("cmdline") or []
cmdline_str = " ".join(cmdline)
# 匹配 uvicorn 进程且包含对应端口
if "uvicorn" in cmdline_str and str(port) in cmdline_str:
return proc
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
return None
def _run_cmd(cmd: str | list[str], cwd: str | None = None, timeout: int = 30) -> tuple[bool, str]:
"""执行命令并返回 (成功, 输出)。"""
try:
result = subprocess.run(
cmd,
cwd=cwd,
capture_output=True,
text=True,
timeout=timeout,
shell=isinstance(cmd, str),
encoding="utf-8",
errors="replace",
)
output = (result.stdout + "\n" + result.stderr).strip()
return result.returncode == 0, output
except subprocess.TimeoutExpired:
return False, "命令执行超时"
except Exception as e:
return False, str(e)
# ---- 系统信息 ----
@router.get("/system", response_model=SystemInfo)
async def get_system_info():
"""获取服务器系统资源概况。"""
mem = psutil.virtual_memory()
disk = psutil.disk_usage("D:\\") if os.path.exists("D:\\") else psutil.disk_usage("/")
boot = datetime.fromtimestamp(psutil.boot_time())
return SystemInfo(
cpu_percent=psutil.cpu_percent(interval=0.5),
memory_total_gb=round(mem.total / (1024 ** 3), 2),
memory_used_gb=round(mem.used / (1024 ** 3), 2),
memory_percent=mem.percent,
disk_total_gb=round(disk.total / (1024 ** 3), 2),
disk_used_gb=round(disk.used / (1024 ** 3), 2),
disk_percent=disk.percent,
boot_time=boot.isoformat(),
)
# ---- 服务状态 ----
@router.get("/services", response_model=list[ServiceStatus])
async def get_services_status():
"""获取所有环境的服务运行状态。"""
results = []
for env_key, env_cfg in ENVIRONMENTS.items():
proc = _find_uvicorn_process(env_cfg["port"])
if proc:
try:
mem_info = proc.memory_info()
create_time = proc.create_time()
results.append(ServiceStatus(
env=env_key,
label=env_cfg["label"],
running=True,
pid=proc.pid,
port=env_cfg["port"],
uptime_seconds=round(datetime.now().timestamp() - create_time, 1),
memory_mb=round(mem_info.rss / (1024 ** 2), 1),
cpu_percent=proc.cpu_percent(interval=0.1),
))
except (psutil.NoSuchProcess, psutil.AccessDenied):
results.append(ServiceStatus(
env=env_key, label=env_cfg["label"],
running=False, port=env_cfg["port"],
))
else:
results.append(ServiceStatus(
env=env_key, label=env_cfg["label"],
running=False, port=env_cfg["port"],
))
return results
# ---- 服务启停 ----
@router.post("/services/{env}/start", response_model=ServiceActionResult)
async def start_service(env: str):
"""启动指定环境的后端服务。"""
if env not in ENVIRONMENTS:
raise HTTPException(404, f"未知环境: {env}")
cfg = ENVIRONMENTS[env]
proc = _find_uvicorn_process(cfg["port"])
if proc:
return ServiceActionResult(
env=env, action="start", success=True,
message=f"服务已在运行中 (PID: {proc.pid})",
)
bat_path = cfg["bat_script"]
if not os.path.exists(bat_path):
raise HTTPException(400, f"启动脚本不存在: {bat_path}")
# 通过 start 命令在新窗口中启动 bat 脚本
try:
subprocess.Popen(
f'start "{cfg["window_title"]}" cmd /c "{bat_path}"',
shell=True,
)
# 等待进程启动
await asyncio.sleep(3)
new_proc = _find_uvicorn_process(cfg["port"])
if new_proc:
return ServiceActionResult(
env=env, action="start", success=True,
message=f"服务已启动 (PID: {new_proc.pid})",
)
else:
return ServiceActionResult(
env=env, action="start", success=False,
message="启动命令已执行,但未检测到进程,请检查日志",
)
except Exception as e:
return ServiceActionResult(
env=env, action="start", success=False, message=str(e),
)
@router.post("/services/{env}/stop", response_model=ServiceActionResult)
async def stop_service(env: str):
"""停止指定环境的后端服务。"""
if env not in ENVIRONMENTS:
raise HTTPException(404, f"未知环境: {env}")
cfg = ENVIRONMENTS[env]
proc = _find_uvicorn_process(cfg["port"])
if not proc:
return ServiceActionResult(
env=env, action="stop", success=True, message="服务未在运行",
)
try:
# 终止进程树(包括子进程)
parent = psutil.Process(proc.pid)
children = parent.children(recursive=True)
for child in children:
child.terminate()
parent.terminate()
# 等待进程退出
gone, alive = psutil.wait_procs([parent] + children, timeout=5)
for p in alive:
p.kill()
return ServiceActionResult(
env=env, action="stop", success=True, message="服务已停止",
)
except Exception as e:
return ServiceActionResult(
env=env, action="stop", success=False, message=str(e),
)
@router.post("/services/{env}/restart", response_model=ServiceActionResult)
async def restart_service(env: str):
"""重启指定环境的后端服务。"""
stop_result = await stop_service(env)
if not stop_result.success and "未在运行" not in stop_result.message:
return ServiceActionResult(
env=env, action="restart", success=False,
message=f"停止失败: {stop_result.message}",
)
await asyncio.sleep(1)
start_result = await start_service(env)
return ServiceActionResult(
env=env, action="restart",
success=start_result.success,
message=start_result.message,
)
# ---- Git 操作 ----
@router.get("/git", response_model=list[GitInfo])
async def get_git_info():
"""获取所有环境的 Git 状态。"""
results = []
for env_key, env_cfg in ENVIRONMENTS.items():
repo = env_cfg["repo_path"]
if not os.path.isdir(os.path.join(repo, ".git")):
results.append(GitInfo(
env=env_key, branch="N/A",
last_commit_hash="N/A", last_commit_message="仓库不存在",
last_commit_time="", has_local_changes=False,
))
continue
_, branch = _run_cmd(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=repo)
_, log_out = _run_cmd(
["git", "log", "-1", "--format=%H|%s|%ci"],
cwd=repo,
)
_, status_out = _run_cmd(["git", "status", "--porcelain"], cwd=repo)
parts = log_out.strip().split("|", 2) if log_out else ["", "", ""]
results.append(GitInfo(
env=env_key,
branch=branch.strip(),
last_commit_hash=parts[0][:8] if parts[0] else "N/A",
last_commit_message=parts[1] if len(parts) > 1 else "",
last_commit_time=parts[2] if len(parts) > 2 else "",
has_local_changes=bool(status_out.strip()),
))
return results
@router.post("/git/{env}/pull", response_model=GitPullResult)
async def git_pull(env: str):
"""对指定环境执行 git pull。"""
if env not in ENVIRONMENTS:
raise HTTPException(404, f"未知环境: {env}")
cfg = ENVIRONMENTS[env]
repo = cfg["repo_path"]
if not os.path.isdir(os.path.join(repo, ".git")):
raise HTTPException(400, f"仓库路径不存在: {repo}")
success, output = _run_cmd(["git", "pull", "--ff-only"], cwd=repo, timeout=60)
return GitPullResult(env=env, success=success, output=output)
@router.post("/git/{env}/sync-deps", response_model=ServiceActionResult)
async def sync_deps(env: str):
"""对指定环境执行 uv sync --all-packages。"""
if env not in ENVIRONMENTS:
raise HTTPException(404, f"未知环境: {env}")
cfg = ENVIRONMENTS[env]
repo = cfg["repo_path"]
success, output = _run_cmd(["uv", "sync", "--all-packages"], cwd=repo, timeout=120)
return ServiceActionResult(
env=env, action="sync-deps", success=success, message=output[:500],
)
# ---- 环境配置管理 ----
@router.get("/env-file/{env}")
async def get_env_file(env: str):
"""读取指定环境的 .env 文件(敏感值脱敏)。"""
if env not in ENVIRONMENTS:
raise HTTPException(404, f"未知环境: {env}")
env_path = Path(ENVIRONMENTS[env]["repo_path"]) / ".env"
if not env_path.exists():
raise HTTPException(404, f".env 文件不存在: {env_path}")
lines = env_path.read_text(encoding="utf-8").splitlines()
masked_lines = []
sensitive_keys = {"PASSWORD", "SECRET", "TOKEN", "DSN", "APP_SECRET"}
for line in lines:
stripped = line.strip()
if stripped and not stripped.startswith("#") and "=" in stripped:
key = stripped.split("=", 1)[0].strip()
if any(s in key.upper() for s in sensitive_keys):
masked_lines.append(f"{key}=********")
continue
masked_lines.append(line)
return {"env": env, "content": "\n".join(masked_lines)}

View File

@@ -4,7 +4,7 @@
提供 4 个端点:
- GET /api/tasks/registry — 按业务域分组的任务列表
- GET /api/tasks/dwd-tables — 按业务域分组的 DWD 表定义
- GET /api/tasks/flows — 7 种 Flow + 3 种处理模式
- GET /api/tasks/flows — 7 种 Flow + 4 种处理模式
- POST /api/tasks/validate — 验证 TaskConfig 并返回 CLI 命令预览
所有端点需要 JWT 认证。validate 端点从 JWT 注入 store_id。
@@ -103,6 +103,7 @@ PROCESSING_MODE_DEFINITIONS: list[ProcessingModeDefinition] = [
ProcessingModeDefinition(id="increment_only", name="仅增量处理", description="只处理新增和变更的数据"),
ProcessingModeDefinition(id="verify_only", name="仅校验修复", description="校验现有数据并修复不一致"),
ProcessingModeDefinition(id="increment_verify", name="增量 + 校验修复", description="先增量处理,再校验并修复"),
ProcessingModeDefinition(id="full_window", name="全窗口处理", description="用 API 返回数据的实际时间范围处理全部层,无需校验"),
]
@@ -163,7 +164,7 @@ async def get_dwd_tables(
async def get_flows(
user: CurrentUser = Depends(get_current_user),
) -> FlowsResponse:
"""返回 7 种 Flow 定义和 3 种处理模式定义"""
"""返回 7 种 Flow 定义和 4 种处理模式定义"""
return FlowsResponse(
flows=FLOW_DEFINITIONS,
processing_modes=PROCESSING_MODE_DEFINITIONS,
@@ -183,8 +184,9 @@ async def validate_task_config(
errors: list[str] = []
# 验证 Flow ID
if config.pipeline not in FLOW_LAYER_MAP:
errors.append(f"无效的执行流程: {config.pipeline}")
# CHANGE [2026-02-20] intent: pipeline → flow统一命名
if config.flow not in FLOW_LAYER_MAP:
errors.append(f"无效的执行流程: {config.flow}")
# 验证任务列表非空
if not config.tasks:

View File

@@ -0,0 +1,19 @@
"""
会员生日手动补录相关 Pydantic 模型。
- MemberBirthdaySubmit助教提交会员生日请求体
"""
from datetime import date
from pydantic import BaseModel, Field
class MemberBirthdaySubmit(BaseModel):
"""助教提交会员生日请求。"""
member_id: int = Field(..., gt=0, description="会员 ID")
birthday_value: date = Field(..., description="生日日期")
assistant_id: int = Field(..., gt=0, description="助教 ID")
assistant_name: str = Field(..., min_length=1, max_length=50, description="助教姓名")
site_id: int = Field(..., gt=0, description="门店 ID")

View File

@@ -13,8 +13,8 @@ class TaskConfigSchema(BaseModel):
"""任务配置 — 前后端传输格式
字段与 CLI 参数的映射关系:
- pipeline → --pipelineFlow ID7 种之一)
- processing_mode → --processing-mode3 种处理模式)
- flow → --flowFlow ID7 种之一)
- processing_mode → --processing-mode4 种处理模式)
- tasks → --tasks逗号分隔
- dry_run → --dry-run布尔标志
- window_mode → 决定使用 lookback 还是 custom 时间窗口(仅前端逻辑,不直接映射 CLI 参数)
@@ -30,7 +30,8 @@ class TaskConfigSchema(BaseModel):
"""
tasks: list[str]
pipeline: str = "api_ods_dwd"
# CHANGE [2026-02-20] intent: pipeline → flow统一命名消除历史别名
flow: str = "api_ods_dwd"
processing_mode: str = "increment_only"
dry_run: bool = False
window_mode: str = "lookback"

View File

@@ -6,7 +6,7 @@
支持:
- 7 种 Flowapi_ods / api_ods_dwd / api_full / ods_dwd / dwd_dws / dwd_dws_index / dwd_index
- 3 种处理模式increment_only / verify_only / increment_verify
- 4 种处理模式increment_only / verify_only / increment_verify / full_window
- 自动注入 --store-id 参数
"""
@@ -30,6 +30,7 @@ VALID_PROCESSING_MODES: set[str] = {
"increment_only",
"verify_only",
"increment_verify",
"full_window",
}
# CLI 支持的 extra_args 键(值类型 + 布尔类型)
@@ -72,7 +73,8 @@ class CLIBuilder:
cmd: list[str] = [python_executable, "-m", "cli.main"]
# -- Flow执行流程 --
cmd.extend(["--flow", config.pipeline])
# CHANGE [2026-02-20] intent: pipeline → flow统一命名
cmd.extend(["--flow", config.flow])
# -- 处理模式 --
if config.processing_mode:

View File

@@ -46,7 +46,7 @@ ODS_TASKS: list[TaskDefinition] = [
TaskDefinition("ODS_ASSISTANT_LEDGER", "助教服务记录", "抽取助教服务流水", "助教", "ODS", is_ods=True),
TaskDefinition("ODS_ASSISTANT_ABOLISH", "助教取消记录", "抽取助教取消/作废记录", "助教", "ODS", is_ods=True),
TaskDefinition("ODS_SETTLEMENT_RECORDS", "结算记录", "抽取订单结算记录", "结算", "ODS", is_ods=True),
TaskDefinition("ODS_SETTLEMENT_TICKET", "结账小票", "抽取结账小票明细", "结算", "ODS", is_ods=True),
# CHANGE [2026-07-20] intent: 同步 ETL 侧移除——ODS_SETTLEMENT_TICKET 已在 Task 7.3 中彻底移除
TaskDefinition("ODS_TABLE_USE", "台费流水", "抽取台费使用流水", "台桌", "ODS", is_ods=True),
TaskDefinition("ODS_TABLE_FEE_DISCOUNT", "台费折扣", "抽取台费折扣记录", "台桌", "ODS", is_ods=True),
TaskDefinition("ODS_TABLES", "台桌主数据", "抽取门店台桌信息", "台桌", "ODS", is_ods=True, requires_window=False),
@@ -59,7 +59,7 @@ ODS_TASKS: list[TaskDefinition] = [
TaskDefinition("ODS_RECHARGE_SETTLE", "充值结算", "抽取充值结算记录", "会员", "ODS", is_ods=True),
TaskDefinition("ODS_GROUP_PACKAGE", "团购套餐", "抽取团购套餐定义", "团购", "ODS", is_ods=True, requires_window=False),
TaskDefinition("ODS_GROUP_BUY_REDEMPTION", "团购核销", "抽取团购核销记录", "团购", "ODS", is_ods=True),
TaskDefinition("ODS_INVENTORY_STOCK", "库存快照", "抽取商品库存汇总", "库存", "ODS", is_ods=True, requires_window=False),
TaskDefinition("ODS_INVENTORY_STOCK", "库存快照", "抽取商品库存汇总", "库存", "ODS", is_ods=True),
TaskDefinition("ODS_INVENTORY_CHANGE", "库存变动", "抽取库存出入库记录", "库存", "ODS", is_ods=True),
TaskDefinition("ODS_GOODS_CATEGORY", "商品分类", "抽取商品分类树", "商品", "ODS", is_ods=True, requires_window=False),
TaskDefinition("ODS_STORE_GOODS", "门店商品", "抽取门店商品主数据", "商品", "ODS", is_ods=True, requires_window=False),
@@ -91,6 +91,10 @@ DWS_TASKS: list[TaskDefinition] = [
TaskDefinition("DWS_FINANCE_DISCOUNT_DETAIL", "折扣明细", "汇总折扣明细", "财务", "DWS"),
# CHANGE [2026-02-19] intent: 同步 ETL 侧合并——原 DWS_RETENTION_CLEANUP / DWS_MV_REFRESH_* 已合并为 DWS_MAINTENANCE
TaskDefinition("DWS_MAINTENANCE", "DWS 维护", "刷新物化视图 + 清理过期留存数据", "通用", "DWS", requires_window=False, is_common=False),
# CHANGE [2026-07-20] intent: 注册 DWS 库存汇总任务(日/周/月),依赖 DWD goods_stock_summary 加载完成(需求 12.9
TaskDefinition("DWS_GOODS_STOCK_DAILY", "库存日报", "按日粒度汇总商品库存数据", "库存", "DWS"),
TaskDefinition("DWS_GOODS_STOCK_WEEKLY", "库存周报", "按周粒度汇总商品库存数据", "库存", "DWS"),
TaskDefinition("DWS_GOODS_STOCK_MONTHLY", "库存月报", "按月粒度汇总商品库存数据", "库存", "DWS"),
]
# ── INDEX 任务定义 ────────────────────────────────────────────
@@ -99,7 +103,8 @@ INDEX_TASKS: list[TaskDefinition] = [
TaskDefinition("DWS_WINBACK_INDEX", "回流指数 (WBI)", "计算会员回流指数", "指数", "INDEX"),
TaskDefinition("DWS_NEWCONV_INDEX", "新客转化指数 (NCI)", "计算新客转化指数", "指数", "INDEX"),
TaskDefinition("DWS_ML_MANUAL_IMPORT", "手动导入 (ML)", "手动导入机器学习数据", "指数", "INDEX", requires_window=False, is_common=False),
TaskDefinition("DWS_RELATION_INDEX", "关系指数 (RS)", "计算助教-客户关系指数", "指数", "INDEX"),
# CHANGE [2026-02-19] intent: 补充说明 RelationIndexTask 产出 RS/OS/MS/ML 四个子指数
TaskDefinition("DWS_RELATION_INDEX", "关系指数 (RS)", "产出 RS/OS/MS/ML 四个子指数", "指数", "INDEX"),
]
# ── 工具类任务定义 ────────────────────────────────────────────
@@ -210,6 +215,9 @@ DWD_TABLES: list[DwdTableDefinition] = [
DwdTableDefinition("dwd.dwd_payment", "支付流水", "结算", "ods.payment_transactions"),
DwdTableDefinition("dwd.dwd_refund", "退款流水", "结算", "ods.refund_transactions"),
DwdTableDefinition("dwd.dwd_refund_ex", "退款流水(扩展)", "结算", "ods.refund_transactions"),
# CHANGE [2026-07-20] intent: 同步 Task 6.1/6.2 新建的 DWD 库存表
DwdTableDefinition("dwd.dwd_goods_stock_summary", "库存汇总", "库存", "ods.goods_stock_summary"),
DwdTableDefinition("dwd.dwd_goods_stock_movement", "库存变动", "库存", "ods.goods_stock_movements"),
]