在前后端开发联调前 的提交20260223

This commit is contained in:
Neo
2026-02-23 23:02:20 +08:00
parent 254ccb1e77
commit fafc95e64c
1142 changed files with 10366960 additions and 36957 deletions

View File

@@ -12,7 +12,9 @@ from fastapi.middleware.cors import CORSMiddleware
from app import config
# CHANGE 2026-02-19 | 新增 xcx_test 路由MVP 验证)+ wx_callback 路由(微信消息推送)
from app.routers import auth, execution, schedules, tasks, env_config, db_viewer, etl_status, xcx_test, wx_callback
# CHANGE 2026-02-22 | 新增 member_birthday 路由(助教手动补录会员生日)
# CHANGE 2026-02-23 | 新增 ops_panel 路由(运维控制面板)
from app.routers import auth, execution, schedules, tasks, env_config, db_viewer, etl_status, xcx_test, wx_callback, member_birthday, ops_panel
from app.services.scheduler import scheduler
from app.services.task_queue import task_queue
from app.ws.logs import ws_router
@@ -60,6 +62,8 @@ app.include_router(etl_status.router)
app.include_router(ws_router)
app.include_router(xcx_test.router)
app.include_router(wx_callback.router)
app.include_router(member_birthday.router)
app.include_router(ops_panel.router)
@app.get("/health", tags=["系统"])

View File

@@ -0,0 +1,57 @@
"""
会员生日手动补录路由。
- POST /api/member-birthday — 助教提交会员生日UPSERT
"""
import logging
from fastapi import APIRouter, HTTPException, status
from app.database import get_connection
from app.schemas.member_birthday import MemberBirthdaySubmit
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api", tags=["会员生日"])
@router.post("/member-birthday")
async def submit_member_birthday(body: MemberBirthdaySubmit):
"""
助教提交会员生日UPSERT
同一 (member_id, assistant_id) 组合重复提交时,
更新 birthday_value 和 recorded_at保留其他助教的记录。
"""
sql = """
INSERT INTO member_birthday_manual
(member_id, birthday_value, recorded_by_assistant_id, recorded_by_name, site_id)
VALUES (%s, %s, %s, %s, %s)
ON CONFLICT (member_id, recorded_by_assistant_id)
DO UPDATE SET
birthday_value = EXCLUDED.birthday_value,
recorded_at = NOW()
"""
conn = get_connection()
try:
with conn.cursor() as cur:
cur.execute(sql, (
body.member_id,
body.birthday_value,
body.assistant_id,
body.assistant_name,
body.site_id,
))
conn.commit()
except Exception:
conn.rollback()
logger.exception("会员生日 UPSERT 失败: member_id=%s", body.member_id)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="生日提交失败,请稍后重试",
)
finally:
conn.close()
return {"status": "ok"}

View File

@@ -0,0 +1,372 @@
"""
运维控制面板 API
提供服务器各环境的服务状态查看、启停控制、Git 操作和配置管理。
面向管理后台的运维面板页面。
"""
import asyncio
import os
import subprocess
import platform
from datetime import datetime
from pathlib import Path
from typing import Any
import psutil
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
router = APIRouter(prefix="/api/ops", tags=["运维面板"])
# ---- 环境定义 ----
# 服务器上的两套环境;开发机上回退到本机路径(方便调试)
_SERVER_BASE = Path("D:/NeoZQYY")
ENVIRONMENTS: dict[str, dict[str, Any]] = {
"test": {
"label": "测试环境",
"repo_path": str(_SERVER_BASE / "test" / "repo"),
"branch": "test",
"port": 8001,
"bat_script": str(_SERVER_BASE / "scripts" / "start-test-api.bat"),
"window_title": "NeoZQYY Test API",
},
"prod": {
"label": "正式环境",
"repo_path": str(_SERVER_BASE / "prod" / "repo"),
"branch": "master",
"port": 8000,
"bat_script": str(_SERVER_BASE / "scripts" / "start-prod-api.bat"),
"window_title": "NeoZQYY Prod API",
},
}
# ---- 数据模型 ----
class ServiceStatus(BaseModel):
env: str
label: str
running: bool
pid: int | None = None
port: int
uptime_seconds: float | None = None
memory_mb: float | None = None
cpu_percent: float | None = None
class GitInfo(BaseModel):
env: str
branch: str
last_commit_hash: str
last_commit_message: str
last_commit_time: str
has_local_changes: bool
class SystemInfo(BaseModel):
cpu_percent: float
memory_total_gb: float
memory_used_gb: float
memory_percent: float
disk_total_gb: float
disk_used_gb: float
disk_percent: float
boot_time: str
class EnvFileContent(BaseModel):
content: str
class GitPullResult(BaseModel):
env: str
success: bool
output: str
class ServiceActionResult(BaseModel):
env: str
action: str
success: bool
message: str
# ---- 辅助函数 ----
def _find_uvicorn_process(port: int) -> psutil.Process | None:
"""查找监听指定端口的 uvicorn 进程。"""
for proc in psutil.process_iter(["pid", "name", "cmdline"]):
try:
cmdline = proc.info.get("cmdline") or []
cmdline_str = " ".join(cmdline)
# 匹配 uvicorn 进程且包含对应端口
if "uvicorn" in cmdline_str and str(port) in cmdline_str:
return proc
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
return None
def _run_cmd(cmd: str | list[str], cwd: str | None = None, timeout: int = 30) -> tuple[bool, str]:
"""执行命令并返回 (成功, 输出)。"""
try:
result = subprocess.run(
cmd,
cwd=cwd,
capture_output=True,
text=True,
timeout=timeout,
shell=isinstance(cmd, str),
encoding="utf-8",
errors="replace",
)
output = (result.stdout + "\n" + result.stderr).strip()
return result.returncode == 0, output
except subprocess.TimeoutExpired:
return False, "命令执行超时"
except Exception as e:
return False, str(e)
# ---- 系统信息 ----
@router.get("/system", response_model=SystemInfo)
async def get_system_info():
"""获取服务器系统资源概况。"""
mem = psutil.virtual_memory()
disk = psutil.disk_usage("D:\\") if os.path.exists("D:\\") else psutil.disk_usage("/")
boot = datetime.fromtimestamp(psutil.boot_time())
return SystemInfo(
cpu_percent=psutil.cpu_percent(interval=0.5),
memory_total_gb=round(mem.total / (1024 ** 3), 2),
memory_used_gb=round(mem.used / (1024 ** 3), 2),
memory_percent=mem.percent,
disk_total_gb=round(disk.total / (1024 ** 3), 2),
disk_used_gb=round(disk.used / (1024 ** 3), 2),
disk_percent=disk.percent,
boot_time=boot.isoformat(),
)
# ---- 服务状态 ----
@router.get("/services", response_model=list[ServiceStatus])
async def get_services_status():
"""获取所有环境的服务运行状态。"""
results = []
for env_key, env_cfg in ENVIRONMENTS.items():
proc = _find_uvicorn_process(env_cfg["port"])
if proc:
try:
mem_info = proc.memory_info()
create_time = proc.create_time()
results.append(ServiceStatus(
env=env_key,
label=env_cfg["label"],
running=True,
pid=proc.pid,
port=env_cfg["port"],
uptime_seconds=round(datetime.now().timestamp() - create_time, 1),
memory_mb=round(mem_info.rss / (1024 ** 2), 1),
cpu_percent=proc.cpu_percent(interval=0.1),
))
except (psutil.NoSuchProcess, psutil.AccessDenied):
results.append(ServiceStatus(
env=env_key, label=env_cfg["label"],
running=False, port=env_cfg["port"],
))
else:
results.append(ServiceStatus(
env=env_key, label=env_cfg["label"],
running=False, port=env_cfg["port"],
))
return results
# ---- 服务启停 ----
@router.post("/services/{env}/start", response_model=ServiceActionResult)
async def start_service(env: str):
"""启动指定环境的后端服务。"""
if env not in ENVIRONMENTS:
raise HTTPException(404, f"未知环境: {env}")
cfg = ENVIRONMENTS[env]
proc = _find_uvicorn_process(cfg["port"])
if proc:
return ServiceActionResult(
env=env, action="start", success=True,
message=f"服务已在运行中 (PID: {proc.pid})",
)
bat_path = cfg["bat_script"]
if not os.path.exists(bat_path):
raise HTTPException(400, f"启动脚本不存在: {bat_path}")
# 通过 start 命令在新窗口中启动 bat 脚本
try:
subprocess.Popen(
f'start "{cfg["window_title"]}" cmd /c "{bat_path}"',
shell=True,
)
# 等待进程启动
await asyncio.sleep(3)
new_proc = _find_uvicorn_process(cfg["port"])
if new_proc:
return ServiceActionResult(
env=env, action="start", success=True,
message=f"服务已启动 (PID: {new_proc.pid})",
)
else:
return ServiceActionResult(
env=env, action="start", success=False,
message="启动命令已执行,但未检测到进程,请检查日志",
)
except Exception as e:
return ServiceActionResult(
env=env, action="start", success=False, message=str(e),
)
@router.post("/services/{env}/stop", response_model=ServiceActionResult)
async def stop_service(env: str):
"""停止指定环境的后端服务。"""
if env not in ENVIRONMENTS:
raise HTTPException(404, f"未知环境: {env}")
cfg = ENVIRONMENTS[env]
proc = _find_uvicorn_process(cfg["port"])
if not proc:
return ServiceActionResult(
env=env, action="stop", success=True, message="服务未在运行",
)
try:
# 终止进程树(包括子进程)
parent = psutil.Process(proc.pid)
children = parent.children(recursive=True)
for child in children:
child.terminate()
parent.terminate()
# 等待进程退出
gone, alive = psutil.wait_procs([parent] + children, timeout=5)
for p in alive:
p.kill()
return ServiceActionResult(
env=env, action="stop", success=True, message="服务已停止",
)
except Exception as e:
return ServiceActionResult(
env=env, action="stop", success=False, message=str(e),
)
@router.post("/services/{env}/restart", response_model=ServiceActionResult)
async def restart_service(env: str):
"""重启指定环境的后端服务。"""
stop_result = await stop_service(env)
if not stop_result.success and "未在运行" not in stop_result.message:
return ServiceActionResult(
env=env, action="restart", success=False,
message=f"停止失败: {stop_result.message}",
)
await asyncio.sleep(1)
start_result = await start_service(env)
return ServiceActionResult(
env=env, action="restart",
success=start_result.success,
message=start_result.message,
)
# ---- Git 操作 ----
@router.get("/git", response_model=list[GitInfo])
async def get_git_info():
"""获取所有环境的 Git 状态。"""
results = []
for env_key, env_cfg in ENVIRONMENTS.items():
repo = env_cfg["repo_path"]
if not os.path.isdir(os.path.join(repo, ".git")):
results.append(GitInfo(
env=env_key, branch="N/A",
last_commit_hash="N/A", last_commit_message="仓库不存在",
last_commit_time="", has_local_changes=False,
))
continue
_, branch = _run_cmd(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=repo)
_, log_out = _run_cmd(
["git", "log", "-1", "--format=%H|%s|%ci"],
cwd=repo,
)
_, status_out = _run_cmd(["git", "status", "--porcelain"], cwd=repo)
parts = log_out.strip().split("|", 2) if log_out else ["", "", ""]
results.append(GitInfo(
env=env_key,
branch=branch.strip(),
last_commit_hash=parts[0][:8] if parts[0] else "N/A",
last_commit_message=parts[1] if len(parts) > 1 else "",
last_commit_time=parts[2] if len(parts) > 2 else "",
has_local_changes=bool(status_out.strip()),
))
return results
@router.post("/git/{env}/pull", response_model=GitPullResult)
async def git_pull(env: str):
"""对指定环境执行 git pull。"""
if env not in ENVIRONMENTS:
raise HTTPException(404, f"未知环境: {env}")
cfg = ENVIRONMENTS[env]
repo = cfg["repo_path"]
if not os.path.isdir(os.path.join(repo, ".git")):
raise HTTPException(400, f"仓库路径不存在: {repo}")
success, output = _run_cmd(["git", "pull", "--ff-only"], cwd=repo, timeout=60)
return GitPullResult(env=env, success=success, output=output)
@router.post("/git/{env}/sync-deps", response_model=ServiceActionResult)
async def sync_deps(env: str):
"""对指定环境执行 uv sync --all-packages。"""
if env not in ENVIRONMENTS:
raise HTTPException(404, f"未知环境: {env}")
cfg = ENVIRONMENTS[env]
repo = cfg["repo_path"]
success, output = _run_cmd(["uv", "sync", "--all-packages"], cwd=repo, timeout=120)
return ServiceActionResult(
env=env, action="sync-deps", success=success, message=output[:500],
)
# ---- 环境配置管理 ----
@router.get("/env-file/{env}")
async def get_env_file(env: str):
"""读取指定环境的 .env 文件(敏感值脱敏)。"""
if env not in ENVIRONMENTS:
raise HTTPException(404, f"未知环境: {env}")
env_path = Path(ENVIRONMENTS[env]["repo_path"]) / ".env"
if not env_path.exists():
raise HTTPException(404, f".env 文件不存在: {env_path}")
lines = env_path.read_text(encoding="utf-8").splitlines()
masked_lines = []
sensitive_keys = {"PASSWORD", "SECRET", "TOKEN", "DSN", "APP_SECRET"}
for line in lines:
stripped = line.strip()
if stripped and not stripped.startswith("#") and "=" in stripped:
key = stripped.split("=", 1)[0].strip()
if any(s in key.upper() for s in sensitive_keys):
masked_lines.append(f"{key}=********")
continue
masked_lines.append(line)
return {"env": env, "content": "\n".join(masked_lines)}

View File

@@ -4,7 +4,7 @@
提供 4 个端点:
- GET /api/tasks/registry — 按业务域分组的任务列表
- GET /api/tasks/dwd-tables — 按业务域分组的 DWD 表定义
- GET /api/tasks/flows — 7 种 Flow + 3 种处理模式
- GET /api/tasks/flows — 7 种 Flow + 4 种处理模式
- POST /api/tasks/validate — 验证 TaskConfig 并返回 CLI 命令预览
所有端点需要 JWT 认证。validate 端点从 JWT 注入 store_id。
@@ -103,6 +103,7 @@ PROCESSING_MODE_DEFINITIONS: list[ProcessingModeDefinition] = [
ProcessingModeDefinition(id="increment_only", name="仅增量处理", description="只处理新增和变更的数据"),
ProcessingModeDefinition(id="verify_only", name="仅校验修复", description="校验现有数据并修复不一致"),
ProcessingModeDefinition(id="increment_verify", name="增量 + 校验修复", description="先增量处理,再校验并修复"),
ProcessingModeDefinition(id="full_window", name="全窗口处理", description="用 API 返回数据的实际时间范围处理全部层,无需校验"),
]
@@ -163,7 +164,7 @@ async def get_dwd_tables(
async def get_flows(
user: CurrentUser = Depends(get_current_user),
) -> FlowsResponse:
"""返回 7 种 Flow 定义和 3 种处理模式定义"""
"""返回 7 种 Flow 定义和 4 种处理模式定义"""
return FlowsResponse(
flows=FLOW_DEFINITIONS,
processing_modes=PROCESSING_MODE_DEFINITIONS,
@@ -183,8 +184,9 @@ async def validate_task_config(
errors: list[str] = []
# 验证 Flow ID
if config.pipeline not in FLOW_LAYER_MAP:
errors.append(f"无效的执行流程: {config.pipeline}")
# CHANGE [2026-02-20] intent: pipeline → flow统一命名
if config.flow not in FLOW_LAYER_MAP:
errors.append(f"无效的执行流程: {config.flow}")
# 验证任务列表非空
if not config.tasks:

View File

@@ -0,0 +1,19 @@
"""
会员生日手动补录相关 Pydantic 模型。
- MemberBirthdaySubmit助教提交会员生日请求体
"""
from datetime import date
from pydantic import BaseModel, Field
class MemberBirthdaySubmit(BaseModel):
"""助教提交会员生日请求。"""
member_id: int = Field(..., gt=0, description="会员 ID")
birthday_value: date = Field(..., description="生日日期")
assistant_id: int = Field(..., gt=0, description="助教 ID")
assistant_name: str = Field(..., min_length=1, max_length=50, description="助教姓名")
site_id: int = Field(..., gt=0, description="门店 ID")

View File

@@ -13,8 +13,8 @@ class TaskConfigSchema(BaseModel):
"""任务配置 — 前后端传输格式
字段与 CLI 参数的映射关系:
- pipeline → --pipelineFlow ID7 种之一)
- processing_mode → --processing-mode3 种处理模式)
- flow → --flowFlow ID7 种之一)
- processing_mode → --processing-mode4 种处理模式)
- tasks → --tasks逗号分隔
- dry_run → --dry-run布尔标志
- window_mode → 决定使用 lookback 还是 custom 时间窗口(仅前端逻辑,不直接映射 CLI 参数)
@@ -30,7 +30,8 @@ class TaskConfigSchema(BaseModel):
"""
tasks: list[str]
pipeline: str = "api_ods_dwd"
# CHANGE [2026-02-20] intent: pipeline → flow统一命名消除历史别名
flow: str = "api_ods_dwd"
processing_mode: str = "increment_only"
dry_run: bool = False
window_mode: str = "lookback"

View File

@@ -6,7 +6,7 @@
支持:
- 7 种 Flowapi_ods / api_ods_dwd / api_full / ods_dwd / dwd_dws / dwd_dws_index / dwd_index
- 3 种处理模式increment_only / verify_only / increment_verify
- 4 种处理模式increment_only / verify_only / increment_verify / full_window
- 自动注入 --store-id 参数
"""
@@ -30,6 +30,7 @@ VALID_PROCESSING_MODES: set[str] = {
"increment_only",
"verify_only",
"increment_verify",
"full_window",
}
# CLI 支持的 extra_args 键(值类型 + 布尔类型)
@@ -72,7 +73,8 @@ class CLIBuilder:
cmd: list[str] = [python_executable, "-m", "cli.main"]
# -- Flow执行流程 --
cmd.extend(["--flow", config.pipeline])
# CHANGE [2026-02-20] intent: pipeline → flow统一命名
cmd.extend(["--flow", config.flow])
# -- 处理模式 --
if config.processing_mode:

View File

@@ -46,7 +46,7 @@ ODS_TASKS: list[TaskDefinition] = [
TaskDefinition("ODS_ASSISTANT_LEDGER", "助教服务记录", "抽取助教服务流水", "助教", "ODS", is_ods=True),
TaskDefinition("ODS_ASSISTANT_ABOLISH", "助教取消记录", "抽取助教取消/作废记录", "助教", "ODS", is_ods=True),
TaskDefinition("ODS_SETTLEMENT_RECORDS", "结算记录", "抽取订单结算记录", "结算", "ODS", is_ods=True),
TaskDefinition("ODS_SETTLEMENT_TICKET", "结账小票", "抽取结账小票明细", "结算", "ODS", is_ods=True),
# CHANGE [2026-07-20] intent: 同步 ETL 侧移除——ODS_SETTLEMENT_TICKET 已在 Task 7.3 中彻底移除
TaskDefinition("ODS_TABLE_USE", "台费流水", "抽取台费使用流水", "台桌", "ODS", is_ods=True),
TaskDefinition("ODS_TABLE_FEE_DISCOUNT", "台费折扣", "抽取台费折扣记录", "台桌", "ODS", is_ods=True),
TaskDefinition("ODS_TABLES", "台桌主数据", "抽取门店台桌信息", "台桌", "ODS", is_ods=True, requires_window=False),
@@ -59,7 +59,7 @@ ODS_TASKS: list[TaskDefinition] = [
TaskDefinition("ODS_RECHARGE_SETTLE", "充值结算", "抽取充值结算记录", "会员", "ODS", is_ods=True),
TaskDefinition("ODS_GROUP_PACKAGE", "团购套餐", "抽取团购套餐定义", "团购", "ODS", is_ods=True, requires_window=False),
TaskDefinition("ODS_GROUP_BUY_REDEMPTION", "团购核销", "抽取团购核销记录", "团购", "ODS", is_ods=True),
TaskDefinition("ODS_INVENTORY_STOCK", "库存快照", "抽取商品库存汇总", "库存", "ODS", is_ods=True, requires_window=False),
TaskDefinition("ODS_INVENTORY_STOCK", "库存快照", "抽取商品库存汇总", "库存", "ODS", is_ods=True),
TaskDefinition("ODS_INVENTORY_CHANGE", "库存变动", "抽取库存出入库记录", "库存", "ODS", is_ods=True),
TaskDefinition("ODS_GOODS_CATEGORY", "商品分类", "抽取商品分类树", "商品", "ODS", is_ods=True, requires_window=False),
TaskDefinition("ODS_STORE_GOODS", "门店商品", "抽取门店商品主数据", "商品", "ODS", is_ods=True, requires_window=False),
@@ -91,6 +91,10 @@ DWS_TASKS: list[TaskDefinition] = [
TaskDefinition("DWS_FINANCE_DISCOUNT_DETAIL", "折扣明细", "汇总折扣明细", "财务", "DWS"),
# CHANGE [2026-02-19] intent: 同步 ETL 侧合并——原 DWS_RETENTION_CLEANUP / DWS_MV_REFRESH_* 已合并为 DWS_MAINTENANCE
TaskDefinition("DWS_MAINTENANCE", "DWS 维护", "刷新物化视图 + 清理过期留存数据", "通用", "DWS", requires_window=False, is_common=False),
# CHANGE [2026-07-20] intent: 注册 DWS 库存汇总任务(日/周/月),依赖 DWD goods_stock_summary 加载完成(需求 12.9
TaskDefinition("DWS_GOODS_STOCK_DAILY", "库存日报", "按日粒度汇总商品库存数据", "库存", "DWS"),
TaskDefinition("DWS_GOODS_STOCK_WEEKLY", "库存周报", "按周粒度汇总商品库存数据", "库存", "DWS"),
TaskDefinition("DWS_GOODS_STOCK_MONTHLY", "库存月报", "按月粒度汇总商品库存数据", "库存", "DWS"),
]
# ── INDEX 任务定义 ────────────────────────────────────────────
@@ -99,7 +103,8 @@ INDEX_TASKS: list[TaskDefinition] = [
TaskDefinition("DWS_WINBACK_INDEX", "回流指数 (WBI)", "计算会员回流指数", "指数", "INDEX"),
TaskDefinition("DWS_NEWCONV_INDEX", "新客转化指数 (NCI)", "计算新客转化指数", "指数", "INDEX"),
TaskDefinition("DWS_ML_MANUAL_IMPORT", "手动导入 (ML)", "手动导入机器学习数据", "指数", "INDEX", requires_window=False, is_common=False),
TaskDefinition("DWS_RELATION_INDEX", "关系指数 (RS)", "计算助教-客户关系指数", "指数", "INDEX"),
# CHANGE [2026-02-19] intent: 补充说明 RelationIndexTask 产出 RS/OS/MS/ML 四个子指数
TaskDefinition("DWS_RELATION_INDEX", "关系指数 (RS)", "产出 RS/OS/MS/ML 四个子指数", "指数", "INDEX"),
]
# ── 工具类任务定义 ────────────────────────────────────────────
@@ -210,6 +215,9 @@ DWD_TABLES: list[DwdTableDefinition] = [
DwdTableDefinition("dwd.dwd_payment", "支付流水", "结算", "ods.payment_transactions"),
DwdTableDefinition("dwd.dwd_refund", "退款流水", "结算", "ods.refund_transactions"),
DwdTableDefinition("dwd.dwd_refund_ex", "退款流水(扩展)", "结算", "ods.refund_transactions"),
# CHANGE [2026-07-20] intent: 同步 Task 6.1/6.2 新建的 DWD 库存表
DwdTableDefinition("dwd.dwd_goods_stock_summary", "库存汇总", "库存", "ods.goods_stock_summary"),
DwdTableDefinition("dwd.dwd_goods_stock_movement", "库存变动", "库存", "ods.goods_stock_movements"),
]

View File

@@ -16,6 +16,7 @@ dependencies = [
"python-dotenv>=1.0",
"python-jose[cryptography]>=3.3",
"bcrypt>=4.0",
"psutil>=5.9",
]
[tool.uv.sources]

View File

@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""CLIBuilder 单元测试
覆盖7 种 Flow、3 种处理模式、时间窗口、store_id 自动注入、extra_args 等。
覆盖7 种 Flow、4 种处理模式、时间窗口、store_id 自动注入、extra_args 等。
"""
import pytest
@@ -24,11 +24,11 @@ ETL_PATH = "/fake/etl/project"
class TestBasicCommand:
def test_minimal_command(self, builder: CLIBuilder):
"""最小配置应生成 python -m cli.main --pipeline ... --processing-mode ..."""
"""最小配置应生成 python -m cli.main --flow ... --processing-mode ..."""
config = TaskConfigSchema(tasks=["ODS_MEMBER"])
cmd = builder.build_command(config, ETL_PATH)
assert cmd[:3] == ["python", "-m", "cli.main"]
assert "--pipeline" in cmd
assert "--flow" in cmd
assert "--processing-mode" in cmd
def test_custom_python_executable(self, builder: CLIBuilder):
@@ -56,20 +56,20 @@ class TestBasicCommand:
class TestFlows:
@pytest.mark.parametrize("flow_id", sorted(VALID_FLOWS))
def test_all_flows_accepted(self, builder: CLIBuilder, flow_id: str):
config = TaskConfigSchema(tasks=["ODS_MEMBER"], pipeline=flow_id)
config = TaskConfigSchema(tasks=["ODS_MEMBER"], flow=flow_id)
cmd = builder.build_command(config, ETL_PATH)
idx = cmd.index("--pipeline")
idx = cmd.index("--flow")
assert cmd[idx + 1] == flow_id
def test_default_flow_is_api_ods_dwd(self, builder: CLIBuilder):
config = TaskConfigSchema(tasks=["ODS_MEMBER"])
cmd = builder.build_command(config, ETL_PATH)
idx = cmd.index("--pipeline")
idx = cmd.index("--flow")
assert cmd[idx + 1] == "api_ods_dwd"
# ---------------------------------------------------------------------------
# 3 种处理模式
# 4 种处理模式
# ---------------------------------------------------------------------------
class TestProcessingModes:

View File

@@ -36,7 +36,7 @@ _NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
# 构造测试用的 TaskConfig payload
_VALID_CONFIG = {
"tasks": ["ODS_MEMBER"],
"pipeline": "api_ods",
"flow": "api_ods",
}
@@ -74,7 +74,7 @@ class TestRunTask:
def test_run_invalid_config_returns_422(self):
"""缺少必填字段 tasks 时返回 422"""
resp = client.post("/api/execution/run", json={"pipeline": "api_ods"})
resp = client.post("/api/execution/run", json={"flow": "api_ods"})
assert resp.status_code == 422

View File

@@ -40,7 +40,7 @@ _task_codes = ["ODS_MEMBER", "ODS_PAYMENT", "ODS_ORDER", "DWD_LOAD_FROM_ODS", "D
_simple_config_st = st.builds(
TaskConfigSchema,
tasks=st.lists(st.sampled_from(_task_codes), min_size=1, max_size=3, unique=True),
pipeline=st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd"]),
flow=st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd"]),
)
@@ -254,7 +254,7 @@ def test_queue_crud_invariant(mock_get_conn, config, site_id, initial_count):
db.rows[tid] = {
"id": tid,
"site_id": site_id,
"config": {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"},
"config": {"tasks": ["ODS_MEMBER"], "flow": "api_ods"},
"status": "pending",
"position": i + 1,
}
@@ -322,7 +322,7 @@ def test_queue_dequeue_order(mock_get_conn, site_id, num_tasks, positions):
db.rows[tid] = {
"id": tid,
"site_id": site_id,
"config": {"tasks": [_task_codes[i % len(_task_codes)]], "pipeline": "api_ods"},
"config": {"tasks": [_task_codes[i % len(_task_codes)]], "flow": "api_ods"},
"status": "pending",
"position": pos,
}
@@ -372,7 +372,7 @@ def test_queue_reorder_consistency(mock_get_conn, site_id, num_tasks, data):
db.rows[tid] = {
"id": tid,
"site_id": site_id,
"config": {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"},
"config": {"tasks": ["ODS_MEMBER"], "flow": "api_ods"},
"status": "pending",
"position": i + 1,
}

View File

@@ -42,7 +42,7 @@ _task_codes = ["ODS_MEMBER", "ODS_PAYMENT", "ODS_ORDER", "DWD_LOAD_FROM_ODS", "D
_simple_task_config_st = st.fixed_dictionaries({
"tasks": st.lists(st.sampled_from(_task_codes), min_size=1, max_size=3, unique=True),
"pipeline": st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd", "api_full"]),
"flow": st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd", "api_full"]),
})
# 调度配置策略:覆盖 5 种调度类型
@@ -324,8 +324,8 @@ def test_due_schedule_auto_enqueue(
assert enqueued_config.tasks == task_config["tasks"], (
f"入队的 tasks 应为 {task_config['tasks']},实际 {enqueued_config.tasks}"
)
assert enqueued_config.pipeline == task_config["pipeline"], (
f"入队的 pipeline 应为 {task_config['pipeline']},实际 {enqueued_config.pipeline}"
assert enqueued_config.flow == task_config["flow"], (
f"入队的 flow 应为 {task_config['flow']},实际 {enqueued_config.flow}"
)

View File

@@ -230,7 +230,7 @@ class TestCheckAndEnqueue:
@patch("app.services.scheduler.task_queue")
def test_enqueues_due_tasks(self, mock_tq, mock_get_conn, sched):
"""到期任务应被入队,且更新 last_run_at / run_count / next_run_at"""
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
schedule_config = {
"schedule_type": "interval",
"interval_value": 1,
@@ -280,7 +280,7 @@ class TestCheckAndEnqueue:
def test_skips_invalid_config(self, mock_tq, mock_get_conn, sched):
"""配置反序列化失败的任务应被跳过"""
# task_config 缺少必填字段 tasks
bad_config = {"pipeline": "api_ods_dwd"}
bad_config = {"flow": "api_ods_dwd"}
schedule_config = {"schedule_type": "once"}
cur = _mock_cursor(
@@ -300,7 +300,7 @@ class TestCheckAndEnqueue:
@patch("app.services.scheduler.task_queue")
def test_enqueue_failure_continues(self, mock_tq, mock_get_conn, sched):
"""入队失败时应跳过该任务,继续处理后续任务"""
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
schedule_config = {"schedule_type": "once"}
cur = _mock_cursor(
@@ -327,7 +327,7 @@ class TestCheckAndEnqueue:
@patch("app.services.scheduler.task_queue")
def test_once_type_sets_next_run_none(self, mock_tq, mock_get_conn, sched):
"""once 类型任务入队后next_run_at 应被设为 NULL"""
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
schedule_config = {"schedule_type": "once"}
select_cur = _mock_cursor(

View File

@@ -40,14 +40,14 @@ _SCHEDULE_CONFIG = {
_VALID_CREATE = {
"name": "每日全量同步",
"task_codes": ["ODS_MEMBER", "ODS_ORDER"],
"task_config": {"tasks": ["ODS_MEMBER", "ODS_ORDER"], "pipeline": "api_ods"},
"task_config": {"tasks": ["ODS_MEMBER", "ODS_ORDER"], "flow": "api_ods"},
"schedule_config": _SCHEDULE_CONFIG,
}
# 模拟数据库返回的完整行13 列,与 _SELECT_COLS 对应)
_DB_ROW = (
"sched-1", 100, "每日全量同步", ["ODS_MEMBER", "ODS_ORDER"],
json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}),
json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"}),
json.dumps(_SCHEDULE_CONFIG),
True, None, _NEXT, 0, None, _NOW, _NOW,
)

View File

@@ -53,7 +53,7 @@ def _make_queue_rows(site_id: int, count: int) -> list[tuple]:
rows.append((
str(uuid.uuid4()), # id
site_id, # site_id
json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}), # config
json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"}), # config
"pending", # status
i + 1, # position
datetime(2024, 1, 1, tzinfo=timezone.utc), # created_at
@@ -75,7 +75,7 @@ def _make_schedule_rows(site_id: int, count: int) -> list[tuple]:
site_id, # site_id
f"调度任务_{i}", # name
["ODS_MEMBER"], # task_codes
{"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}, # task_config
{"tasks": ["ODS_MEMBER"], "flow": "api_ods"}, # task_config
{"schedule_type": "daily", "daily_time": "04:00", # schedule_config
"interval_value": 1, "interval_unit": "hours",
"weekly_days": [1], "weekly_time": "04:00",

View File

@@ -31,7 +31,7 @@ _tasks_st = st.lists(
unique=True,
)
_pipeline_st = st.sampled_from(sorted(VALID_FLOWS))
_flow_st = st.sampled_from(sorted(VALID_FLOWS))
_processing_mode_st = st.sampled_from(sorted(VALID_PROCESSING_MODES))
_window_mode_st = st.sampled_from(["lookback", "custom"])
@@ -69,7 +69,7 @@ def _valid_task_config_st():
@st.composite
def _build(draw):
tasks = draw(_tasks_st)
pipeline = draw(_pipeline_st)
flow_id = draw(_flow_st)
processing_mode = draw(_processing_mode_st)
dry_run = draw(st.booleans())
window_mode = draw(_window_mode_st)
@@ -103,7 +103,7 @@ def _valid_task_config_st():
return TaskConfigSchema(
tasks=tasks,
pipeline=pipeline,
flow=flow_id,
processing_mode=processing_mode,
dry_run=dry_run,
window_mode=window_mode,
@@ -204,10 +204,10 @@ def test_task_config_to_cli_completeness(config: TaskConfigSchema):
"""Property 7: CLIBuilder 生成的命令应包含 TaskConfig 中所有非空字段对应的 CLI 参数。"""
cmd = _builder.build_command(config, _ETL_PATH)
# 1) --pipeline 始终存在且值正确
assert "--pipeline" in cmd
idx = cmd.index("--pipeline")
assert cmd[idx + 1] == config.pipeline
# 1) --flow 始终存在且值正确
assert "--flow" in cmd
idx = cmd.index("--flow")
assert cmd[idx + 1] == config.flow
# 2) --processing-mode 始终存在且值正确
assert "--processing-mode" in cmd

View File

@@ -24,7 +24,7 @@ def executor() -> TaskExecutor:
def sample_config() -> TaskConfigSchema:
return TaskConfigSchema(
tasks=["ODS_MEMBER", "ODS_PAYMENT"],
pipeline="api_ods_dwd",
flow="api_ods_dwd",
store_id=42,
)

View File

@@ -25,7 +25,7 @@ def queue() -> TaskQueue:
def sample_config() -> TaskConfigSchema:
return TaskConfigSchema(
tasks=["ODS_MEMBER", "ODS_PAYMENT"],
pipeline="api_ods_dwd",
flow="api_ods_dwd",
store_id=42,
)
@@ -107,7 +107,7 @@ class TestEnqueue:
config_json_str = insert_call[0][1][2]
parsed = json.loads(config_json_str)
assert parsed["tasks"] == ["ODS_MEMBER", "ODS_PAYMENT"]
assert parsed["pipeline"] == "api_ods_dwd"
assert parsed["flow"] == "api_ods_dwd"
# ---------------------------------------------------------------------------
@@ -129,7 +129,7 @@ class TestDequeue:
@patch("app.services.task_queue.get_connection")
def test_dequeue_returns_task(self, mock_get_conn, queue):
task_id = str(uuid.uuid4())
config_dict = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}
config_dict = {"tasks": ["ODS_MEMBER"], "flow": "api_ods"}
row = (
task_id, 42, json.dumps(config_dict), "pending", 1,
None, None, None, None, None,
@@ -149,7 +149,7 @@ class TestDequeue:
@patch("app.services.task_queue.get_connection")
def test_dequeue_updates_status_to_running(self, mock_get_conn, queue):
task_id = str(uuid.uuid4())
config_dict = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}
config_dict = {"tasks": ["ODS_MEMBER"], "flow": "api_ods"}
row = (
task_id, 42, json.dumps(config_dict), "pending", 1,
None, None, None, None, None,
@@ -285,7 +285,7 @@ class TestQuery:
@patch("app.services.task_queue.get_connection")
def test_list_pending_returns_tasks(self, mock_get_conn, queue):
tid = str(uuid.uuid4())
config = json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"})
config = json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"})
rows = [(tid, 42, config, "pending", 1, None, None, None, None, None)]
cur = _mock_cursor(fetchall_val=rows)
conn = _mock_conn(cur)
@@ -353,7 +353,7 @@ class TestProcessLoop:
task_id = str(uuid.uuid4())
config_dict = {
"tasks": ["ODS_MEMBER"],
"pipeline": "api_ods_dwd",
"flow": "api_ods_dwd",
"processing_mode": "increment_only",
"dry_run": False,
"window_mode": "lookback",

View File

@@ -152,7 +152,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["ODS_MEMBER", "ODS_PAYMENT"],
"pipeline": "api_ods",
"flow": "api_ods",
}
})
assert resp.status_code == 200
@@ -169,7 +169,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["DWD_LOAD_FROM_ODS"],
"pipeline": "ods_dwd",
"flow": "ods_dwd",
"store_id": 999,
}
})
@@ -184,7 +184,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["ODS_MEMBER"],
"pipeline": "nonexistent_flow",
"flow": "nonexistent_flow",
}
})
assert resp.status_code == 200
@@ -196,7 +196,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": [],
"pipeline": "api_ods",
"flow": "api_ods",
}
})
assert resp.status_code == 200
@@ -208,7 +208,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["ODS_MEMBER"],
"pipeline": "api_ods",
"flow": "api_ods",
"window_mode": "custom",
"window_start": "2024-01-01",
"window_end": "2024-01-31",
@@ -225,7 +225,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["ODS_MEMBER"],
"pipeline": "api_ods",
"flow": "api_ods",
"window_mode": "custom",
"window_start": "2024-12-31",
"window_end": "2024-01-01",
@@ -237,7 +237,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["ODS_MEMBER"],
"pipeline": "api_ods",
"flow": "api_ods",
"dry_run": True,
}
})