在前后端开发联调前 的提交20260223
This commit is contained in:
@@ -12,7 +12,9 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app import config
|
||||
# CHANGE 2026-02-19 | 新增 xcx_test 路由(MVP 验证)+ wx_callback 路由(微信消息推送)
|
||||
from app.routers import auth, execution, schedules, tasks, env_config, db_viewer, etl_status, xcx_test, wx_callback
|
||||
# CHANGE 2026-02-22 | 新增 member_birthday 路由(助教手动补录会员生日)
|
||||
# CHANGE 2026-02-23 | 新增 ops_panel 路由(运维控制面板)
|
||||
from app.routers import auth, execution, schedules, tasks, env_config, db_viewer, etl_status, xcx_test, wx_callback, member_birthday, ops_panel
|
||||
from app.services.scheduler import scheduler
|
||||
from app.services.task_queue import task_queue
|
||||
from app.ws.logs import ws_router
|
||||
@@ -60,6 +62,8 @@ app.include_router(etl_status.router)
|
||||
app.include_router(ws_router)
|
||||
app.include_router(xcx_test.router)
|
||||
app.include_router(wx_callback.router)
|
||||
app.include_router(member_birthday.router)
|
||||
app.include_router(ops_panel.router)
|
||||
|
||||
|
||||
@app.get("/health", tags=["系统"])
|
||||
|
||||
57
apps/backend/app/routers/member_birthday.py
Normal file
57
apps/backend/app/routers/member_birthday.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""
|
||||
会员生日手动补录路由。
|
||||
|
||||
- POST /api/member-birthday — 助教提交会员生日(UPSERT)
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
|
||||
from app.database import get_connection
|
||||
from app.schemas.member_birthday import MemberBirthdaySubmit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["会员生日"])
|
||||
|
||||
|
||||
@router.post("/member-birthday")
|
||||
async def submit_member_birthday(body: MemberBirthdaySubmit):
|
||||
"""
|
||||
助教提交会员生日(UPSERT)。
|
||||
|
||||
同一 (member_id, assistant_id) 组合重复提交时,
|
||||
更新 birthday_value 和 recorded_at,保留其他助教的记录。
|
||||
"""
|
||||
sql = """
|
||||
INSERT INTO member_birthday_manual
|
||||
(member_id, birthday_value, recorded_by_assistant_id, recorded_by_name, site_id)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
ON CONFLICT (member_id, recorded_by_assistant_id)
|
||||
DO UPDATE SET
|
||||
birthday_value = EXCLUDED.birthday_value,
|
||||
recorded_at = NOW()
|
||||
"""
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (
|
||||
body.member_id,
|
||||
body.birthday_value,
|
||||
body.assistant_id,
|
||||
body.assistant_name,
|
||||
body.site_id,
|
||||
))
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
logger.exception("会员生日 UPSERT 失败: member_id=%s", body.member_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="生日提交失败,请稍后重试",
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return {"status": "ok"}
|
||||
372
apps/backend/app/routers/ops_panel.py
Normal file
372
apps/backend/app/routers/ops_panel.py
Normal file
@@ -0,0 +1,372 @@
|
||||
"""
|
||||
运维控制面板 API
|
||||
|
||||
提供服务器各环境的服务状态查看、启停控制、Git 操作和配置管理。
|
||||
面向管理后台的运维面板页面。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import subprocess
|
||||
import platform
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import psutil
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
router = APIRouter(prefix="/api/ops", tags=["运维面板"])
|
||||
|
||||
# ---- 环境定义 ----
|
||||
# 服务器上的两套环境;开发机上回退到本机路径(方便调试)
|
||||
|
||||
_SERVER_BASE = Path("D:/NeoZQYY")
|
||||
|
||||
ENVIRONMENTS: dict[str, dict[str, Any]] = {
|
||||
"test": {
|
||||
"label": "测试环境",
|
||||
"repo_path": str(_SERVER_BASE / "test" / "repo"),
|
||||
"branch": "test",
|
||||
"port": 8001,
|
||||
"bat_script": str(_SERVER_BASE / "scripts" / "start-test-api.bat"),
|
||||
"window_title": "NeoZQYY Test API",
|
||||
},
|
||||
"prod": {
|
||||
"label": "正式环境",
|
||||
"repo_path": str(_SERVER_BASE / "prod" / "repo"),
|
||||
"branch": "master",
|
||||
"port": 8000,
|
||||
"bat_script": str(_SERVER_BASE / "scripts" / "start-prod-api.bat"),
|
||||
"window_title": "NeoZQYY Prod API",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---- 数据模型 ----
|
||||
|
||||
class ServiceStatus(BaseModel):
|
||||
env: str
|
||||
label: str
|
||||
running: bool
|
||||
pid: int | None = None
|
||||
port: int
|
||||
uptime_seconds: float | None = None
|
||||
memory_mb: float | None = None
|
||||
cpu_percent: float | None = None
|
||||
|
||||
|
||||
class GitInfo(BaseModel):
|
||||
env: str
|
||||
branch: str
|
||||
last_commit_hash: str
|
||||
last_commit_message: str
|
||||
last_commit_time: str
|
||||
has_local_changes: bool
|
||||
|
||||
|
||||
class SystemInfo(BaseModel):
|
||||
cpu_percent: float
|
||||
memory_total_gb: float
|
||||
memory_used_gb: float
|
||||
memory_percent: float
|
||||
disk_total_gb: float
|
||||
disk_used_gb: float
|
||||
disk_percent: float
|
||||
boot_time: str
|
||||
|
||||
|
||||
class EnvFileContent(BaseModel):
|
||||
content: str
|
||||
|
||||
|
||||
class GitPullResult(BaseModel):
|
||||
env: str
|
||||
success: bool
|
||||
output: str
|
||||
|
||||
|
||||
class ServiceActionResult(BaseModel):
|
||||
env: str
|
||||
action: str
|
||||
success: bool
|
||||
message: str
|
||||
|
||||
|
||||
# ---- 辅助函数 ----
|
||||
|
||||
def _find_uvicorn_process(port: int) -> psutil.Process | None:
|
||||
"""查找监听指定端口的 uvicorn 进程。"""
|
||||
for proc in psutil.process_iter(["pid", "name", "cmdline"]):
|
||||
try:
|
||||
cmdline = proc.info.get("cmdline") or []
|
||||
cmdline_str = " ".join(cmdline)
|
||||
# 匹配 uvicorn 进程且包含对应端口
|
||||
if "uvicorn" in cmdline_str and str(port) in cmdline_str:
|
||||
return proc
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _run_cmd(cmd: str | list[str], cwd: str | None = None, timeout: int = 30) -> tuple[bool, str]:
|
||||
"""执行命令并返回 (成功, 输出)。"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
shell=isinstance(cmd, str),
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
output = (result.stdout + "\n" + result.stderr).strip()
|
||||
return result.returncode == 0, output
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "命令执行超时"
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
# ---- 系统信息 ----
|
||||
|
||||
@router.get("/system", response_model=SystemInfo)
|
||||
async def get_system_info():
|
||||
"""获取服务器系统资源概况。"""
|
||||
mem = psutil.virtual_memory()
|
||||
disk = psutil.disk_usage("D:\\") if os.path.exists("D:\\") else psutil.disk_usage("/")
|
||||
boot = datetime.fromtimestamp(psutil.boot_time())
|
||||
return SystemInfo(
|
||||
cpu_percent=psutil.cpu_percent(interval=0.5),
|
||||
memory_total_gb=round(mem.total / (1024 ** 3), 2),
|
||||
memory_used_gb=round(mem.used / (1024 ** 3), 2),
|
||||
memory_percent=mem.percent,
|
||||
disk_total_gb=round(disk.total / (1024 ** 3), 2),
|
||||
disk_used_gb=round(disk.used / (1024 ** 3), 2),
|
||||
disk_percent=disk.percent,
|
||||
boot_time=boot.isoformat(),
|
||||
)
|
||||
|
||||
|
||||
# ---- 服务状态 ----
|
||||
|
||||
@router.get("/services", response_model=list[ServiceStatus])
|
||||
async def get_services_status():
|
||||
"""获取所有环境的服务运行状态。"""
|
||||
results = []
|
||||
for env_key, env_cfg in ENVIRONMENTS.items():
|
||||
proc = _find_uvicorn_process(env_cfg["port"])
|
||||
if proc:
|
||||
try:
|
||||
mem_info = proc.memory_info()
|
||||
create_time = proc.create_time()
|
||||
results.append(ServiceStatus(
|
||||
env=env_key,
|
||||
label=env_cfg["label"],
|
||||
running=True,
|
||||
pid=proc.pid,
|
||||
port=env_cfg["port"],
|
||||
uptime_seconds=round(datetime.now().timestamp() - create_time, 1),
|
||||
memory_mb=round(mem_info.rss / (1024 ** 2), 1),
|
||||
cpu_percent=proc.cpu_percent(interval=0.1),
|
||||
))
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
results.append(ServiceStatus(
|
||||
env=env_key, label=env_cfg["label"],
|
||||
running=False, port=env_cfg["port"],
|
||||
))
|
||||
else:
|
||||
results.append(ServiceStatus(
|
||||
env=env_key, label=env_cfg["label"],
|
||||
running=False, port=env_cfg["port"],
|
||||
))
|
||||
return results
|
||||
|
||||
|
||||
# ---- 服务启停 ----
|
||||
|
||||
@router.post("/services/{env}/start", response_model=ServiceActionResult)
|
||||
async def start_service(env: str):
|
||||
"""启动指定环境的后端服务。"""
|
||||
if env not in ENVIRONMENTS:
|
||||
raise HTTPException(404, f"未知环境: {env}")
|
||||
|
||||
cfg = ENVIRONMENTS[env]
|
||||
proc = _find_uvicorn_process(cfg["port"])
|
||||
if proc:
|
||||
return ServiceActionResult(
|
||||
env=env, action="start", success=True,
|
||||
message=f"服务已在运行中 (PID: {proc.pid})",
|
||||
)
|
||||
|
||||
bat_path = cfg["bat_script"]
|
||||
if not os.path.exists(bat_path):
|
||||
raise HTTPException(400, f"启动脚本不存在: {bat_path}")
|
||||
|
||||
# 通过 start 命令在新窗口中启动 bat 脚本
|
||||
try:
|
||||
subprocess.Popen(
|
||||
f'start "{cfg["window_title"]}" cmd /c "{bat_path}"',
|
||||
shell=True,
|
||||
)
|
||||
# 等待进程启动
|
||||
await asyncio.sleep(3)
|
||||
new_proc = _find_uvicorn_process(cfg["port"])
|
||||
if new_proc:
|
||||
return ServiceActionResult(
|
||||
env=env, action="start", success=True,
|
||||
message=f"服务已启动 (PID: {new_proc.pid})",
|
||||
)
|
||||
else:
|
||||
return ServiceActionResult(
|
||||
env=env, action="start", success=False,
|
||||
message="启动命令已执行,但未检测到进程,请检查日志",
|
||||
)
|
||||
except Exception as e:
|
||||
return ServiceActionResult(
|
||||
env=env, action="start", success=False, message=str(e),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/services/{env}/stop", response_model=ServiceActionResult)
|
||||
async def stop_service(env: str):
|
||||
"""停止指定环境的后端服务。"""
|
||||
if env not in ENVIRONMENTS:
|
||||
raise HTTPException(404, f"未知环境: {env}")
|
||||
|
||||
cfg = ENVIRONMENTS[env]
|
||||
proc = _find_uvicorn_process(cfg["port"])
|
||||
if not proc:
|
||||
return ServiceActionResult(
|
||||
env=env, action="stop", success=True, message="服务未在运行",
|
||||
)
|
||||
|
||||
try:
|
||||
# 终止进程树(包括子进程)
|
||||
parent = psutil.Process(proc.pid)
|
||||
children = parent.children(recursive=True)
|
||||
for child in children:
|
||||
child.terminate()
|
||||
parent.terminate()
|
||||
# 等待进程退出
|
||||
gone, alive = psutil.wait_procs([parent] + children, timeout=5)
|
||||
for p in alive:
|
||||
p.kill()
|
||||
return ServiceActionResult(
|
||||
env=env, action="stop", success=True, message="服务已停止",
|
||||
)
|
||||
except Exception as e:
|
||||
return ServiceActionResult(
|
||||
env=env, action="stop", success=False, message=str(e),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/services/{env}/restart", response_model=ServiceActionResult)
|
||||
async def restart_service(env: str):
|
||||
"""重启指定环境的后端服务。"""
|
||||
stop_result = await stop_service(env)
|
||||
if not stop_result.success and "未在运行" not in stop_result.message:
|
||||
return ServiceActionResult(
|
||||
env=env, action="restart", success=False,
|
||||
message=f"停止失败: {stop_result.message}",
|
||||
)
|
||||
await asyncio.sleep(1)
|
||||
start_result = await start_service(env)
|
||||
return ServiceActionResult(
|
||||
env=env, action="restart",
|
||||
success=start_result.success,
|
||||
message=start_result.message,
|
||||
)
|
||||
|
||||
|
||||
# ---- Git 操作 ----
|
||||
|
||||
@router.get("/git", response_model=list[GitInfo])
|
||||
async def get_git_info():
|
||||
"""获取所有环境的 Git 状态。"""
|
||||
results = []
|
||||
for env_key, env_cfg in ENVIRONMENTS.items():
|
||||
repo = env_cfg["repo_path"]
|
||||
if not os.path.isdir(os.path.join(repo, ".git")):
|
||||
results.append(GitInfo(
|
||||
env=env_key, branch="N/A",
|
||||
last_commit_hash="N/A", last_commit_message="仓库不存在",
|
||||
last_commit_time="", has_local_changes=False,
|
||||
))
|
||||
continue
|
||||
|
||||
_, branch = _run_cmd(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=repo)
|
||||
_, log_out = _run_cmd(
|
||||
["git", "log", "-1", "--format=%H|%s|%ci"],
|
||||
cwd=repo,
|
||||
)
|
||||
_, status_out = _run_cmd(["git", "status", "--porcelain"], cwd=repo)
|
||||
|
||||
parts = log_out.strip().split("|", 2) if log_out else ["", "", ""]
|
||||
results.append(GitInfo(
|
||||
env=env_key,
|
||||
branch=branch.strip(),
|
||||
last_commit_hash=parts[0][:8] if parts[0] else "N/A",
|
||||
last_commit_message=parts[1] if len(parts) > 1 else "",
|
||||
last_commit_time=parts[2] if len(parts) > 2 else "",
|
||||
has_local_changes=bool(status_out.strip()),
|
||||
))
|
||||
return results
|
||||
|
||||
|
||||
@router.post("/git/{env}/pull", response_model=GitPullResult)
|
||||
async def git_pull(env: str):
|
||||
"""对指定环境执行 git pull。"""
|
||||
if env not in ENVIRONMENTS:
|
||||
raise HTTPException(404, f"未知环境: {env}")
|
||||
|
||||
cfg = ENVIRONMENTS[env]
|
||||
repo = cfg["repo_path"]
|
||||
if not os.path.isdir(os.path.join(repo, ".git")):
|
||||
raise HTTPException(400, f"仓库路径不存在: {repo}")
|
||||
|
||||
success, output = _run_cmd(["git", "pull", "--ff-only"], cwd=repo, timeout=60)
|
||||
return GitPullResult(env=env, success=success, output=output)
|
||||
|
||||
|
||||
@router.post("/git/{env}/sync-deps", response_model=ServiceActionResult)
|
||||
async def sync_deps(env: str):
|
||||
"""对指定环境执行 uv sync --all-packages。"""
|
||||
if env not in ENVIRONMENTS:
|
||||
raise HTTPException(404, f"未知环境: {env}")
|
||||
|
||||
cfg = ENVIRONMENTS[env]
|
||||
repo = cfg["repo_path"]
|
||||
success, output = _run_cmd(["uv", "sync", "--all-packages"], cwd=repo, timeout=120)
|
||||
return ServiceActionResult(
|
||||
env=env, action="sync-deps", success=success, message=output[:500],
|
||||
)
|
||||
|
||||
|
||||
# ---- 环境配置管理 ----
|
||||
|
||||
@router.get("/env-file/{env}")
|
||||
async def get_env_file(env: str):
|
||||
"""读取指定环境的 .env 文件(敏感值脱敏)。"""
|
||||
if env not in ENVIRONMENTS:
|
||||
raise HTTPException(404, f"未知环境: {env}")
|
||||
|
||||
env_path = Path(ENVIRONMENTS[env]["repo_path"]) / ".env"
|
||||
if not env_path.exists():
|
||||
raise HTTPException(404, f".env 文件不存在: {env_path}")
|
||||
|
||||
lines = env_path.read_text(encoding="utf-8").splitlines()
|
||||
masked_lines = []
|
||||
sensitive_keys = {"PASSWORD", "SECRET", "TOKEN", "DSN", "APP_SECRET"}
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if stripped and not stripped.startswith("#") and "=" in stripped:
|
||||
key = stripped.split("=", 1)[0].strip()
|
||||
if any(s in key.upper() for s in sensitive_keys):
|
||||
masked_lines.append(f"{key}=********")
|
||||
continue
|
||||
masked_lines.append(line)
|
||||
return {"env": env, "content": "\n".join(masked_lines)}
|
||||
@@ -4,7 +4,7 @@
|
||||
提供 4 个端点:
|
||||
- GET /api/tasks/registry — 按业务域分组的任务列表
|
||||
- GET /api/tasks/dwd-tables — 按业务域分组的 DWD 表定义
|
||||
- GET /api/tasks/flows — 7 种 Flow + 3 种处理模式
|
||||
- GET /api/tasks/flows — 7 种 Flow + 4 种处理模式
|
||||
- POST /api/tasks/validate — 验证 TaskConfig 并返回 CLI 命令预览
|
||||
|
||||
所有端点需要 JWT 认证。validate 端点从 JWT 注入 store_id。
|
||||
@@ -103,6 +103,7 @@ PROCESSING_MODE_DEFINITIONS: list[ProcessingModeDefinition] = [
|
||||
ProcessingModeDefinition(id="increment_only", name="仅增量处理", description="只处理新增和变更的数据"),
|
||||
ProcessingModeDefinition(id="verify_only", name="仅校验修复", description="校验现有数据并修复不一致"),
|
||||
ProcessingModeDefinition(id="increment_verify", name="增量 + 校验修复", description="先增量处理,再校验并修复"),
|
||||
ProcessingModeDefinition(id="full_window", name="全窗口处理", description="用 API 返回数据的实际时间范围处理全部层,无需校验"),
|
||||
]
|
||||
|
||||
|
||||
@@ -163,7 +164,7 @@ async def get_dwd_tables(
|
||||
async def get_flows(
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> FlowsResponse:
|
||||
"""返回 7 种 Flow 定义和 3 种处理模式定义"""
|
||||
"""返回 7 种 Flow 定义和 4 种处理模式定义"""
|
||||
return FlowsResponse(
|
||||
flows=FLOW_DEFINITIONS,
|
||||
processing_modes=PROCESSING_MODE_DEFINITIONS,
|
||||
@@ -183,8 +184,9 @@ async def validate_task_config(
|
||||
errors: list[str] = []
|
||||
|
||||
# 验证 Flow ID
|
||||
if config.pipeline not in FLOW_LAYER_MAP:
|
||||
errors.append(f"无效的执行流程: {config.pipeline}")
|
||||
# CHANGE [2026-02-20] intent: pipeline → flow,统一命名
|
||||
if config.flow not in FLOW_LAYER_MAP:
|
||||
errors.append(f"无效的执行流程: {config.flow}")
|
||||
|
||||
# 验证任务列表非空
|
||||
if not config.tasks:
|
||||
|
||||
19
apps/backend/app/schemas/member_birthday.py
Normal file
19
apps/backend/app/schemas/member_birthday.py
Normal file
@@ -0,0 +1,19 @@
|
||||
"""
|
||||
会员生日手动补录相关 Pydantic 模型。
|
||||
|
||||
- MemberBirthdaySubmit:助教提交会员生日请求体
|
||||
"""
|
||||
|
||||
from datetime import date
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class MemberBirthdaySubmit(BaseModel):
|
||||
"""助教提交会员生日请求。"""
|
||||
|
||||
member_id: int = Field(..., gt=0, description="会员 ID")
|
||||
birthday_value: date = Field(..., description="生日日期")
|
||||
assistant_id: int = Field(..., gt=0, description="助教 ID")
|
||||
assistant_name: str = Field(..., min_length=1, max_length=50, description="助教姓名")
|
||||
site_id: int = Field(..., gt=0, description="门店 ID")
|
||||
@@ -13,8 +13,8 @@ class TaskConfigSchema(BaseModel):
|
||||
"""任务配置 — 前后端传输格式
|
||||
|
||||
字段与 CLI 参数的映射关系:
|
||||
- pipeline → --pipeline(Flow ID,7 种之一)
|
||||
- processing_mode → --processing-mode(3 种处理模式)
|
||||
- flow → --flow(Flow ID,7 种之一)
|
||||
- processing_mode → --processing-mode(4 种处理模式)
|
||||
- tasks → --tasks(逗号分隔)
|
||||
- dry_run → --dry-run(布尔标志)
|
||||
- window_mode → 决定使用 lookback 还是 custom 时间窗口(仅前端逻辑,不直接映射 CLI 参数)
|
||||
@@ -30,7 +30,8 @@ class TaskConfigSchema(BaseModel):
|
||||
"""
|
||||
|
||||
tasks: list[str]
|
||||
pipeline: str = "api_ods_dwd"
|
||||
# CHANGE [2026-02-20] intent: pipeline → flow,统一命名(消除历史别名)
|
||||
flow: str = "api_ods_dwd"
|
||||
processing_mode: str = "increment_only"
|
||||
dry_run: bool = False
|
||||
window_mode: str = "lookback"
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
支持:
|
||||
- 7 种 Flow(api_ods / api_ods_dwd / api_full / ods_dwd / dwd_dws / dwd_dws_index / dwd_index)
|
||||
- 3 种处理模式(increment_only / verify_only / increment_verify)
|
||||
- 4 种处理模式(increment_only / verify_only / increment_verify / full_window)
|
||||
- 自动注入 --store-id 参数
|
||||
"""
|
||||
|
||||
@@ -30,6 +30,7 @@ VALID_PROCESSING_MODES: set[str] = {
|
||||
"increment_only",
|
||||
"verify_only",
|
||||
"increment_verify",
|
||||
"full_window",
|
||||
}
|
||||
|
||||
# CLI 支持的 extra_args 键(值类型 + 布尔类型)
|
||||
@@ -72,7 +73,8 @@ class CLIBuilder:
|
||||
cmd: list[str] = [python_executable, "-m", "cli.main"]
|
||||
|
||||
# -- Flow(执行流程) --
|
||||
cmd.extend(["--flow", config.pipeline])
|
||||
# CHANGE [2026-02-20] intent: pipeline → flow,统一命名
|
||||
cmd.extend(["--flow", config.flow])
|
||||
|
||||
# -- 处理模式 --
|
||||
if config.processing_mode:
|
||||
|
||||
@@ -46,7 +46,7 @@ ODS_TASKS: list[TaskDefinition] = [
|
||||
TaskDefinition("ODS_ASSISTANT_LEDGER", "助教服务记录", "抽取助教服务流水", "助教", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_ASSISTANT_ABOLISH", "助教取消记录", "抽取助教取消/作废记录", "助教", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_SETTLEMENT_RECORDS", "结算记录", "抽取订单结算记录", "结算", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_SETTLEMENT_TICKET", "结账小票", "抽取结账小票明细", "结算", "ODS", is_ods=True),
|
||||
# CHANGE [2026-07-20] intent: 同步 ETL 侧移除——ODS_SETTLEMENT_TICKET 已在 Task 7.3 中彻底移除
|
||||
TaskDefinition("ODS_TABLE_USE", "台费流水", "抽取台费使用流水", "台桌", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_TABLE_FEE_DISCOUNT", "台费折扣", "抽取台费折扣记录", "台桌", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_TABLES", "台桌主数据", "抽取门店台桌信息", "台桌", "ODS", is_ods=True, requires_window=False),
|
||||
@@ -59,7 +59,7 @@ ODS_TASKS: list[TaskDefinition] = [
|
||||
TaskDefinition("ODS_RECHARGE_SETTLE", "充值结算", "抽取充值结算记录", "会员", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_GROUP_PACKAGE", "团购套餐", "抽取团购套餐定义", "团购", "ODS", is_ods=True, requires_window=False),
|
||||
TaskDefinition("ODS_GROUP_BUY_REDEMPTION", "团购核销", "抽取团购核销记录", "团购", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_INVENTORY_STOCK", "库存快照", "抽取商品库存汇总", "库存", "ODS", is_ods=True, requires_window=False),
|
||||
TaskDefinition("ODS_INVENTORY_STOCK", "库存快照", "抽取商品库存汇总", "库存", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_INVENTORY_CHANGE", "库存变动", "抽取库存出入库记录", "库存", "ODS", is_ods=True),
|
||||
TaskDefinition("ODS_GOODS_CATEGORY", "商品分类", "抽取商品分类树", "商品", "ODS", is_ods=True, requires_window=False),
|
||||
TaskDefinition("ODS_STORE_GOODS", "门店商品", "抽取门店商品主数据", "商品", "ODS", is_ods=True, requires_window=False),
|
||||
@@ -91,6 +91,10 @@ DWS_TASKS: list[TaskDefinition] = [
|
||||
TaskDefinition("DWS_FINANCE_DISCOUNT_DETAIL", "折扣明细", "汇总折扣明细", "财务", "DWS"),
|
||||
# CHANGE [2026-02-19] intent: 同步 ETL 侧合并——原 DWS_RETENTION_CLEANUP / DWS_MV_REFRESH_* 已合并为 DWS_MAINTENANCE
|
||||
TaskDefinition("DWS_MAINTENANCE", "DWS 维护", "刷新物化视图 + 清理过期留存数据", "通用", "DWS", requires_window=False, is_common=False),
|
||||
# CHANGE [2026-07-20] intent: 注册 DWS 库存汇总任务(日/周/月),依赖 DWD goods_stock_summary 加载完成(需求 12.9)
|
||||
TaskDefinition("DWS_GOODS_STOCK_DAILY", "库存日报", "按日粒度汇总商品库存数据", "库存", "DWS"),
|
||||
TaskDefinition("DWS_GOODS_STOCK_WEEKLY", "库存周报", "按周粒度汇总商品库存数据", "库存", "DWS"),
|
||||
TaskDefinition("DWS_GOODS_STOCK_MONTHLY", "库存月报", "按月粒度汇总商品库存数据", "库存", "DWS"),
|
||||
]
|
||||
|
||||
# ── INDEX 任务定义 ────────────────────────────────────────────
|
||||
@@ -99,7 +103,8 @@ INDEX_TASKS: list[TaskDefinition] = [
|
||||
TaskDefinition("DWS_WINBACK_INDEX", "回流指数 (WBI)", "计算会员回流指数", "指数", "INDEX"),
|
||||
TaskDefinition("DWS_NEWCONV_INDEX", "新客转化指数 (NCI)", "计算新客转化指数", "指数", "INDEX"),
|
||||
TaskDefinition("DWS_ML_MANUAL_IMPORT", "手动导入 (ML)", "手动导入机器学习数据", "指数", "INDEX", requires_window=False, is_common=False),
|
||||
TaskDefinition("DWS_RELATION_INDEX", "关系指数 (RS)", "计算助教-客户关系指数", "指数", "INDEX"),
|
||||
# CHANGE [2026-02-19] intent: 补充说明 RelationIndexTask 产出 RS/OS/MS/ML 四个子指数
|
||||
TaskDefinition("DWS_RELATION_INDEX", "关系指数 (RS)", "产出 RS/OS/MS/ML 四个子指数", "指数", "INDEX"),
|
||||
]
|
||||
|
||||
# ── 工具类任务定义 ────────────────────────────────────────────
|
||||
@@ -210,6 +215,9 @@ DWD_TABLES: list[DwdTableDefinition] = [
|
||||
DwdTableDefinition("dwd.dwd_payment", "支付流水", "结算", "ods.payment_transactions"),
|
||||
DwdTableDefinition("dwd.dwd_refund", "退款流水", "结算", "ods.refund_transactions"),
|
||||
DwdTableDefinition("dwd.dwd_refund_ex", "退款流水(扩展)", "结算", "ods.refund_transactions"),
|
||||
# CHANGE [2026-07-20] intent: 同步 Task 6.1/6.2 新建的 DWD 库存表
|
||||
DwdTableDefinition("dwd.dwd_goods_stock_summary", "库存汇总", "库存", "ods.goods_stock_summary"),
|
||||
DwdTableDefinition("dwd.dwd_goods_stock_movement", "库存变动", "库存", "ods.goods_stock_movements"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ dependencies = [
|
||||
"python-dotenv>=1.0",
|
||||
"python-jose[cryptography]>=3.3",
|
||||
"bcrypt>=4.0",
|
||||
"psutil>=5.9",
|
||||
]
|
||||
|
||||
[tool.uv.sources]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""CLIBuilder 单元测试
|
||||
|
||||
覆盖:7 种 Flow、3 种处理模式、时间窗口、store_id 自动注入、extra_args 等。
|
||||
覆盖:7 种 Flow、4 种处理模式、时间窗口、store_id 自动注入、extra_args 等。
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -24,11 +24,11 @@ ETL_PATH = "/fake/etl/project"
|
||||
|
||||
class TestBasicCommand:
|
||||
def test_minimal_command(self, builder: CLIBuilder):
|
||||
"""最小配置应生成 python -m cli.main --pipeline ... --processing-mode ..."""
|
||||
"""最小配置应生成 python -m cli.main --flow ... --processing-mode ..."""
|
||||
config = TaskConfigSchema(tasks=["ODS_MEMBER"])
|
||||
cmd = builder.build_command(config, ETL_PATH)
|
||||
assert cmd[:3] == ["python", "-m", "cli.main"]
|
||||
assert "--pipeline" in cmd
|
||||
assert "--flow" in cmd
|
||||
assert "--processing-mode" in cmd
|
||||
|
||||
def test_custom_python_executable(self, builder: CLIBuilder):
|
||||
@@ -56,20 +56,20 @@ class TestBasicCommand:
|
||||
class TestFlows:
|
||||
@pytest.mark.parametrize("flow_id", sorted(VALID_FLOWS))
|
||||
def test_all_flows_accepted(self, builder: CLIBuilder, flow_id: str):
|
||||
config = TaskConfigSchema(tasks=["ODS_MEMBER"], pipeline=flow_id)
|
||||
config = TaskConfigSchema(tasks=["ODS_MEMBER"], flow=flow_id)
|
||||
cmd = builder.build_command(config, ETL_PATH)
|
||||
idx = cmd.index("--pipeline")
|
||||
idx = cmd.index("--flow")
|
||||
assert cmd[idx + 1] == flow_id
|
||||
|
||||
def test_default_flow_is_api_ods_dwd(self, builder: CLIBuilder):
|
||||
config = TaskConfigSchema(tasks=["ODS_MEMBER"])
|
||||
cmd = builder.build_command(config, ETL_PATH)
|
||||
idx = cmd.index("--pipeline")
|
||||
idx = cmd.index("--flow")
|
||||
assert cmd[idx + 1] == "api_ods_dwd"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3 种处理模式
|
||||
# 4 种处理模式
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestProcessingModes:
|
||||
|
||||
@@ -36,7 +36,7 @@ _NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
# 构造测试用的 TaskConfig payload
|
||||
_VALID_CONFIG = {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
}
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ class TestRunTask:
|
||||
|
||||
def test_run_invalid_config_returns_422(self):
|
||||
"""缺少必填字段 tasks 时返回 422"""
|
||||
resp = client.post("/api/execution/run", json={"pipeline": "api_ods"})
|
||||
resp = client.post("/api/execution/run", json={"flow": "api_ods"})
|
||||
assert resp.status_code == 422
|
||||
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ _task_codes = ["ODS_MEMBER", "ODS_PAYMENT", "ODS_ORDER", "DWD_LOAD_FROM_ODS", "D
|
||||
_simple_config_st = st.builds(
|
||||
TaskConfigSchema,
|
||||
tasks=st.lists(st.sampled_from(_task_codes), min_size=1, max_size=3, unique=True),
|
||||
pipeline=st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd"]),
|
||||
flow=st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd"]),
|
||||
)
|
||||
|
||||
|
||||
@@ -254,7 +254,7 @@ def test_queue_crud_invariant(mock_get_conn, config, site_id, initial_count):
|
||||
db.rows[tid] = {
|
||||
"id": tid,
|
||||
"site_id": site_id,
|
||||
"config": {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"},
|
||||
"config": {"tasks": ["ODS_MEMBER"], "flow": "api_ods"},
|
||||
"status": "pending",
|
||||
"position": i + 1,
|
||||
}
|
||||
@@ -322,7 +322,7 @@ def test_queue_dequeue_order(mock_get_conn, site_id, num_tasks, positions):
|
||||
db.rows[tid] = {
|
||||
"id": tid,
|
||||
"site_id": site_id,
|
||||
"config": {"tasks": [_task_codes[i % len(_task_codes)]], "pipeline": "api_ods"},
|
||||
"config": {"tasks": [_task_codes[i % len(_task_codes)]], "flow": "api_ods"},
|
||||
"status": "pending",
|
||||
"position": pos,
|
||||
}
|
||||
@@ -372,7 +372,7 @@ def test_queue_reorder_consistency(mock_get_conn, site_id, num_tasks, data):
|
||||
db.rows[tid] = {
|
||||
"id": tid,
|
||||
"site_id": site_id,
|
||||
"config": {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"},
|
||||
"config": {"tasks": ["ODS_MEMBER"], "flow": "api_ods"},
|
||||
"status": "pending",
|
||||
"position": i + 1,
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ _task_codes = ["ODS_MEMBER", "ODS_PAYMENT", "ODS_ORDER", "DWD_LOAD_FROM_ODS", "D
|
||||
|
||||
_simple_task_config_st = st.fixed_dictionaries({
|
||||
"tasks": st.lists(st.sampled_from(_task_codes), min_size=1, max_size=3, unique=True),
|
||||
"pipeline": st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd", "api_full"]),
|
||||
"flow": st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd", "api_full"]),
|
||||
})
|
||||
|
||||
# 调度配置策略:覆盖 5 种调度类型
|
||||
@@ -324,8 +324,8 @@ def test_due_schedule_auto_enqueue(
|
||||
assert enqueued_config.tasks == task_config["tasks"], (
|
||||
f"入队的 tasks 应为 {task_config['tasks']},实际 {enqueued_config.tasks}"
|
||||
)
|
||||
assert enqueued_config.pipeline == task_config["pipeline"], (
|
||||
f"入队的 pipeline 应为 {task_config['pipeline']},实际 {enqueued_config.pipeline}"
|
||||
assert enqueued_config.flow == task_config["flow"], (
|
||||
f"入队的 flow 应为 {task_config['flow']},实际 {enqueued_config.flow}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -230,7 +230,7 @@ class TestCheckAndEnqueue:
|
||||
@patch("app.services.scheduler.task_queue")
|
||||
def test_enqueues_due_tasks(self, mock_tq, mock_get_conn, sched):
|
||||
"""到期任务应被入队,且更新 last_run_at / run_count / next_run_at"""
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
|
||||
schedule_config = {
|
||||
"schedule_type": "interval",
|
||||
"interval_value": 1,
|
||||
@@ -280,7 +280,7 @@ class TestCheckAndEnqueue:
|
||||
def test_skips_invalid_config(self, mock_tq, mock_get_conn, sched):
|
||||
"""配置反序列化失败的任务应被跳过"""
|
||||
# task_config 缺少必填字段 tasks
|
||||
bad_config = {"pipeline": "api_ods_dwd"}
|
||||
bad_config = {"flow": "api_ods_dwd"}
|
||||
schedule_config = {"schedule_type": "once"}
|
||||
|
||||
cur = _mock_cursor(
|
||||
@@ -300,7 +300,7 @@ class TestCheckAndEnqueue:
|
||||
@patch("app.services.scheduler.task_queue")
|
||||
def test_enqueue_failure_continues(self, mock_tq, mock_get_conn, sched):
|
||||
"""入队失败时应跳过该任务,继续处理后续任务"""
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
|
||||
schedule_config = {"schedule_type": "once"}
|
||||
|
||||
cur = _mock_cursor(
|
||||
@@ -327,7 +327,7 @@ class TestCheckAndEnqueue:
|
||||
@patch("app.services.scheduler.task_queue")
|
||||
def test_once_type_sets_next_run_none(self, mock_tq, mock_get_conn, sched):
|
||||
"""once 类型任务入队后,next_run_at 应被设为 NULL"""
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
|
||||
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
|
||||
schedule_config = {"schedule_type": "once"}
|
||||
|
||||
select_cur = _mock_cursor(
|
||||
|
||||
@@ -40,14 +40,14 @@ _SCHEDULE_CONFIG = {
|
||||
_VALID_CREATE = {
|
||||
"name": "每日全量同步",
|
||||
"task_codes": ["ODS_MEMBER", "ODS_ORDER"],
|
||||
"task_config": {"tasks": ["ODS_MEMBER", "ODS_ORDER"], "pipeline": "api_ods"},
|
||||
"task_config": {"tasks": ["ODS_MEMBER", "ODS_ORDER"], "flow": "api_ods"},
|
||||
"schedule_config": _SCHEDULE_CONFIG,
|
||||
}
|
||||
|
||||
# 模拟数据库返回的完整行(13 列,与 _SELECT_COLS 对应)
|
||||
_DB_ROW = (
|
||||
"sched-1", 100, "每日全量同步", ["ODS_MEMBER", "ODS_ORDER"],
|
||||
json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}),
|
||||
json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"}),
|
||||
json.dumps(_SCHEDULE_CONFIG),
|
||||
True, None, _NEXT, 0, None, _NOW, _NOW,
|
||||
)
|
||||
|
||||
@@ -53,7 +53,7 @@ def _make_queue_rows(site_id: int, count: int) -> list[tuple]:
|
||||
rows.append((
|
||||
str(uuid.uuid4()), # id
|
||||
site_id, # site_id
|
||||
json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}), # config
|
||||
json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"}), # config
|
||||
"pending", # status
|
||||
i + 1, # position
|
||||
datetime(2024, 1, 1, tzinfo=timezone.utc), # created_at
|
||||
@@ -75,7 +75,7 @@ def _make_schedule_rows(site_id: int, count: int) -> list[tuple]:
|
||||
site_id, # site_id
|
||||
f"调度任务_{i}", # name
|
||||
["ODS_MEMBER"], # task_codes
|
||||
{"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}, # task_config
|
||||
{"tasks": ["ODS_MEMBER"], "flow": "api_ods"}, # task_config
|
||||
{"schedule_type": "daily", "daily_time": "04:00", # schedule_config
|
||||
"interval_value": 1, "interval_unit": "hours",
|
||||
"weekly_days": [1], "weekly_time": "04:00",
|
||||
|
||||
@@ -31,7 +31,7 @@ _tasks_st = st.lists(
|
||||
unique=True,
|
||||
)
|
||||
|
||||
_pipeline_st = st.sampled_from(sorted(VALID_FLOWS))
|
||||
_flow_st = st.sampled_from(sorted(VALID_FLOWS))
|
||||
_processing_mode_st = st.sampled_from(sorted(VALID_PROCESSING_MODES))
|
||||
_window_mode_st = st.sampled_from(["lookback", "custom"])
|
||||
|
||||
@@ -69,7 +69,7 @@ def _valid_task_config_st():
|
||||
@st.composite
|
||||
def _build(draw):
|
||||
tasks = draw(_tasks_st)
|
||||
pipeline = draw(_pipeline_st)
|
||||
flow_id = draw(_flow_st)
|
||||
processing_mode = draw(_processing_mode_st)
|
||||
dry_run = draw(st.booleans())
|
||||
window_mode = draw(_window_mode_st)
|
||||
@@ -103,7 +103,7 @@ def _valid_task_config_st():
|
||||
|
||||
return TaskConfigSchema(
|
||||
tasks=tasks,
|
||||
pipeline=pipeline,
|
||||
flow=flow_id,
|
||||
processing_mode=processing_mode,
|
||||
dry_run=dry_run,
|
||||
window_mode=window_mode,
|
||||
@@ -204,10 +204,10 @@ def test_task_config_to_cli_completeness(config: TaskConfigSchema):
|
||||
"""Property 7: CLIBuilder 生成的命令应包含 TaskConfig 中所有非空字段对应的 CLI 参数。"""
|
||||
cmd = _builder.build_command(config, _ETL_PATH)
|
||||
|
||||
# 1) --pipeline 始终存在且值正确
|
||||
assert "--pipeline" in cmd
|
||||
idx = cmd.index("--pipeline")
|
||||
assert cmd[idx + 1] == config.pipeline
|
||||
# 1) --flow 始终存在且值正确
|
||||
assert "--flow" in cmd
|
||||
idx = cmd.index("--flow")
|
||||
assert cmd[idx + 1] == config.flow
|
||||
|
||||
# 2) --processing-mode 始终存在且值正确
|
||||
assert "--processing-mode" in cmd
|
||||
|
||||
@@ -24,7 +24,7 @@ def executor() -> TaskExecutor:
|
||||
def sample_config() -> TaskConfigSchema:
|
||||
return TaskConfigSchema(
|
||||
tasks=["ODS_MEMBER", "ODS_PAYMENT"],
|
||||
pipeline="api_ods_dwd",
|
||||
flow="api_ods_dwd",
|
||||
store_id=42,
|
||||
)
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ def queue() -> TaskQueue:
|
||||
def sample_config() -> TaskConfigSchema:
|
||||
return TaskConfigSchema(
|
||||
tasks=["ODS_MEMBER", "ODS_PAYMENT"],
|
||||
pipeline="api_ods_dwd",
|
||||
flow="api_ods_dwd",
|
||||
store_id=42,
|
||||
)
|
||||
|
||||
@@ -107,7 +107,7 @@ class TestEnqueue:
|
||||
config_json_str = insert_call[0][1][2]
|
||||
parsed = json.loads(config_json_str)
|
||||
assert parsed["tasks"] == ["ODS_MEMBER", "ODS_PAYMENT"]
|
||||
assert parsed["pipeline"] == "api_ods_dwd"
|
||||
assert parsed["flow"] == "api_ods_dwd"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -129,7 +129,7 @@ class TestDequeue:
|
||||
@patch("app.services.task_queue.get_connection")
|
||||
def test_dequeue_returns_task(self, mock_get_conn, queue):
|
||||
task_id = str(uuid.uuid4())
|
||||
config_dict = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}
|
||||
config_dict = {"tasks": ["ODS_MEMBER"], "flow": "api_ods"}
|
||||
row = (
|
||||
task_id, 42, json.dumps(config_dict), "pending", 1,
|
||||
None, None, None, None, None,
|
||||
@@ -149,7 +149,7 @@ class TestDequeue:
|
||||
@patch("app.services.task_queue.get_connection")
|
||||
def test_dequeue_updates_status_to_running(self, mock_get_conn, queue):
|
||||
task_id = str(uuid.uuid4())
|
||||
config_dict = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}
|
||||
config_dict = {"tasks": ["ODS_MEMBER"], "flow": "api_ods"}
|
||||
row = (
|
||||
task_id, 42, json.dumps(config_dict), "pending", 1,
|
||||
None, None, None, None, None,
|
||||
@@ -285,7 +285,7 @@ class TestQuery:
|
||||
@patch("app.services.task_queue.get_connection")
|
||||
def test_list_pending_returns_tasks(self, mock_get_conn, queue):
|
||||
tid = str(uuid.uuid4())
|
||||
config = json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"})
|
||||
config = json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"})
|
||||
rows = [(tid, 42, config, "pending", 1, None, None, None, None, None)]
|
||||
cur = _mock_cursor(fetchall_val=rows)
|
||||
conn = _mock_conn(cur)
|
||||
@@ -353,7 +353,7 @@ class TestProcessLoop:
|
||||
task_id = str(uuid.uuid4())
|
||||
config_dict = {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "api_ods_dwd",
|
||||
"flow": "api_ods_dwd",
|
||||
"processing_mode": "increment_only",
|
||||
"dry_run": False,
|
||||
"window_mode": "lookback",
|
||||
|
||||
@@ -152,7 +152,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["ODS_MEMBER", "ODS_PAYMENT"],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
}
|
||||
})
|
||||
assert resp.status_code == 200
|
||||
@@ -169,7 +169,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["DWD_LOAD_FROM_ODS"],
|
||||
"pipeline": "ods_dwd",
|
||||
"flow": "ods_dwd",
|
||||
"store_id": 999,
|
||||
}
|
||||
})
|
||||
@@ -184,7 +184,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "nonexistent_flow",
|
||||
"flow": "nonexistent_flow",
|
||||
}
|
||||
})
|
||||
assert resp.status_code == 200
|
||||
@@ -196,7 +196,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": [],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
}
|
||||
})
|
||||
assert resp.status_code == 200
|
||||
@@ -208,7 +208,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
"window_mode": "custom",
|
||||
"window_start": "2024-01-01",
|
||||
"window_end": "2024-01-31",
|
||||
@@ -225,7 +225,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
"window_mode": "custom",
|
||||
"window_start": "2024-12-31",
|
||||
"window_end": "2024-01-01",
|
||||
@@ -237,7 +237,7 @@ class TestValidate:
|
||||
resp = client.post("/api/tasks/validate", json={
|
||||
"config": {
|
||||
"tasks": ["ODS_MEMBER"],
|
||||
"pipeline": "api_ods",
|
||||
"flow": "api_ods",
|
||||
"dry_run": True,
|
||||
}
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user