root/Neo-ZQYY
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

在前后端开发联调前 的提交20260223

Browse Source
This commit is contained in:
Neo
2026-02-23 23:02:20 +08:00
parent 254ccb1e77
commit fafc95e64c
1142 changed files with 10366960 additions and 36957 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
apps/backend/tests/test_cli_builder.py
View File

@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""CLIBuilder 单元测试
覆盖：7 种 Flow、3 种处理模式、时间窗口、store_id 自动注入、extra_args 等。
覆盖：7 种 Flow、4 种处理模式、时间窗口、store_id 自动注入、extra_args 等。
"""
import pytest
@@ -24,11 +24,11 @@ ETL_PATH = "/fake/etl/project"
class TestBasicCommand:
def test_minimal_command(self, builder: CLIBuilder):
"""最小配置应生成 python -m cli.main --pipeline ... --processing-mode ..."""
"""最小配置应生成 python -m cli.main --flow ... --processing-mode ..."""
config = TaskConfigSchema(tasks=["ODS_MEMBER"])
cmd = builder.build_command(config, ETL_PATH)
assert cmd[:3] == ["python", "-m", "cli.main"]
assert "--pipeline" in cmd
assert "--flow" in cmd
assert "--processing-mode" in cmd
def test_custom_python_executable(self, builder: CLIBuilder):
@@ -56,20 +56,20 @@ class TestBasicCommand:
class TestFlows:
@pytest.mark.parametrize("flow_id", sorted(VALID_FLOWS))
def test_all_flows_accepted(self, builder: CLIBuilder, flow_id: str):
config = TaskConfigSchema(tasks=["ODS_MEMBER"], pipeline=flow_id)
config = TaskConfigSchema(tasks=["ODS_MEMBER"], flow=flow_id)
cmd = builder.build_command(config, ETL_PATH)
idx = cmd.index("--pipeline")
idx = cmd.index("--flow")
assert cmd[idx + 1] == flow_id
def test_default_flow_is_api_ods_dwd(self, builder: CLIBuilder):
config = TaskConfigSchema(tasks=["ODS_MEMBER"])
cmd = builder.build_command(config, ETL_PATH)
idx = cmd.index("--pipeline")
idx = cmd.index("--flow")
assert cmd[idx + 1] == "api_ods_dwd"
# ---------------------------------------------------------------------------
# 3 种处理模式
# 4 种处理模式
# ---------------------------------------------------------------------------
class TestProcessingModes:

4
apps/backend/tests/test_execution_router.py
View File

@@ -36,7 +36,7 @@ _NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
# 构造测试用的 TaskConfig payload
_VALID_CONFIG = {
"tasks": ["ODS_MEMBER"],
"pipeline": "api_ods",
"flow": "api_ods",
}
@@ -74,7 +74,7 @@ class TestRunTask:
def test_run_invalid_config_returns_422(self):
"""缺少必填字段 tasks 时返回 422"""
resp = client.post("/api/execution/run", json={"pipeline": "api_ods"})
resp = client.post("/api/execution/run", json={"flow": "api_ods"})
assert resp.status_code == 422

8
apps/backend/tests/test_queue_properties.py
View File

@@ -40,7 +40,7 @@ _task_codes = ["ODS_MEMBER", "ODS_PAYMENT", "ODS_ORDER", "DWD_LOAD_FROM_ODS", "D
_simple_config_st = st.builds(
TaskConfigSchema,
tasks=st.lists(st.sampled_from(_task_codes), min_size=1, max_size=3, unique=True),
pipeline=st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd"]),
flow=st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd"]),
)
@@ -254,7 +254,7 @@ def test_queue_crud_invariant(mock_get_conn, config, site_id, initial_count):
db.rows[tid] = {
"id": tid,
"site_id": site_id,
"config": {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"},
"config": {"tasks": ["ODS_MEMBER"], "flow": "api_ods"},
"status": "pending",
"position": i + 1,
}
@@ -322,7 +322,7 @@ def test_queue_dequeue_order(mock_get_conn, site_id, num_tasks, positions):
db.rows[tid] = {
"id": tid,
"site_id": site_id,
"config": {"tasks": [_task_codes[i % len(_task_codes)]], "pipeline": "api_ods"},
"config": {"tasks": [_task_codes[i % len(_task_codes)]], "flow": "api_ods"},
"status": "pending",
"position": pos,
}
@@ -372,7 +372,7 @@ def test_queue_reorder_consistency(mock_get_conn, site_id, num_tasks, data):
db.rows[tid] = {
"id": tid,
"site_id": site_id,
"config": {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"},
"config": {"tasks": ["ODS_MEMBER"], "flow": "api_ods"},
"status": "pending",
"position": i + 1,
}

6
apps/backend/tests/test_schedule_properties.py
View File

@@ -42,7 +42,7 @@ _task_codes = ["ODS_MEMBER", "ODS_PAYMENT", "ODS_ORDER", "DWD_LOAD_FROM_ODS", "D
_simple_task_config_st = st.fixed_dictionaries({
"tasks": st.lists(st.sampled_from(_task_codes), min_size=1, max_size=3, unique=True),
"pipeline": st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd", "api_full"]),
"flow": st.sampled_from(["api_ods", "api_ods_dwd", "ods_dwd", "api_full"]),
})
# 调度配置策略：覆盖 5 种调度类型
@@ -324,8 +324,8 @@ def test_due_schedule_auto_enqueue(
assert enqueued_config.tasks == task_config["tasks"], (
f"入队的 tasks 应为 {task_config['tasks']}，实际 {enqueued_config.tasks}"
)
assert enqueued_config.pipeline == task_config["pipeline"], (
f"入队的 pipeline 应为 {task_config['pipeline']}，实际 {enqueued_config.pipeline}"
assert enqueued_config.flow == task_config["flow"], (
f"入队的 flow 应为 {task_config['flow']}，实际 {enqueued_config.flow}"
)

8
apps/backend/tests/test_scheduler.py
View File

@@ -230,7 +230,7 @@ class TestCheckAndEnqueue:
@patch("app.services.scheduler.task_queue")
def test_enqueues_due_tasks(self, mock_tq, mock_get_conn, sched):
"""到期任务应被入队，且更新 last_run_at / run_count / next_run_at"""
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
schedule_config = {
"schedule_type": "interval",
"interval_value": 1,
@@ -280,7 +280,7 @@ class TestCheckAndEnqueue:
def test_skips_invalid_config(self, mock_tq, mock_get_conn, sched):
"""配置反序列化失败的任务应被跳过"""
# task_config 缺少必填字段 tasks
bad_config = {"pipeline": "api_ods_dwd"}
bad_config = {"flow": "api_ods_dwd"}
schedule_config = {"schedule_type": "once"}
cur = _mock_cursor(
@@ -300,7 +300,7 @@ class TestCheckAndEnqueue:
@patch("app.services.scheduler.task_queue")
def test_enqueue_failure_continues(self, mock_tq, mock_get_conn, sched):
"""入队失败时应跳过该任务，继续处理后续任务"""
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
schedule_config = {"schedule_type": "once"}
cur = _mock_cursor(
@@ -327,7 +327,7 @@ class TestCheckAndEnqueue:
@patch("app.services.scheduler.task_queue")
def test_once_type_sets_next_run_none(self, mock_tq, mock_get_conn, sched):
"""once 类型任务入队后，next_run_at 应被设为 NULL"""
task_config = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods_dwd"}
task_config = {"tasks": ["ODS_MEMBER"], "flow": "api_ods_dwd"}
schedule_config = {"schedule_type": "once"}
select_cur = _mock_cursor(

4
apps/backend/tests/test_schedules_router.py
View File

@@ -40,14 +40,14 @@ _SCHEDULE_CONFIG = {
_VALID_CREATE = {
"name": "每日全量同步",
"task_codes": ["ODS_MEMBER", "ODS_ORDER"],
"task_config": {"tasks": ["ODS_MEMBER", "ODS_ORDER"], "pipeline": "api_ods"},
"task_config": {"tasks": ["ODS_MEMBER", "ODS_ORDER"], "flow": "api_ods"},
"schedule_config": _SCHEDULE_CONFIG,
}
# 模拟数据库返回的完整行（13 列，与 _SELECT_COLS 对应）
_DB_ROW = (
"sched-1", 100, "每日全量同步", ["ODS_MEMBER", "ODS_ORDER"],
json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}),
json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"}),
json.dumps(_SCHEDULE_CONFIG),
True, None, _NEXT, 0, None, _NOW, _NOW,
)

4
apps/backend/tests/test_site_isolation_properties.py
View File

@@ -53,7 +53,7 @@ def _make_queue_rows(site_id: int, count: int) -> list[tuple]:
rows.append((
str(uuid.uuid4()), # id
site_id, # site_id
json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}), # config
json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"}), # config
"pending", # status
i + 1, # position
datetime(2024, 1, 1, tzinfo=timezone.utc), # created_at
@@ -75,7 +75,7 @@ def _make_schedule_rows(site_id: int, count: int) -> list[tuple]:
site_id, # site_id
f"调度任务_{i}", # name
["ODS_MEMBER"], # task_codes
{"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}, # task_config
{"tasks": ["ODS_MEMBER"], "flow": "api_ods"}, # task_config
{"schedule_type": "daily", "daily_time": "04:00", # schedule_config
"interval_value": 1, "interval_unit": "hours",
"weekly_days": [1], "weekly_time": "04:00",

14
apps/backend/tests/test_task_config_properties.py
View File

@@ -31,7 +31,7 @@ _tasks_st = st.lists(
unique=True,
)
_pipeline_st = st.sampled_from(sorted(VALID_FLOWS))
_flow_st = st.sampled_from(sorted(VALID_FLOWS))
_processing_mode_st = st.sampled_from(sorted(VALID_PROCESSING_MODES))
_window_mode_st = st.sampled_from(["lookback", "custom"])
@@ -69,7 +69,7 @@ def _valid_task_config_st():
@st.composite
def _build(draw):
tasks = draw(_tasks_st)
pipeline = draw(_pipeline_st)
flow_id = draw(_flow_st)
processing_mode = draw(_processing_mode_st)
dry_run = draw(st.booleans())
window_mode = draw(_window_mode_st)
@@ -103,7 +103,7 @@ def _valid_task_config_st():
return TaskConfigSchema(
tasks=tasks,
pipeline=pipeline,
flow=flow_id,
processing_mode=processing_mode,
dry_run=dry_run,
window_mode=window_mode,
@@ -204,10 +204,10 @@ def test_task_config_to_cli_completeness(config: TaskConfigSchema):
"""Property 7: CLIBuilder 生成的命令应包含 TaskConfig 中所有非空字段对应的 CLI 参数。"""
cmd = _builder.build_command(config, _ETL_PATH)
# 1) --pipeline 始终存在且值正确
assert "--pipeline" in cmd
idx = cmd.index("--pipeline")
assert cmd[idx + 1] == config.pipeline
# 1) --flow 始终存在且值正确
assert "--flow" in cmd
idx = cmd.index("--flow")
assert cmd[idx + 1] == config.flow
# 2) --processing-mode 始终存在且值正确
assert "--processing-mode" in cmd

2
apps/backend/tests/test_task_executor.py
View File

@@ -24,7 +24,7 @@ def executor() -> TaskExecutor:
def sample_config() -> TaskConfigSchema:
return TaskConfigSchema(
tasks=["ODS_MEMBER", "ODS_PAYMENT"],
pipeline="api_ods_dwd",
flow="api_ods_dwd",
store_id=42,
)

12
apps/backend/tests/test_task_queue.py
View File

@@ -25,7 +25,7 @@ def queue() -> TaskQueue:
def sample_config() -> TaskConfigSchema:
return TaskConfigSchema(
tasks=["ODS_MEMBER", "ODS_PAYMENT"],
pipeline="api_ods_dwd",
flow="api_ods_dwd",
store_id=42,
)
@@ -107,7 +107,7 @@ class TestEnqueue:
config_json_str = insert_call[0][1][2]
parsed = json.loads(config_json_str)
assert parsed["tasks"] == ["ODS_MEMBER", "ODS_PAYMENT"]
assert parsed["pipeline"] == "api_ods_dwd"
assert parsed["flow"] == "api_ods_dwd"
# ---------------------------------------------------------------------------
@@ -129,7 +129,7 @@ class TestDequeue:
@patch("app.services.task_queue.get_connection")
def test_dequeue_returns_task(self, mock_get_conn, queue):
task_id = str(uuid.uuid4())
config_dict = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}
config_dict = {"tasks": ["ODS_MEMBER"], "flow": "api_ods"}
row = (
task_id, 42, json.dumps(config_dict), "pending", 1,
None, None, None, None, None,
@@ -149,7 +149,7 @@ class TestDequeue:
@patch("app.services.task_queue.get_connection")
def test_dequeue_updates_status_to_running(self, mock_get_conn, queue):
task_id = str(uuid.uuid4())
config_dict = {"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"}
config_dict = {"tasks": ["ODS_MEMBER"], "flow": "api_ods"}
row = (
task_id, 42, json.dumps(config_dict), "pending", 1,
None, None, None, None, None,
@@ -285,7 +285,7 @@ class TestQuery:
@patch("app.services.task_queue.get_connection")
def test_list_pending_returns_tasks(self, mock_get_conn, queue):
tid = str(uuid.uuid4())
config = json.dumps({"tasks": ["ODS_MEMBER"], "pipeline": "api_ods"})
config = json.dumps({"tasks": ["ODS_MEMBER"], "flow": "api_ods"})
rows = [(tid, 42, config, "pending", 1, None, None, None, None, None)]
cur = _mock_cursor(fetchall_val=rows)
conn = _mock_conn(cur)
@@ -353,7 +353,7 @@ class TestProcessLoop:
task_id = str(uuid.uuid4())
config_dict = {
"tasks": ["ODS_MEMBER"],
"pipeline": "api_ods_dwd",
"flow": "api_ods_dwd",
"processing_mode": "increment_only",
"dry_run": False,
"window_mode": "lookback",

14
apps/backend/tests/test_tasks_router.py
View File

@@ -152,7 +152,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["ODS_MEMBER", "ODS_PAYMENT"],
"pipeline": "api_ods",
"flow": "api_ods",
}
})
assert resp.status_code == 200
@@ -169,7 +169,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["DWD_LOAD_FROM_ODS"],
"pipeline": "ods_dwd",
"flow": "ods_dwd",
"store_id": 999,
}
})
@@ -184,7 +184,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["ODS_MEMBER"],
"pipeline": "nonexistent_flow",
"flow": "nonexistent_flow",
}
})
assert resp.status_code == 200
@@ -196,7 +196,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": [],
"pipeline": "api_ods",
"flow": "api_ods",
}
})
assert resp.status_code == 200
@@ -208,7 +208,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["ODS_MEMBER"],
"pipeline": "api_ods",
"flow": "api_ods",
"window_mode": "custom",
"window_start": "2024-01-01",
"window_end": "2024-01-31",
@@ -225,7 +225,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["ODS_MEMBER"],
"pipeline": "api_ods",
"flow": "api_ods",
"window_mode": "custom",
"window_start": "2024-12-31",
"window_end": "2024-01-01",
@@ -237,7 +237,7 @@ class TestValidate:
resp = client.post("/api/tasks/validate", json={
"config": {
"tasks": ["ODS_MEMBER"],
"pipeline": "api_ods",
"flow": "api_ods",
"dry_run": True,
}
})