276 lines
9.5 KiB
Python
276 lines
9.5 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""TaskConfig 属性测试（Property-Based Testing）。
|
||
|
||
使用 hypothesis 验证 TaskConfig 相关的通用正确性属性：
|
||
- Property 1: TaskConfig 序列化往返一致性
|
||
- Property 6: 时间窗口验证
|
||
- Property 7: TaskConfig 到 CLI 命令转换完整性
|
||
"""
|
||
|
||
import datetime
|
||
|
||
from hypothesis import given, settings, assume
|
||
from hypothesis import strategies as st
|
||
from pydantic import ValidationError
|
||
|
||
from app.schemas.tasks import TaskConfigSchema
|
||
from app.services.cli_builder import CLIBuilder, VALID_FLOWS, VALID_PROCESSING_MODES
|
||
from app.services.task_registry import ALL_TASKS
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 策略（Strategies）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# 从真实任务注册表中采样任务代码
|
||
_task_codes = [t.code for t in ALL_TASKS]
|
||
|
||
_tasks_st = st.lists(
|
||
st.sampled_from(_task_codes),
|
||
min_size=1,
|
||
max_size=5,
|
||
unique=True,
|
||
)
|
||
|
||
_flow_st = st.sampled_from(sorted(VALID_FLOWS))
|
||
_processing_mode_st = st.sampled_from(sorted(VALID_PROCESSING_MODES))
|
||
_window_mode_st = st.sampled_from(["lookback", "custom"])
|
||
|
||
# 日期策略：生成 YYYY-MM-DD 格式字符串
|
||
_date_st = st.dates(
|
||
min_value=datetime.date(2020, 1, 1),
|
||
max_value=datetime.date(2030, 12, 31),
|
||
).map(lambda d: d.isoformat())
|
||
|
||
_window_split_st = st.sampled_from([None, "none", "day"])
|
||
_window_split_days_st = st.one_of(st.none(), st.sampled_from([1, 10, 30]))
|
||
_lookback_hours_st = st.integers(min_value=1, max_value=720)
|
||
_overlap_seconds_st = st.integers(min_value=0, max_value=7200)
|
||
_store_id_st = st.one_of(st.none(), st.integers(min_value=1, max_value=2**31 - 1))
|
||
|
||
# DWD 表名采样
|
||
_dwd_table_names = [
|
||
"dwd.dim_site",
|
||
"dwd.dim_member",
|
||
"dwd.dwd_settlement_head",
|
||
]
|
||
_dwd_only_tables_st = st.one_of(
|
||
st.none(),
|
||
st.lists(st.sampled_from(_dwd_table_names), min_size=1, max_size=3, unique=True),
|
||
)
|
||
|
||
|
||
def _valid_task_config_st():
|
||
"""生成有效的 TaskConfigSchema 的复合策略。
|
||
|
||
确保 window_mode=custom 时 window_end >= window_start，
|
||
避免触发 Pydantic 验证错误。
|
||
"""
|
||
|
||
@st.composite
|
||
def _build(draw):
|
||
tasks = draw(_tasks_st)
|
||
flow_id = draw(_flow_st)
|
||
processing_mode = draw(_processing_mode_st)
|
||
dry_run = draw(st.booleans())
|
||
window_mode = draw(_window_mode_st)
|
||
store_id = draw(_store_id_st)
|
||
dwd_only_tables = draw(_dwd_only_tables_st)
|
||
window_split = draw(_window_split_st)
|
||
window_split_days = draw(_window_split_days_st)
|
||
fetch_before_verify = draw(st.booleans())
|
||
skip_ods = draw(st.booleans())
|
||
ods_local = draw(st.booleans())
|
||
|
||
if window_mode == "custom":
|
||
d1 = draw(st.dates(
|
||
min_value=datetime.date(2020, 1, 1),
|
||
max_value=datetime.date(2030, 12, 31),
|
||
))
|
||
d2 = draw(st.dates(
|
||
min_value=datetime.date(2020, 1, 1),
|
||
max_value=datetime.date(2030, 12, 31),
|
||
))
|
||
# 保证 end >= start
|
||
window_start = min(d1, d2).isoformat()
|
||
window_end = max(d1, d2).isoformat()
|
||
lookback_hours = 24
|
||
overlap_seconds = 600
|
||
else:
|
||
window_start = None
|
||
window_end = None
|
||
lookback_hours = draw(_lookback_hours_st)
|
||
overlap_seconds = draw(_overlap_seconds_st)
|
||
|
||
return TaskConfigSchema(
|
||
tasks=tasks,
|
||
flow=flow_id,
|
||
processing_mode=processing_mode,
|
||
dry_run=dry_run,
|
||
window_mode=window_mode,
|
||
window_start=window_start,
|
||
window_end=window_end,
|
||
window_split=window_split,
|
||
window_split_days=window_split_days,
|
||
lookback_hours=lookback_hours,
|
||
overlap_seconds=overlap_seconds,
|
||
fetch_before_verify=fetch_before_verify,
|
||
skip_ods_when_fetch_before_verify=skip_ods,
|
||
ods_use_local_json=ods_local,
|
||
store_id=store_id,
|
||
dwd_only_tables=dwd_only_tables,
|
||
extra_args={},
|
||
)
|
||
|
||
return _build()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Feature: admin-web-console, Property 1: TaskConfig 序列化往返一致性
|
||
# **Validates: Requirements 11.1, 11.2, 11.3**
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@settings(max_examples=200)
|
||
@given(config=_valid_task_config_st())
|
||
def test_task_config_round_trip(config: TaskConfigSchema):
|
||
"""Property 1: 序列化为 JSON 后再反序列化，应产生与原始对象等价的结果。"""
|
||
json_str = config.model_dump_json()
|
||
restored = TaskConfigSchema.model_validate_json(json_str)
|
||
assert restored == config, (
|
||
f"往返不一致：\n原始={config}\n还原={restored}"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Feature: admin-web-console, Property 6: 时间窗口验证
|
||
# **Validates: Requirements 2.3**
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@settings(max_examples=200)
|
||
@given(
|
||
d1=st.dates(
|
||
min_value=datetime.date(2020, 1, 1),
|
||
max_value=datetime.date(2030, 12, 31),
|
||
),
|
||
d2=st.dates(
|
||
min_value=datetime.date(2020, 1, 1),
|
||
max_value=datetime.date(2030, 12, 31),
|
||
),
|
||
)
|
||
def test_time_window_validation(d1: datetime.date, d2: datetime.date):
|
||
"""Property 6: window_end < window_start 时验证应失败，否则应通过。"""
|
||
start_str = d1.isoformat()
|
||
end_str = d2.isoformat()
|
||
|
||
if end_str < start_str:
|
||
# window_end 早于 window_start → 验证应失败
|
||
try:
|
||
TaskConfigSchema(
|
||
tasks=["ODS_MEMBER"],
|
||
window_mode="custom",
|
||
window_start=start_str,
|
||
window_end=end_str,
|
||
)
|
||
raise AssertionError(
|
||
f"期望 ValidationError，但验证通过了：start={start_str}, end={end_str}"
|
||
)
|
||
except ValidationError:
|
||
pass # 预期行为
|
||
else:
|
||
# window_end >= window_start → 验证应通过
|
||
config = TaskConfigSchema(
|
||
tasks=["ODS_MEMBER"],
|
||
window_mode="custom",
|
||
window_start=start_str,
|
||
window_end=end_str,
|
||
)
|
||
assert config.window_start == start_str
|
||
assert config.window_end == end_str
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Feature: admin-web-console, Property 7: TaskConfig 到 CLI 命令转换完整性
|
||
# **Validates: Requirements 2.5, 2.6**
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_builder = CLIBuilder()
|
||
_ETL_PATH = "/fake/etl/project"
|
||
|
||
|
||
@settings(max_examples=200)
|
||
@given(config=_valid_task_config_st())
|
||
def test_task_config_to_cli_completeness(config: TaskConfigSchema):
|
||
"""Property 7: CLIBuilder 生成的命令应包含 TaskConfig 中所有非空字段对应的 CLI 参数。"""
|
||
cmd = _builder.build_command(config, _ETL_PATH)
|
||
|
||
# 1) --flow 始终存在且值正确
|
||
assert "--flow" in cmd
|
||
idx = cmd.index("--flow")
|
||
assert cmd[idx + 1] == config.flow
|
||
|
||
# 2) --processing-mode 始终存在且值正确
|
||
assert "--processing-mode" in cmd
|
||
idx = cmd.index("--processing-mode")
|
||
assert cmd[idx + 1] == config.processing_mode
|
||
|
||
# 3) 非空任务列表 → --tasks 存在
|
||
if config.tasks:
|
||
assert "--tasks" in cmd
|
||
idx = cmd.index("--tasks")
|
||
assert set(cmd[idx + 1].split(",")) == set(config.tasks)
|
||
|
||
# 4) 时间窗口参数
|
||
if config.window_mode == "lookback":
|
||
# lookback 模式 → --lookback-hours 和 --overlap-seconds
|
||
if config.lookback_hours is not None:
|
||
assert "--lookback-hours" in cmd
|
||
idx = cmd.index("--lookback-hours")
|
||
assert cmd[idx + 1] == str(config.lookback_hours)
|
||
if config.overlap_seconds is not None:
|
||
assert "--overlap-seconds" in cmd
|
||
idx = cmd.index("--overlap-seconds")
|
||
assert cmd[idx + 1] == str(config.overlap_seconds)
|
||
# lookback 模式不应出现 custom 参数
|
||
assert "--window-start" not in cmd
|
||
assert "--window-end" not in cmd
|
||
else:
|
||
# custom 模式 → --window-start / --window-end
|
||
if config.window_start:
|
||
assert "--window-start" in cmd
|
||
if config.window_end:
|
||
assert "--window-end" in cmd
|
||
# custom 模式不应出现 lookback 参数
|
||
assert "--lookback-hours" not in cmd
|
||
assert "--overlap-seconds" not in cmd
|
||
|
||
# 5) dry_run → --dry-run
|
||
if config.dry_run:
|
||
assert "--dry-run" in cmd
|
||
else:
|
||
assert "--dry-run" not in cmd
|
||
|
||
# 6) store_id → --store-id
|
||
if config.store_id is not None:
|
||
assert "--store-id" in cmd
|
||
idx = cmd.index("--store-id")
|
||
assert cmd[idx + 1] == str(config.store_id)
|
||
else:
|
||
assert "--store-id" not in cmd
|
||
|
||
# 7) fetch_before_verify → 仅 verify_only 模式下生成
|
||
if config.fetch_before_verify and config.processing_mode == "verify_only":
|
||
assert "--fetch-before-verify" in cmd
|
||
else:
|
||
assert "--fetch-before-verify" not in cmd
|
||
|
||
# 8) window_split（非 None 且非 "none"）→ --window-split
|
||
if config.window_split and config.window_split != "none":
|
||
assert "--window-split" in cmd
|
||
idx = cmd.index("--window-split")
|
||
assert cmd[idx + 1] == config.window_split
|
||
if config.window_split_days is not None:
|
||
assert "--window-split-days" in cmd
|
||
else:
|
||
assert "--window-split" not in cmd
|