80 lines
2.7 KiB
Python
80 lines
2.7 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""任务配置 Pydantic 模型
|
||
|
||
定义 TaskConfigSchema 及相关模型，用于前后端传输和 CLIBuilder 消费。
|
||
"""
|
||
|
||
from typing import Any
|
||
|
||
from pydantic import BaseModel, model_validator
|
||
|
||
|
||
class TaskConfigSchema(BaseModel):
|
||
"""任务配置 — 前后端传输格式
|
||
|
||
字段与 CLI 参数的映射关系：
|
||
- flow → --flow（Flow ID，7 种之一）
|
||
- processing_mode → --processing-mode（4 种处理模式）
|
||
- tasks → --tasks（逗号分隔）
|
||
- dry_run → --dry-run（布尔标志）
|
||
- window_mode → 决定使用 lookback 还是 custom 时间窗口（仅前端逻辑，不直接映射 CLI 参数）
|
||
- window_start → --window-start
|
||
- window_end → --window-end
|
||
- window_split → --window-split
|
||
- window_split_days → --window-split-days
|
||
- lookback_hours → --lookback-hours
|
||
- overlap_seconds → --overlap-seconds
|
||
- fetch_before_verify → --fetch-before-verify（布尔标志）
|
||
- store_id → --store-id（由后端从 JWT 注入，前端不传）
|
||
- dwd_only_tables → 传入 extra_args 或未来扩展
|
||
"""
|
||
|
||
tasks: list[str]
|
||
# CHANGE [2026-02-20] intent: pipeline → flow，统一命名（消除历史别名）
|
||
flow: str = "api_ods_dwd"
|
||
processing_mode: str = "increment_only"
|
||
dry_run: bool = False
|
||
window_mode: str = "lookback"
|
||
window_start: str | None = None
|
||
window_end: str | None = None
|
||
window_split: str | None = None
|
||
window_split_days: int | None = None
|
||
lookback_hours: int = 24
|
||
overlap_seconds: int = 600
|
||
fetch_before_verify: bool = False
|
||
skip_ods_when_fetch_before_verify: bool = False
|
||
ods_use_local_json: bool = False
|
||
store_id: int | None = None
|
||
dwd_only_tables: list[str] | None = None
|
||
force_full: bool = False
|
||
# Pipeline 调优参数（可选，不传则使用 ETL 默认值）
|
||
pipeline_workers: int | None = None
|
||
pipeline_batch_size: int | None = None
|
||
pipeline_rate_min: float | None = None
|
||
pipeline_rate_max: float | None = None
|
||
extra_args: dict[str, Any] = {}
|
||
|
||
@model_validator(mode="after")
|
||
def validate_window(self) -> "TaskConfigSchema":
|
||
"""验证时间窗口：结束日期不早于开始日期"""
|
||
if self.window_start and self.window_end:
|
||
if self.window_end < self.window_start:
|
||
raise ValueError("window_end 不能早于 window_start")
|
||
return self
|
||
|
||
|
||
class FlowDefinition(BaseModel):
|
||
"""执行流程（Flow）定义"""
|
||
|
||
id: str
|
||
name: str
|
||
layers: list[str]
|
||
|
||
|
||
class ProcessingModeDefinition(BaseModel):
|
||
"""处理模式定义"""
|
||
|
||
id: str
|
||
name: str
|
||
description: str
|