This commit is contained in:
Neo
2026-01-27 22:45:50 +08:00
parent a6ad343092
commit 4c192e921c
476 changed files with 381543 additions and 5819 deletions

View File

@@ -0,0 +1,25 @@
# fetch-test
用于放置“接口联调/规则验证”的一次性脚本(不影响主流程)。
## 近期记录 vs 历史记录(Former) 对比
脚本:`fetch-test/compare_recent_former_endpoints.py`
默认对比窗口end 为次日 00:00:00与 ETL 窗口一致):
- 近期2025-12-01 ~ 2025-12-15
- 历史2025-08-01 ~ 2025-08-15
运行:
```bash
cd etl_billiards
python fetch-test/compare_recent_former_endpoints.py
```
输出:
- `etl_billiards/fetch-test/recent_vs_former_report.md`
- `etl_billiards/fetch-test/recent_vs_former_report.json`
依赖:
- `.env` 需配置 `API_TOKEN`(或 `FICOO_TOKEN`)与 `STORE_ID`,并保证 `API_BASE` 正确。

View File

@@ -0,0 +1,434 @@
# -*- coding: utf-8 -*-
"""
对比“近期记录”与“历史记录(Former)”接口:
- 是否能正确响应HTTP + API code==0
- 返回字段(基于 sample records 的 JSON path是否一致
默认时间窗口(与 ETL 窗口语义一致end 为次日 00:00:00
- 近期2025-12-01 ~ 2025-12-15end=2025-12-16 00:00:00
- 历史2025-08-01 ~ 2025-08-15end=2025-08-16 00:00:00
"""
from __future__ import annotations
import argparse
import json
import sys
from dataclasses import asdict, dataclass
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Iterable
from dateutil import parser as dtparser
from zoneinfo import ZoneInfo
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from api.client import APIClient
from api.endpoint_routing import derive_former_endpoint as derive_former_endpoint_shared
from config.settings import AppConfig
from models.parsers import TypeParser
from tasks.ods_json_archive_task import EndpointSpec, OdsJsonArchiveTask
CHINESE_NAMES: dict[str, str] = {
"/MemberProfile/GetMemberCardBalanceChange": "会员余额变动",
"/AssistantPerformance/GetOrderAssistantDetails": "助教服务记录",
"/AssistantPerformance/GetAbolitionAssistant": "助教撤销/作废记录",
"/TenantGoods/GetGoodsSalesList": "商品销售记录",
"/Site/GetSiteTableUseDetails": "团购核销记录",
"/Site/GetSiteTableOrderDetails": "台费订单明细",
"/Site/GetTaiFeeAdjustList": "台费调整/优惠记录",
"/GoodsStockManage/QueryGoodsOutboundReceipt": "出库单/出库记录",
"/Promotion/GetOfflineCouponConsumePageList": "平台券核销记录",
"/Order/GetRefundPayLogList": "退款记录",
"/Site/GetAllOrderSettleList": "结账记录",
"/Site/GetRechargeSettleList": "充值结算记录",
"/PayLog/GetPayLogListPage": "支付记录",
}
@dataclass
class EndpointCheckResult:
name_zh: str
recent_endpoint: str
recent_ok: bool
former_endpoint: str
former_ok: bool
has_schema_diff: str # "是" | "否" | "未知"
diff_detail: str
recent_records: int | None = None
former_records: int | None = None
recent_error: str | None = None
former_error: str | None = None
extracted_list_key: str | None = None
def _reconfigure_stdout_utf8():
try:
sys.stdout.reconfigure(encoding="utf-8")
except Exception:
pass
def derive_former_endpoint(endpoint: str) -> str | None:
# backward compatible wrapper: keep local name but delegate to shared router
return derive_former_endpoint_shared(endpoint)
def _parse_day_start(d: str, tz: ZoneInfo) -> datetime:
dt = dtparser.parse(d)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=tz)
else:
dt = dt.astimezone(tz)
return dt.replace(hour=0, minute=0, second=0, microsecond=0)
def _window_from_dates(start_date: str, end_date_inclusive: str, tz: ZoneInfo) -> tuple[datetime, datetime]:
start = _parse_day_start(start_date, tz)
end_inclusive = _parse_day_start(end_date_inclusive, tz)
end_exclusive = end_inclusive + timedelta(days=1)
return start, end_exclusive
def _build_window_params(
window_style: str,
store_id: int,
window_start: datetime,
window_end: datetime,
tz: ZoneInfo,
) -> dict:
if window_style == "none":
return {}
if window_style == "site":
return {"siteId": store_id}
if window_style == "range":
return {
"siteId": store_id,
"rangeStartTime": TypeParser.format_timestamp(window_start, tz),
"rangeEndTime": TypeParser.format_timestamp(window_end, tz),
}
if window_style == "pay":
return {
"siteId": store_id,
"StartPayTime": TypeParser.format_timestamp(window_start, tz),
"EndPayTime": TypeParser.format_timestamp(window_end, tz),
}
return {
"siteId": store_id,
"startTime": TypeParser.format_timestamp(window_start, tz),
"endTime": TypeParser.format_timestamp(window_end, tz),
}
def _extract_records(payload: dict, spec: EndpointSpec) -> tuple[list, str | None]:
# 优先使用 spec.list_key若拿不到数据再尝试自动推断None
records_primary = APIClient._extract_list(payload, spec.data_path, spec.list_key)
if records_primary:
return records_primary, spec.list_key
records_fallback = APIClient._extract_list(payload, spec.data_path, None)
if records_fallback:
return records_fallback, None
return [], spec.list_key
def _walk_paths(obj: Any, prefix: str, out: set[str], max_depth: int, depth: int, sample_list_elems: int):
if depth > max_depth:
return
if isinstance(obj, dict):
for k, v in obj.items():
if not isinstance(k, str):
k = str(k)
p = f"{prefix}.{k}" if prefix else k
out.add(p)
_walk_paths(v, p, out, max_depth, depth + 1, sample_list_elems)
elif isinstance(obj, list):
p = f"{prefix}[]" if prefix else "[]"
out.add(p)
for v in obj[:sample_list_elems]:
_walk_paths(v, p, out, max_depth, depth + 1, sample_list_elems)
def _schema_from_records(records: list, max_records: int, max_depth: int) -> set[str]:
paths: set[str] = set()
for rec in (records or [])[:max_records]:
_walk_paths(rec, "", paths, max_depth=max_depth, depth=0, sample_list_elems=5)
return paths
def _schema_from_data(payload: dict, data_path: tuple[str, ...], max_depth: int) -> set[str]:
cur: Any = payload
for k in data_path:
if isinstance(cur, dict):
cur = cur.get(k)
else:
cur = None
if cur is None:
break
paths: set[str] = set()
_walk_paths(cur, "", paths, max_depth=max_depth, depth=0, sample_list_elems=5)
return paths
def _cell(text: str) -> str:
# markdown table cell escape
s = (text or "").replace("|", "\\|").replace("\n", "<br>")
return s
def _format_diff(recent_paths: set[str], former_paths: set[str], limit: int = 60) -> tuple[str, str]:
if recent_paths == former_paths:
return "", ""
only_recent = sorted(recent_paths - former_paths)
only_former = sorted(former_paths - recent_paths)
parts: list[str] = []
if only_recent:
truncated = only_recent[:limit]
suffix = "" if len(only_recent) <= limit else f" ...(+{len(only_recent) - limit})"
parts.append(f"仅近期({len(only_recent)}): " + ", ".join(truncated) + suffix)
if only_former:
truncated = only_former[:limit]
suffix = "" if len(only_former) <= limit else f" ...(+{len(only_former) - limit})"
parts.append(f"仅历史({len(only_former)}): " + ", ".join(truncated) + suffix)
return "", "\n".join(parts).strip()
def _post_first_page(
client: APIClient,
endpoint: str,
params: dict,
page_size: int,
spec: EndpointSpec,
) -> tuple[dict, list]:
# 只拉取第 1 页,用于“能否响应”与字段对比
payload: dict | None = None
records: list = []
for _, page_records, _, raw in client.iter_paginated(
endpoint=endpoint,
params=params,
page_size=page_size,
data_path=spec.data_path,
list_key=spec.list_key,
page_end=1,
):
payload = raw
records = page_records or []
break
return (payload or {}), (records or [])
def _load_specs_for_range_only() -> list[EndpointSpec]:
# 以 ODS_JSON_ARCHIVE 的 ENDPOINTS 为准,筛选出“可定义时间范围”的接口
specs: list[EndpointSpec] = []
for spec in OdsJsonArchiveTask.ENDPOINTS:
if spec.window_style in ("start_end", "range", "pay"):
specs.append(spec)
return specs
def main() -> int:
_reconfigure_stdout_utf8()
ap = argparse.ArgumentParser()
ap.add_argument("--recent-start", default="2025-12-01")
ap.add_argument("--recent-end", default="2025-12-15")
ap.add_argument("--former-start", default="2025-08-01")
ap.add_argument("--former-end", default="2025-08-15")
ap.add_argument("--page-size", type=int, default=50)
ap.add_argument("--max-records", type=int, default=50)
ap.add_argument("--max-depth", type=int, default=5)
ap.add_argument(
"--out",
default=str(Path(__file__).with_name("recent_vs_former_report.md")),
help="输出 markdown 路径",
)
ap.add_argument(
"--out-json",
default=str(Path(__file__).with_name("recent_vs_former_report.json")),
help="输出 json 路径(含错误信息与统计)",
)
args = ap.parse_args()
cfg = AppConfig.load().config
tz = ZoneInfo(cfg["app"]["timezone"])
store_id = int(cfg["app"]["store_id"])
recent_start, recent_end = _window_from_dates(args.recent_start, args.recent_end, tz)
former_start, former_end = _window_from_dates(args.former_start, args.former_end, tz)
if not cfg["api"].get("token"):
raise SystemExit("缺少 api.token请在 .env 配置 API_TOKEN 或 FICOO_TOKEN")
client = APIClient(
base_url=cfg["api"]["base_url"],
token=cfg["api"]["token"],
timeout=int(cfg["api"].get("timeout_sec") or 20),
retry_max=int(cfg["api"].get("retries", {}).get("max_attempts") or 3),
headers_extra=cfg["api"].get("headers_extra") or {},
)
common_params = cfg["api"].get("params", {}) or {}
if not isinstance(common_params, dict):
common_params = {}
results: list[EndpointCheckResult] = []
specs = _load_specs_for_range_only()
for spec in specs:
name_zh = CHINESE_NAMES.get(spec.endpoint) or Path(spec.endpoint).name
former_endpoint = derive_former_endpoint(spec.endpoint)
# recent
recent_params = dict(common_params)
recent_params.update(_build_window_params(spec.window_style, store_id, recent_start, recent_end, tz))
recent_ok = False
recent_records: list = []
recent_payload: dict = {}
recent_err: str | None = None
try:
recent_payload, recent_records = _post_first_page(
client=client,
endpoint=spec.endpoint,
params=recent_params,
page_size=args.page_size,
spec=spec,
)
recent_ok = True
except Exception as e:
recent_err = f"{type(e).__name__}: {e}"
# former
former_params = dict(common_params)
former_params.update(_build_window_params(spec.window_style, store_id, former_start, former_end, tz))
former_ok = False
former_records: list = []
former_payload: dict = {}
former_err: str | None = None
if not former_endpoint:
former_err = "未提供历史记录接口 path"
else:
try:
former_payload, former_records = _post_first_page(
client=client,
endpoint=former_endpoint,
params=former_params,
page_size=args.page_size,
spec=spec,
)
former_ok = True
except Exception as e:
former_err = f"{type(e).__name__}: {e}"
extracted_key: str | None = spec.list_key
if recent_ok and former_ok:
# 用“更能提取出 records 的方式”来做字段对比
recent_extracted, recent_key_used = _extract_records(recent_payload, spec)
former_extracted, former_key_used = _extract_records(former_payload, spec)
extracted_key = recent_key_used or former_key_used or spec.list_key
if recent_extracted and former_extracted:
recent_schema = _schema_from_records(recent_extracted, args.max_records, args.max_depth)
former_schema = _schema_from_records(former_extracted, args.max_records, args.max_depth)
has_diff, detail = _format_diff(recent_schema, former_schema)
elif (not recent_extracted) and (not former_extracted):
has_diff, detail = "未知", "两侧 records 均为空,无法判断字段差异"
else:
has_diff, detail = (
"未知",
f"一侧 records 为空(近期={len(recent_extracted)} 历史={len(former_extracted)}),无法判断字段差异",
)
else:
if former_endpoint is None:
has_diff, detail = "未知", "无历史记录接口,跳过字段对比"
else:
has_diff, detail = "未知", "请求失败,无法对比字段"
results.append(
EndpointCheckResult(
name_zh=name_zh,
recent_endpoint=spec.endpoint,
recent_ok=recent_ok,
former_endpoint=former_endpoint or "",
former_ok=former_ok,
has_schema_diff=has_diff,
diff_detail=detail,
recent_records=len(recent_records) if recent_ok else None,
former_records=len(former_records) if former_ok else None,
recent_error=recent_err,
former_error=former_err,
extracted_list_key=extracted_key,
)
)
# markdown report
out_md = Path(args.out)
out_json = Path(args.out_json)
out_md.parent.mkdir(parents=True, exist_ok=True)
header = [
"# 近期记录 vs 历史记录(Former) 接口对比报告",
"",
f"- 近期窗口: `{recent_start.isoformat()}` ~ `{recent_end.isoformat()}`end 为次日 00:00:00",
f"- 历史窗口: `{former_start.isoformat()}` ~ `{former_end.isoformat()}`end 为次日 00:00:00",
f"- store_id: `{store_id}`",
f"- base_url: `{cfg['api']['base_url']}`",
"",
"表头:接口名称(中文);近期记录接口 path近期记录是否返回历史记录接口 path历史记录是否返回是否存在返回字段差异差异字段详情。",
"",
"| 接口名称(中文) | 近期记录接口 path | 近期记录是否返回 | 历史记录接口 path | 历史记录是否返回 | 是否存在返回字段差异 | 差异字段详情 |",
"| --- | --- | --- | --- | --- | --- | --- |",
]
rows: list[str] = []
for r in results:
rows.append(
"| "
+ " | ".join(
[
_cell(r.name_zh),
_cell(r.recent_endpoint),
"" if r.recent_ok else "",
_cell(r.former_endpoint),
"" if r.former_ok else "",
_cell(r.has_schema_diff),
_cell(r.diff_detail or ""),
]
)
+ " |"
)
out_md.write_text("\n".join(header + rows) + "\n", encoding="utf-8")
out_json.write_text(
json.dumps(
{
"recent_window": {"start": recent_start.isoformat(), "end": recent_end.isoformat()},
"former_window": {"start": former_start.isoformat(), "end": former_end.isoformat()},
"store_id": store_id,
"base_url": cfg["api"]["base_url"],
"results": [asdict(r) for r in results],
},
ensure_ascii=False,
indent=2,
)
+ "\n",
encoding="utf-8",
)
print(f"OK: wrote {out_md}")
print(f"OK: wrote {out_json}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,196 @@
{
"recent_window": {
"start": "2025-12-01T00:00:00+08:00",
"end": "2025-12-16T00:00:00+08:00"
},
"former_window": {
"start": "2025-08-01T00:00:00+08:00",
"end": "2025-08-16T00:00:00+08:00"
},
"store_id": 2790685415443269,
"base_url": "https://pc.ficoo.vip/apiprod/admin/v1/",
"results": [
{
"name_zh": "会员余额变动",
"recent_endpoint": "/MemberProfile/GetMemberCardBalanceChange",
"recent_ok": true,
"former_endpoint": "/MemberProfile/GetFormerMemberCardBalanceChange",
"former_ok": true,
"has_schema_diff": "否",
"diff_detail": "",
"recent_records": 50,
"former_records": 50,
"recent_error": null,
"former_error": null,
"extracted_list_key": null
},
{
"name_zh": "助教服务记录",
"recent_endpoint": "/AssistantPerformance/GetOrderAssistantDetails",
"recent_ok": true,
"former_endpoint": "/AssistantPerformance/GetFormerOrderAssistantDetails",
"former_ok": true,
"has_schema_diff": "否",
"diff_detail": "",
"recent_records": 50,
"former_records": 50,
"recent_error": null,
"former_error": null,
"extracted_list_key": "orderAssistantDetails"
},
{
"name_zh": "助教撤销/作废记录",
"recent_endpoint": "/AssistantPerformance/GetAbolitionAssistant",
"recent_ok": true,
"former_endpoint": "无",
"former_ok": false,
"has_schema_diff": "未知",
"diff_detail": "无历史记录接口,跳过字段对比",
"recent_records": 18,
"former_records": null,
"recent_error": null,
"former_error": "未提供历史记录接口 path",
"extracted_list_key": "abolitionAssistants"
},
{
"name_zh": "商品销售记录",
"recent_endpoint": "/TenantGoods/GetGoodsSalesList",
"recent_ok": true,
"former_endpoint": "/TenantGoods/GetFormerGoodsSalesList",
"former_ok": true,
"has_schema_diff": "否",
"diff_detail": "",
"recent_records": 50,
"former_records": 50,
"recent_error": null,
"former_error": null,
"extracted_list_key": "orderGoodsLedgers"
},
{
"name_zh": "团购核销记录",
"recent_endpoint": "/Site/GetSiteTableUseDetails",
"recent_ok": true,
"former_endpoint": "/Site/GetSiteTableUseDetails",
"former_ok": true,
"has_schema_diff": "未知",
"diff_detail": "一侧 records 为空(近期=50 历史=0无法判断字段差异",
"recent_records": 50,
"former_records": 0,
"recent_error": null,
"former_error": null,
"extracted_list_key": "siteTableUseDetailsList"
},
{
"name_zh": "台费订单明细",
"recent_endpoint": "/Site/GetSiteTableOrderDetails",
"recent_ok": true,
"former_endpoint": "/Site/GetFormerSiteTableOrderDetails",
"former_ok": true,
"has_schema_diff": "否",
"diff_detail": "",
"recent_records": 50,
"former_records": 50,
"recent_error": null,
"former_error": null,
"extracted_list_key": "siteTableUseDetailsList"
},
{
"name_zh": "台费调整/优惠记录",
"recent_endpoint": "/Site/GetTaiFeeAdjustList",
"recent_ok": true,
"former_endpoint": "/Site/GetFormerTaiFeeAdjustList",
"former_ok": true,
"has_schema_diff": "否",
"diff_detail": "",
"recent_records": 50,
"former_records": 50,
"recent_error": null,
"former_error": null,
"extracted_list_key": "taiFeeAdjustInfos"
},
{
"name_zh": "出库单/出库记录",
"recent_endpoint": "/GoodsStockManage/QueryGoodsOutboundReceipt",
"recent_ok": true,
"former_endpoint": "/GoodsStockManage/QueryFormerGoodsOutboundReceipt",
"former_ok": true,
"has_schema_diff": "否",
"diff_detail": "",
"recent_records": 50,
"former_records": 50,
"recent_error": null,
"former_error": null,
"extracted_list_key": "queryDeliveryRecordsList"
},
{
"name_zh": "平台券核销记录",
"recent_endpoint": "/Promotion/GetOfflineCouponConsumePageList",
"recent_ok": true,
"former_endpoint": "/Promotion/GetOfflineCouponConsumePageList",
"former_ok": true,
"has_schema_diff": "否",
"diff_detail": "",
"recent_records": 50,
"former_records": 50,
"recent_error": null,
"former_error": null,
"extracted_list_key": null
},
{
"name_zh": "退款记录",
"recent_endpoint": "/Order/GetRefundPayLogList",
"recent_ok": true,
"former_endpoint": "无",
"former_ok": false,
"has_schema_diff": "未知",
"diff_detail": "无历史记录接口,跳过字段对比",
"recent_records": 5,
"former_records": null,
"recent_error": null,
"former_error": "未提供历史记录接口 path",
"extracted_list_key": null
},
{
"name_zh": "结账记录",
"recent_endpoint": "/Site/GetAllOrderSettleList",
"recent_ok": true,
"former_endpoint": "/Site/GetFormerOrderSettleList",
"former_ok": true,
"has_schema_diff": "否",
"diff_detail": "",
"recent_records": 50,
"former_records": 50,
"recent_error": null,
"former_error": null,
"extracted_list_key": "settleList"
},
{
"name_zh": "充值结算记录",
"recent_endpoint": "/Site/GetRechargeSettleList",
"recent_ok": true,
"former_endpoint": "/Site/GetFormerRechargeSettleList",
"former_ok": true,
"has_schema_diff": "否",
"diff_detail": "",
"recent_records": 39,
"former_records": 47,
"recent_error": null,
"former_error": null,
"extracted_list_key": "settleList"
},
{
"name_zh": "支付记录",
"recent_endpoint": "/PayLog/GetPayLogListPage",
"recent_ok": true,
"former_endpoint": "/PayLog/GetFormerPayLogListPage",
"former_ok": true,
"has_schema_diff": "否",
"diff_detail": "",
"recent_records": 50,
"former_records": 50,
"recent_error": null,
"former_error": null,
"extracted_list_key": null
}
]
}

View File

@@ -0,0 +1,24 @@
# 近期记录 vs 历史记录(Former) 接口对比报告
- 近期窗口: `2025-12-01T00:00:00+08:00` ~ `2025-12-16T00:00:00+08:00`end 为次日 00:00:00
- 历史窗口: `2025-08-01T00:00:00+08:00` ~ `2025-08-16T00:00:00+08:00`end 为次日 00:00:00
- store_id: `2790685415443269`
- base_url: `https://pc.ficoo.vip/apiprod/admin/v1/`
表头:接口名称(中文);近期记录接口 path近期记录是否返回历史记录接口 path历史记录是否返回是否存在返回字段差异差异字段详情。
| 接口名称(中文) | 近期记录接口 path | 近期记录是否返回 | 历史记录接口 path | 历史记录是否返回 | 是否存在返回字段差异 | 差异字段详情 |
| --- | --- | --- | --- | --- | --- | --- |
| 会员余额变动 | /MemberProfile/GetMemberCardBalanceChange | 是 | /MemberProfile/GetFormerMemberCardBalanceChange | 是 | 否 | |
| 助教服务记录 | /AssistantPerformance/GetOrderAssistantDetails | 是 | /AssistantPerformance/GetFormerOrderAssistantDetails | 是 | 否 | |
| 助教撤销/作废记录 | /AssistantPerformance/GetAbolitionAssistant | 是 | 无 | 否 | 未知 | 无历史记录接口,跳过字段对比 |
| 商品销售记录 | /TenantGoods/GetGoodsSalesList | 是 | /TenantGoods/GetFormerGoodsSalesList | 是 | 否 | |
| 团购核销记录 | /Site/GetSiteTableUseDetails | 是 | /Site/GetSiteTableUseDetails | 是 | 未知 | 一侧 records 为空(近期=50 历史=0无法判断字段差异 |
| 台费订单明细 | /Site/GetSiteTableOrderDetails | 是 | /Site/GetFormerSiteTableOrderDetails | 是 | 否 | |
| 台费调整/优惠记录 | /Site/GetTaiFeeAdjustList | 是 | /Site/GetFormerTaiFeeAdjustList | 是 | 否 | |
| 出库单/出库记录 | /GoodsStockManage/QueryGoodsOutboundReceipt | 是 | /GoodsStockManage/QueryFormerGoodsOutboundReceipt | 是 | 否 | |
| 平台券核销记录 | /Promotion/GetOfflineCouponConsumePageList | 是 | /Promotion/GetOfflineCouponConsumePageList | 是 | 否 | |
| 退款记录 | /Order/GetRefundPayLogList | 是 | 无 | 否 | 未知 | 无历史记录接口,跳过字段对比 |
| 结账记录 | /Site/GetAllOrderSettleList | 是 | /Site/GetFormerOrderSettleList | 是 | 否 | |
| 充值结算记录 | /Site/GetRechargeSettleList | 是 | /Site/GetFormerRechargeSettleList | 是 | 否 | |
| 支付记录 | /PayLog/GetPayLogListPage | 是 | /PayLog/GetFormerPayLogListPage | 是 | 否 | |