阶段性更新

etl_billiards/api/client.py

@@ -54,42 +54,76 @@ class APIClient:
         resp.raise_for_status()
         return resp.json()
     
+    def iter_paginated(
+        self,
+        endpoint: str,
+        params: dict | None,
+        page_size: int = 200,
+        page_field: str = "pageIndex",
+        size_field: str = "pageSize",
+        data_path: tuple = ("data",),
+        list_key: str | None = None,
+    ):
+        """分页迭代器：逐页拉取数据并产出 (page_no, records, request_params, raw_response)。"""
+        base_params = dict(params or {})
+        page = 1
+
+        while True:
+            page_params = dict(base_params)
+            page_params[page_field] = page
+            page_params[size_field] = page_size
+
+            payload = self.get(endpoint, page_params)
+            records = self._extract_list(payload, data_path, list_key)
+
+            yield page, records, page_params, payload
+
+            if len(records) < page_size:
+                break
+
+            if len(records) == 0:
+                break
+
+            page += 1
+
     def get_paginated(self, endpoint: str, params: dict, page_size: int = 200,
                       page_field: str = "pageIndex", size_field: str = "pageSize",
                       data_path: tuple = ("data",), list_key: str = None) -> tuple:
-        """分页获取数据"""
+        """分页获取数据并将所有记录汇总在一个列表中。"""
         records, pages_meta = [], []
-        page = 1
-        
-        while True:
-            p = dict(params)
-            p[page_field] = page
-            p[size_field] = page_size
-            
-            obj = self.get(endpoint, p)
-            
-            # 解析数据路径
-            cur = obj
-            for k in data_path:
-                if isinstance(cur, dict) and k in cur:
-                    cur = cur[k]
-            
-            if list_key:
-                cur = (cur or {}).get(list_key, [])
-            
-            if not isinstance(cur, list):
-                cur = []
-            
-            records.extend(cur)
-            
-            if len(cur) == 0:
-                break
-            
-            pages_meta.append({"page": page, "request": p, "response": obj})
-            
-            if len(cur) < page_size:
-                break
-            
-            page += 1
-        
+
+        for page_no, page_records, request_params, response in self.iter_paginated(
+            endpoint=endpoint,
+            params=params,
+            page_size=page_size,
+            page_field=page_field,
+            size_field=size_field,
+            data_path=data_path,
+            list_key=list_key,
+        ):
+            records.extend(page_records)
+            pages_meta.append(
+                {"page": page_no, "request": request_params, "response": response}
+            )
+
         return records, pages_meta
+
+    @staticmethod
+    def _extract_list(payload: dict, data_path: tuple, list_key: str | None):
+        """辅助函数：根据 data_path/list_key 提取列表结构。"""
+        cur = payload
+        for key in data_path:
+            if isinstance(cur, dict):
+                cur = cur.get(key)
+            else:
+                cur = None
+            if cur is None:
+                break
+
+        if list_key and isinstance(cur, dict):
+            cur = cur.get(list_key)
+
+        if not isinstance(cur, list):
+            cur = []
+
+        return cur