代码迁移

etl_billiards/api/client.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+"""API客户端"""
+import requests
+from urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter
+
+class APIClient:
+    """HTTP API客户端"""
+    
+    def __init__(self, base_url: str, token: str = None, timeout: int = 20,
+                 retry_max: int = 3, headers_extra: dict = None):
+        self.base_url = base_url.rstrip("/")
+        self.token = token
+        self.timeout = timeout
+        self.retry_max = retry_max
+        self.headers_extra = headers_extra or {}
+        self._session = None
+    
+    def _get_session(self):
+        """获取或创建会话"""
+        if self._session is None:
+            self._session = requests.Session()
+            
+            retries = max(0, int(self.retry_max) - 1)
+            retry = Retry(
+                total=None,
+                connect=retries,
+                read=retries,
+                status=retries,
+                allowed_methods=frozenset(["GET"]),
+                status_forcelist=(429, 500, 502, 503, 504),
+                backoff_factor=1.0,
+                respect_retry_after_header=True,
+                raise_on_status=False,
+            )
+            
+            adapter = HTTPAdapter(max_retries=retry)
+            self._session.mount("http://", adapter)
+            self._session.mount("https://", adapter)
+            
+            if self.headers_extra:
+                self._session.headers.update(self.headers_extra)
+        
+        return self._session
+    
+    def get(self, endpoint: str, params: dict = None) -> dict:
+        """执行GET请求"""
+        url = f"{self.base_url}/{endpoint.lstrip('/')}"
+        headers = {"Authorization": self.token} if self.token else {}
+        headers.update(self.headers_extra)
+        
+        sess = self._get_session()
+        resp = sess.get(url, headers=headers, params=params, timeout=self.timeout)
+        resp.raise_for_status()
+        return resp.json()
+    
+    def get_paginated(self, endpoint: str, params: dict, page_size: int = 200,
+                      page_field: str = "pageIndex", size_field: str = "pageSize",
+                      data_path: tuple = ("data",), list_key: str = None) -> tuple:
+        """分页获取数据"""
+        records, pages_meta = [], []
+        page = 1
+        
+        while True:
+            p = dict(params)
+            p[page_field] = page
+            p[size_field] = page_size
+            
+            obj = self.get(endpoint, p)
+            
+            # 解析数据路径
+            cur = obj
+            for k in data_path:
+                if isinstance(cur, dict) and k in cur:
+                    cur = cur[k]
+            
+            if list_key:
+                cur = (cur or {}).get(list_key, [])
+            
+            if not isinstance(cur, list):
+                cur = []
+            
+            records.extend(cur)
+            
+            if len(cur) == 0:
+                break
+            
+            pages_meta.append({"page": page, "request": p, "response": obj})
+            
+            if len(cur) < page_size:
+                break
+            
+            page += 1
+        
+        return records, pages_meta