root/feiqiu-ETL
Template
1
0
Fork 0
Code Pull Requests Packages Releases Wiki Activity

ODS 完成

Browse Source
This commit is contained in:
Neo
2025-11-30 07:18:55 +08:00
parent cbd16a39ba
commit b9b050bb5d
28 changed files with 41867 additions and 977 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37238
etl_billiards/tests/source-data-doc/结账记录.json Normal file
View File

File diff suppressed because it is too large Load Diff

49
etl_billiards/tests/unit/task_test_utils.py
View File

@@ -68,6 +68,7 @@ def create_test_config(mode: str, archive_dir: Path, temp_dir: Path) -> AppConfi
archive_dir.mkdir(parents=True, exist_ok=True)
temp_dir.mkdir(parents=True, exist_ok=True)
flow = "FULL" if str(mode or "").upper() == "ONLINE" else "INGEST_ONLY"
overrides = {
"app": {"store_id": DEFAULT_STORE_ID, "timezone": "Asia/Taipei"},
"db": {"dsn": "postgresql://user:pass@localhost:5432/etl_billiards_test"},
@@ -77,10 +78,10 @@ def create_test_config(mode: str, archive_dir: Path, temp_dir: Path) -> AppConfi
"timeout_sec": 3,
"page_size": 50,
},
"testing": {
"mode": mode,
"json_archive_dir": str(archive_dir),
"temp_json_dir": str(temp_dir),
"pipeline": {
"flow": flow,
"fetch_root": str(temp_dir / "json_fetch"),
"ingest_source_dir": str(archive_dir),
},
"io": {
"export_root": str(temp_dir / "export"),
@@ -191,8 +192,8 @@ class FakeAPIClient:
endpoint: str,
params=None,
page_size: int = 200,
page_field: str = "pageIndex",
size_field: str = "pageSize",
page_field: str = "page",
size_field: str = "limit",
data_path: Tuple[str, ...] = (),
list_key: str | None = None,
):
@@ -228,8 +229,8 @@ class OfflineAPIClient:
endpoint: str,
params=None,
page_size: int = 200,
page_field: str = "pageIndex",
size_field: str = "pageSize",
page_field: str = "page",
size_field: str = "limit",
data_path: Tuple[str, ...] = (),
list_key: str | None = None,
):
@@ -328,7 +329,7 @@ TASK_SPECS: List[TaskSpec] = [
code="PRODUCTS",
task_cls=ProductsTask,
endpoint="/TenantGoods/QueryTenantGoods",
data_path=("data",),
data_path=("data", "tenantGoodsList"),
sample_records=[
{
"siteGoodsId": 101,
@@ -379,7 +380,7 @@ TASK_SPECS: List[TaskSpec] = [
code="MEMBERS",
task_cls=MembersTask,
endpoint="/MemberProfile/GetTenantMemberList",
data_path=("data",),
data_path=("data", "tenantMemberInfos"),
sample_records=[
{
"memberId": 401,
@@ -394,7 +395,7 @@ TASK_SPECS: List[TaskSpec] = [
TaskSpec(
code="ASSISTANTS",
task_cls=AssistantsTask,
endpoint="/Assistant/List",
endpoint="/PersonnelManagement/SearchAssistantInfo",
data_path=("data", "assistantInfos"),
sample_records=[
{
@@ -432,7 +433,7 @@ TASK_SPECS: List[TaskSpec] = [
TaskSpec(
code="PACKAGES_DEF",
task_cls=PackagesDefTask,
endpoint="/Package/List",
endpoint="/PackageCoupon/QueryPackageCouponList",
data_path=("data", "packageCouponList"),
sample_records=[
{
@@ -462,8 +463,8 @@ TASK_SPECS: List[TaskSpec] = [
TaskSpec(
code="ORDERS",
task_cls=OrdersTask,
endpoint="/order/list",
data_path=("data",),
endpoint="/Site/GetAllOrderSettleList",
data_path=("data", "settleList"),
sample_records=[
{
"orderId": 701,
@@ -484,7 +485,7 @@ TASK_SPECS: List[TaskSpec] = [
TaskSpec(
code="PAYMENTS",
task_cls=PaymentsTask,
endpoint="/pay/records",
endpoint="/PayLog/GetPayLogListPage",
data_path=("data",),
sample_records=[
{
@@ -501,8 +502,8 @@ TASK_SPECS: List[TaskSpec] = [
TaskSpec(
code="REFUNDS",
task_cls=RefundsTask,
endpoint="/Pay/RefundList",
data_path=(),
endpoint="/Order/GetRefundPayLogList",
data_path=("data",),
sample_records=[
{
"id": 901,
@@ -530,8 +531,8 @@ TASK_SPECS: List[TaskSpec] = [
TaskSpec(
code="COUPON_USAGE",
task_cls=CouponUsageTask,
endpoint="/Coupon/UsageList",
data_path=(),
endpoint="/Promotion/GetOfflineCouponConsumePageList",
data_path=("data",),
sample_records=[
{
"id": 1001,
@@ -560,7 +561,7 @@ TASK_SPECS: List[TaskSpec] = [
TaskSpec(
code="INVENTORY_CHANGE",
task_cls=InventoryChangeTask,
endpoint="/Inventory/ChangeList",
endpoint="/GoodsStockManage/QueryGoodsOutboundReceipt",
data_path=("data", "queryDeliveryRecordsList"),
sample_records=[
{
@@ -584,7 +585,7 @@ TASK_SPECS: List[TaskSpec] = [
TaskSpec(
code="TOPUPS",
task_cls=TopupsTask,
endpoint="/Topup/SettleList",
endpoint="/Site/GetRechargeSettleList",
data_path=("data", "settleList"),
sample_records=[
{
@@ -623,7 +624,7 @@ TASK_SPECS: List[TaskSpec] = [
TaskSpec(
code="TABLE_DISCOUNT",
task_cls=TableDiscountTask,
endpoint="/Table/AdjustList",
endpoint="/Site/GetTaiFeeAdjustList",
data_path=("data", "taiFeeAdjustInfos"),
sample_records=[
{
@@ -653,7 +654,7 @@ TASK_SPECS: List[TaskSpec] = [
TaskSpec(
code="ASSISTANT_ABOLISH",
task_cls=AssistantAbolishTask,
endpoint="/Assistant/AbolishList",
endpoint="/AssistantPerformance/GetAbolitionAssistant",
data_path=("data", "abolitionAssistants"),
sample_records=[
{
@@ -674,7 +675,7 @@ TASK_SPECS: List[TaskSpec] = [
TaskSpec(
code="LEDGER",
task_cls=LedgerTask,
endpoint="/Assistant/LedgerList",
endpoint="/AssistantPerformance/GetOrderAssistantDetails",
data_path=("data", "orderAssistantDetails"),
sample_records=[
{

6
etl_billiards/tests/unit/test_ods_tasks.py
View File

@@ -29,7 +29,7 @@ def test_ods_order_settle_ingest(tmp_path):
"anyField": "value",
}
]
api = FakeAPIClient({"/order/list": sample})
api = FakeAPIClient({"/Site/GetAllOrderSettleList": sample})
task_cls = ODS_TASK_CLASSES["ODS_ORDER_SETTLE"]
with get_db_operations() as db_ops:
@@ -42,7 +42,7 @@ def test_ods_order_settle_ingest(tmp_path):
row = db_ops.upserts[0]["rows"][0]
assert row["order_settle_id"] == 701
assert row["order_trade_no"] == 8001
assert row["source_endpoint"] == "/order/list"
assert row["source_endpoint"] == "/Site/GetAllOrderSettleList"
assert '"orderSettleId": 701' in row["payload"]
@@ -57,7 +57,7 @@ def test_ods_payment_ingest(tmp_path):
"payAmount": "100.00",
}
]
api = FakeAPIClient({"/pay/records": sample})
api = FakeAPIClient({"/PayLog/GetPayLogListPage": sample})
task_cls = ODS_TASK_CLASSES["ODS_PAYMENT"]
with get_db_operations() as db_ops: