root/feiqiu-ETL
Template
1
0
Fork 0
Code Pull Requests Packages Releases Wiki Activity

更新20260201-1

Browse Source
This commit is contained in:
Neo
2026-02-01 22:04:15 +08:00
parent 076f5755ca
commit 9b2c2c5c78
20 changed files with 32463 additions and 408 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1184
tmp/py_inventory.md Normal file
View File

File diff suppressed because it is too large Load Diff

31
tmp/query_skill_mapping.py Normal file
View File

@@ -0,0 +1,31 @@
import psycopg2
import sys
sys.stdout.reconfigure(encoding='utf-8')
dsn = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test'
conn = psycopg2.connect(dsn)
cur = conn.cursor()
# 查询 skill_id 和 skill_name 的对应关系及数量
cur.execute("""
SELECT
skill_id,
skill_name,
COUNT(*) as count
FROM billiards_dwd.dwd_assistant_service_log
GROUP BY skill_id, skill_name
ORDER BY skill_id, count DESC
""")
results = cur.fetchall()
print("| skill_id | skill_name | 记录数 |")
print("|----------|------------|--------|")
for row in results:
skill_id, skill_name, count = row
name = skill_name if skill_name else "(NULL)"
print(f"| {skill_id} | {name} | {count} |")
print(f"\n共 {len(results)} 种组合")
conn.close()

85
tmp/task_inventory.md Normal file
View File

@@ -0,0 +1,85 @@
# Task Inventory
## Registered Tasks (from task_registry.py)
| Task Code | Class |
|---|---|
| PRODUCTS | ProductsTask |
| TABLES | TablesTask |
| MEMBERS | MembersTask |
| ASSISTANTS | AssistantsTask |
| PACKAGES_DEF | PackagesDefTask |
| ORDERS | OrdersTask |
| PAYMENTS | PaymentsTask |
| REFUNDS | RefundsTask |
| COUPON_USAGE | CouponUsageTask |
| INVENTORY_CHANGE | InventoryChangeTask |
| TOPUPS | TopupsTask |
| TABLE_DISCOUNT | TableDiscountTask |
| ASSISTANT_ABOLISH | AssistantAbolishTask |
| LEDGER | LedgerTask |
| TICKET_DWD | TicketDwdTask |
| MANUAL_INGEST | ManualIngestTask |
| PAYMENTS_DWD | PaymentsDwdTask |
| MEMBERS_DWD | MembersDwdTask |
| INIT_ODS_SCHEMA | InitOdsSchemaTask |
| INIT_DWD_SCHEMA | InitDwdSchemaTask |
| DWD_LOAD_FROM_ODS | DwdLoadTask |
| DWD_QUALITY_CHECK | DwdQualityTask |
| ODS_JSON_ARCHIVE | OdsJsonArchiveTask |
| CHECK_CUTOFF | CheckCutoffTask |
| DATA_INTEGRITY_CHECK | DataIntegrityTask |
| INIT_DWS_SCHEMA | InitDwsSchemaTask |
| DWS_BUILD_ORDER_SUMMARY | DwsBuildOrderSummaryTask |
## Task Class Details
| Task Code | Class | File | Flow | API Fetch | Key Calls |
|---|---|---|---|---|---|
| | DwdLoadTask | etl_billiards/tasks/dwd_load_task.py | BaseTask.execute (extract/transform/load) | No | isinstance, c.lower, ', '.join, cur.execute, self._format_table, select_exprs.append, values.append, self._cast_expr |
| | DwdQualityTask | etl_billiards/tasks/dwd_quality_task.py | BaseTask.execute (extract/transform/load) | No | cur.execute, self._split_table_name, cur.fetchone, self._get_numeric_amount_columns, Path, self.REPORT_PATH.parent.mkdir, self.REPORT_PATH.write_text, self.logger.info |
| | InitDwdSchemaTask | etl_billiards/tasks/init_dwd_schema_task.py | BaseTask.execute (extract/transform/load) | No | Path, self.config.get, self.logger.info, cur.execute, dwd_path.exists, FileNotFoundError, dwd_path.read_text, self.db.conn.cursor |
| | InitOdsSchemaTask | etl_billiards/tasks/init_schema_task.py | BaseTask.execute (extract/transform/load) | No | Path, self.config.get, self.logger.info, FileNotFoundError, cur.execute, ods_sql_raw.find, ods_sql_raw.splitlines, '\n'.join |
| | ManualIngestTask | etl_billiards/tasks/manual_ingest_task.py | custom execute | No | isinstance, cur.execute, row_vals.append, site_profile.get, merged_rec.get, self.config.get, any, ', '.join |
| ASSISTANTS | AssistantsTask | etl_billiards/tasks/assistants_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_timestamp, extracted.get, TypeParser.parse_int, TypeParser.parse_decimal, self._merge_common_params, self.api.get_paginated, AssistantLoader |
| ASSISTANT_ABOLISH | AssistantAbolishTask | etl_billiards/tasks/assistant_abolish_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_int, extracted.get, TypeParser.format_timestamp, self._merge_common_params, self.api.get_paginated, AssistantAbolishLoader, loader.upsert_records |
| CHECK_CUTOFF | CheckCutoffTask | etl_billiards/tasks/check_cutoff_task.py | custom execute | No | r.get, _ts, self.logger.info, self.db.query, row.get, sorted, self.config.get, min |
| COUPON_USAGE | CouponUsageTask | etl_billiards/tasks/coupon_usage_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_int, extracted.get, TypeParser.parse_decimal, TypeParser.parse_timestamp, TypeParser.format_timestamp, self._merge_common_params, self.api.get_paginated |
| DWS_BUILD_ORDER_SUMMARY | DwsBuildOrderSummaryTask | etl_billiards/tasks/dws_build_order_summary_task.py | custom execute | No | self.config.get, self.logger.info, cur.execute, load_result.get, delete_args.append, bool, _jsonable_date, _as_date |
| INIT_DWS_SCHEMA | InitDwsSchemaTask | etl_billiards/tasks/init_dws_schema_task.py | BaseTask.execute (extract/transform/load) | No | Path, self.config.get, self.logger.info, cur.execute, bool, dws_path.exists, FileNotFoundError, dws_path.read_text |
| INVENTORY_CHANGE | InventoryChangeTask | etl_billiards/tasks/inventory_change_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_int, extracted.get, TypeParser.format_timestamp, self._merge_common_params, self.api.get_paginated, InventoryChangeLoader, loader.upsert_changes |
| LEDGER | LedgerTask | etl_billiards/tasks/ledger_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_int, TypeParser.parse_decimal, TypeParser.parse_timestamp, extracted.get, TypeParser.format_timestamp, self._merge_common_params, self.api.get_paginated |
| MEMBERS | MembersTask | etl_billiards/tasks/members_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, extracted.get, self._merge_common_params, self.api.get_paginated, MemberLoader, loader.upsert_members, self._parse_member, TypeParser.parse_int |
| ODS_ASSISTANT_ABOLISH | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_ASSISTANT_ACCOUNT | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_ASSISTANT_LEDGER | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_GOODS_CATEGORY | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_GROUP_BUY_REDEMPTION | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_GROUP_PACKAGE | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_INVENTORY_CHANGE | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_INVENTORY_STOCK | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_JSON_ARCHIVE | OdsJsonArchiveTask | etl_billiards/tasks/ods_json_archive_task.py | BaseTask.execute (extract/transform/load) | No | EndpointSpec, Path, TypeParser.format_timestamp, self.logger.info, dump_json, self.config.get, endpoint_to_filename, (rec or {}).get |
| ODS_MEMBER | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_MEMBER_BALANCE | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_MEMBER_CARD | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_PAYMENT | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_PLATFORM_COUPON | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_RECHARGE_SETTLE | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_REFUND | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_SETTLEMENT_RECORDS | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_SETTLEMENT_TICKET | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_STORE_GOODS | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_STORE_GOODS_SALES | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_TABLES | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_TABLE_FEE_DISCOUNT | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_TABLE_USE | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ODS_TENANT_GOODS | (dynamic) | etl_billiards/tasks/ods_tasks.py | BaseOdsTask.execute | Yes | self.api.iter_paginated, _insert_records_schema_aware |
| ORDERS | OrdersTask | etl_billiards/tasks/orders_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_int, TypeParser.parse_decimal, extracted.get, TypeParser.format_timestamp, TypeParser.parse_timestamp, self._merge_common_params, self.api.get_paginated |
| PACKAGES_DEF | PackagesDefTask | etl_billiards/tasks/packages_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_int, extracted.get, TypeParser.parse_decimal, TypeParser.parse_timestamp, self._merge_common_params, self.api.get_paginated, PackageDefinitionLoader |
| PAYMENTS | PaymentsTask | etl_billiards/tasks/payments_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_int, TypeParser.parse_decimal, extracted.get, TypeParser.format_timestamp, TypeParser.parse_timestamp, self._merge_common_params, self.api.get_paginated |
| PRODUCTS | ProductsTask | etl_billiards/tasks/products_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_int, extracted.get, TypeParser.parse_decimal, TypeParser.parse_timestamp, self._merge_common_params, self.api.get_paginated, ProductLoader |
| REFUNDS | RefundsTask | etl_billiards/tasks/refunds_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_int, TypeParser.parse_decimal, extracted.get, TypeParser.parse_timestamp, TypeParser.format_timestamp, self._merge_common_params, self.api.get_paginated |
| TABLES | TablesTask | etl_billiards/tasks/tables_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_int, extracted.get, self._merge_common_params, self.api.get_paginated, TableLoader, loader.upsert_tables, self._parse_table |
| TABLE_DISCOUNT | TableDiscountTask | etl_billiards/tasks/table_discount_task.py | BaseTask.execute (extract/transform/load) | Yes | raw.get, TypeParser.parse_int, table_profile.get, extracted.get, TypeParser.format_timestamp, self._merge_common_params, self.api.get_paginated, TableDiscountLoader |
| TICKET_DWD | TicketDwdTask | etl_billiards/tasks/ticket_dwd_task.py | custom execute | No | self.logger.info, self.get_task_code, self._get_time_window, build_window_segments, TicketLoader, self.config.get, enumerate, self.iter_ods_rows |
| TOPUPS | TopupsTask | etl_billiards/tasks/topups_task.py | BaseTask.execute (extract/transform/load) | Yes | node.get, TypeParser.parse_decimal, TypeParser.parse_int, extracted.get, raw.get, TypeParser.parse_timestamp, TypeParser.format_timestamp, self._merge_common_params |