root/feiqiu-ETL
Template
1
0
Fork 0
Code Pull Requests Packages Releases Wiki Activity

迁移代码到Git

Browse Source
This commit is contained in:
Neo
2025-11-18 02:32:00 +08:00
parent 7f87421678
commit c3749474c6
85 changed files with 185478 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
etl_billiards/loaders/__init__.py Normal file
View File

19
etl_billiards/loaders/base_loader.py Normal file
View File

@@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
"""数据加载器基类"""
class BaseLoader:
"""数据加载器基类"""
def __init__(self, db_ops):
self.db = db_ops
def upsert(self, records: list) -> tuple:
"""
执行UPSERT操作
返回: (inserted_count, updated_count, skipped_count)
"""
raise NotImplementedError("子类需实现 upsert 方法")
def _batch_size(self) -> int:
"""批次大小"""
return 1000

0
etl_billiards/loaders/dimensions/__init__.py Normal file
View File

34
etl_billiards/loaders/dimensions/member.py Normal file
View File

@@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
"""会员维度表加载器"""
from ..base_loader import BaseLoader
class MemberLoader(BaseLoader):
"""会员维度加载器"""
def upsert_members(self, records: list, store_id: int) -> tuple:
"""加载会员数据"""
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.dim_member (
store_id, member_id, member_name, phone, balance,
status, register_time, raw_data
)
VALUES (
%(store_id)s, %(member_id)s, %(member_name)s, %(phone)s, %(balance)s,
%(status)s, %(register_time)s, %(raw_data)s
)
ON CONFLICT (store_id, member_id) DO UPDATE SET
member_name = EXCLUDED.member_name,
phone = EXCLUDED.phone,
balance = EXCLUDED.balance,
status = EXCLUDED.status,
register_time = EXCLUDED.register_time,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(sql, records, page_size=self._batch_size())
return (inserted, updated, 0)

134
etl_billiards/loaders/dimensions/product.py Normal file
View File

@@ -0,0 +1,134 @@
# -*- coding: utf-8 -*-
"""商品维度 + 价格SCD2 加载器"""
from ..base_loader import BaseLoader
from scd.scd2_handler import SCD2Handler
class ProductLoader(BaseLoader):
"""商品维度加载器（dim_product + dim_product_price_scd）"""
def __init__(self, db_ops):
super().__init__(db_ops)
# SCD2 处理器，复用通用逻辑
self.scd_handler = SCD2Handler(db_ops)
def upsert_products(self, records: list, store_id: int) -> tuple:
"""
加载商品维度及价格SCD
返回: (inserted_count, updated_count, skipped_count)
"""
if not records:
return (0, 0, 0)
# 1) 维度主表：billiards.dim_product
sql_base = """
INSERT INTO billiards.dim_product (
store_id,
product_id,
site_product_id,
product_name,
category_id,
category_name,
second_category_id,
unit,
cost_price,
sale_price,
allow_discount,
status,
supplier_id,
barcode,
is_combo,
created_time,
updated_time,
raw_data
)
VALUES (
%(store_id)s,
%(product_id)s,
%(site_product_id)s,
%(product_name)s,
%(category_id)s,
%(category_name)s,
%(second_category_id)s,
%(unit)s,
%(cost_price)s,
%(sale_price)s,
%(allow_discount)s,
%(status)s,
%(supplier_id)s,
%(barcode)s,
%(is_combo)s,
%(created_time)s,
%(updated_time)s,
%(raw_data)s
)
ON CONFLICT (store_id, product_id) DO UPDATE SET
site_product_id = EXCLUDED.site_product_id,
product_name = EXCLUDED.product_name,
category_id = EXCLUDED.category_id,
category_name = EXCLUDED.category_name,
second_category_id = EXCLUDED.second_category_id,
unit = EXCLUDED.unit,
cost_price = EXCLUDED.cost_price,
sale_price = EXCLUDED.sale_price,
allow_discount = EXCLUDED.allow_discount,
status = EXCLUDED.status,
supplier_id = EXCLUDED.supplier_id,
barcode = EXCLUDED.barcode,
is_combo = EXCLUDED.is_combo,
updated_time = COALESCE(EXCLUDED.updated_time, now()),
raw_data = EXCLUDED.raw_data
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql_base,
records,
page_size=self._batch_size(),
)
# 2) 价格 SCD2：billiards.dim_product_price_scd
# 只追踪 price + 类目 + 名称等字段的历史
tracked_fields = [
"product_name",
"category_id",
"category_name",
"second_category_id",
"cost_price",
"sale_price",
"allow_discount",
"status",
]
natural_key = ["store_id", "product_id"]
for rec in records:
effective_date = rec.get("updated_time") or rec.get("created_time")
scd_record = {
"store_id": rec["store_id"],
"product_id": rec["product_id"],
"product_name": rec.get("product_name"),
"category_id": rec.get("category_id"),
"category_name": rec.get("category_name"),
"second_category_id": rec.get("second_category_id"),
"cost_price": rec.get("cost_price"),
"sale_price": rec.get("sale_price"),
"allow_discount": rec.get("allow_discount"),
"status": rec.get("status"),
# 原表中有 raw_data jsonb 字段，这里直接复用 task 传入的 raw_data
"raw_data": rec.get("raw_data"),
}
# 这里我们不强行区分 INSERT/UPDATE/SKIP，对 ETL 统计来说意义不大
self.scd_handler.upsert(
table_name="billiards.dim_product_price_scd",
natural_key=natural_key,
tracked_fields=tracked_fields,
record=scd_record,
effective_date=effective_date,
)
# skipped_count 统一按 0 返回（真正被丢弃的记录在 Task 端已经过滤）
return (inserted, updated, 0)

0
etl_billiards/loaders/dimensions/table.py Normal file
View File

0
etl_billiards/loaders/facts/__init__.py Normal file
View File

42
etl_billiards/loaders/facts/order.py Normal file
View File

@@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
"""订单事实表加载器"""
from ..base_loader import BaseLoader
class OrderLoader(BaseLoader):
"""订单数据加载器"""
def upsert_orders(self, records: list, store_id: int) -> tuple:
"""加载订单数据"""
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.fact_order (
store_id, order_id, order_no, member_id, table_id,
order_time, end_time, total_amount, discount_amount,
final_amount, pay_status, order_status, remark, raw_data
)
VALUES (
%(store_id)s, %(order_id)s, %(order_no)s, %(member_id)s, %(table_id)s,
%(order_time)s, %(end_time)s, %(total_amount)s, %(discount_amount)s,
%(final_amount)s, %(pay_status)s, %(order_status)s, %(remark)s, %(raw_data)s
)
ON CONFLICT (store_id, order_id) DO UPDATE SET
order_no = EXCLUDED.order_no,
member_id = EXCLUDED.member_id,
table_id = EXCLUDED.table_id,
order_time = EXCLUDED.order_time,
end_time = EXCLUDED.end_time,
total_amount = EXCLUDED.total_amount,
discount_amount = EXCLUDED.discount_amount,
final_amount = EXCLUDED.final_amount,
pay_status = EXCLUDED.pay_status,
order_status = EXCLUDED.order_status,
remark = EXCLUDED.remark,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(sql, records, page_size=self._batch_size())
return (inserted, updated, 0)

35
etl_billiards/loaders/facts/payment.py Normal file
View File

@@ -0,0 +1,35 @@
# -*- coding: utf-8 -*-
"""支付记录加载器"""
from ..base_loader import BaseLoader
class PaymentLoader(BaseLoader):
"""支付记录加载器"""
def upsert_payments(self, records: list, store_id: int) -> tuple:
"""加载支付记录"""
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.fact_payment (
store_id, pay_id, order_id, pay_time, pay_amount,
pay_type, pay_status, remark, raw_data
)
VALUES (
%(store_id)s, %(pay_id)s, %(order_id)s, %(pay_time)s, %(pay_amount)s,
%(pay_type)s, %(pay_status)s, %(remark)s, %(raw_data)s
)
ON CONFLICT (store_id, pay_id) DO UPDATE SET
order_id = EXCLUDED.order_id,
pay_time = EXCLUDED.pay_time,
pay_amount = EXCLUDED.pay_amount,
pay_type = EXCLUDED.pay_type,
pay_status = EXCLUDED.pay_status,
remark = EXCLUDED.remark,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(sql, records, page_size=self._batch_size())
return (inserted, updated, 0)