root/ZQYY.FQ-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

初始提交：飞球 ETL 系统全量代码

Browse Source
This commit is contained in:
Neo
2026-02-13 08:05:34 +08:00
commit 3c51f5485d
441 changed files with 117631 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
loaders/__init__.py Normal file
View File

23
loaders/base_loader.py Normal file
View File

@@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
"""数据加载器基类"""
import logging
class BaseLoader:
"""数据加载器基类"""
def __init__(self, db_ops, logger=None):
self.db = db_ops
self.logger = logger or logging.getLogger(self.__class__.__name__)
def upsert(self, records: list) -> tuple:
"""
执行 UPSERT 操作
返回: (inserted_count, updated_count, skipped_count)
"""
raise NotImplementedError("子类需实现 upsert 方法")
def _batch_size(self) -> int:
"""批次大小"""
return 1000

0
loaders/dimensions/__init__.py Normal file
View File

114
loaders/dimensions/assistant.py Normal file
View File

@@ -0,0 +1,114 @@
# -*- coding: utf-8 -*-
"""助教维度加载器"""
from ..base_loader import BaseLoader
class AssistantLoader(BaseLoader):
"""写入 dim_assistant"""
def upsert_assistants(self, records: list) -> tuple:
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.dim_assistant (
store_id,
assistant_id,
assistant_no,
nickname,
real_name,
gender,
mobile,
level,
team_id,
team_name,
assistant_status,
work_status,
entry_time,
resign_time,
start_time,
end_time,
create_time,
update_time,
system_role_id,
online_status,
allow_cx,
charge_way,
pd_unit_price,
cx_unit_price,
is_guaranteed,
is_team_leader,
serial_number,
show_sort,
is_delete,
raw_data
)
VALUES (
%(store_id)s,
%(assistant_id)s,
%(assistant_no)s,
%(nickname)s,
%(real_name)s,
%(gender)s,
%(mobile)s,
%(level)s,
%(team_id)s,
%(team_name)s,
%(assistant_status)s,
%(work_status)s,
%(entry_time)s,
%(resign_time)s,
%(start_time)s,
%(end_time)s,
%(create_time)s,
%(update_time)s,
%(system_role_id)s,
%(online_status)s,
%(allow_cx)s,
%(charge_way)s,
%(pd_unit_price)s,
%(cx_unit_price)s,
%(is_guaranteed)s,
%(is_team_leader)s,
%(serial_number)s,
%(show_sort)s,
%(is_delete)s,
%(raw_data)s
)
ON CONFLICT (store_id, assistant_id) DO UPDATE SET
assistant_no = EXCLUDED.assistant_no,
nickname = EXCLUDED.nickname,
real_name = EXCLUDED.real_name,
gender = EXCLUDED.gender,
mobile = EXCLUDED.mobile,
level = EXCLUDED.level,
team_id = EXCLUDED.team_id,
team_name = EXCLUDED.team_name,
assistant_status= EXCLUDED.assistant_status,
work_status = EXCLUDED.work_status,
entry_time = EXCLUDED.entry_time,
resign_time = EXCLUDED.resign_time,
start_time = EXCLUDED.start_time,
end_time = EXCLUDED.end_time,
update_time = COALESCE(EXCLUDED.update_time, now()),
system_role_id = EXCLUDED.system_role_id,
online_status = EXCLUDED.online_status,
allow_cx = EXCLUDED.allow_cx,
charge_way = EXCLUDED.charge_way,
pd_unit_price = EXCLUDED.pd_unit_price,
cx_unit_price = EXCLUDED.cx_unit_price,
is_guaranteed = EXCLUDED.is_guaranteed,
is_team_leader = EXCLUDED.is_team_leader,
serial_number = EXCLUDED.serial_number,
show_sort = EXCLUDED.show_sort,
is_delete = EXCLUDED.is_delete,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql, records, page_size=self._batch_size()
)
return (inserted, updated, 0)

34
loaders/dimensions/member.py Normal file
View File

@@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
"""会员维度表加载器"""
from ..base_loader import BaseLoader
class MemberLoader(BaseLoader):
"""会员维度加载器"""
def upsert_members(self, records: list, store_id: int) -> tuple:
"""加载会员数据"""
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.dim_member (
store_id, member_id, member_name, phone, balance,
status, register_time, raw_data
)
VALUES (
%(store_id)s, %(member_id)s, %(member_name)s, %(phone)s, %(balance)s,
%(status)s, %(register_time)s, %(raw_data)s
)
ON CONFLICT (store_id, member_id) DO UPDATE SET
member_name = EXCLUDED.member_name,
phone = EXCLUDED.phone,
balance = EXCLUDED.balance,
status = EXCLUDED.status,
register_time = EXCLUDED.register_time,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(sql, records, page_size=self._batch_size())
return (inserted, updated, 0)

91
loaders/dimensions/package.py Normal file
View File

@@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
"""团购/套餐定义加载器"""
from ..base_loader import BaseLoader
class PackageDefinitionLoader(BaseLoader):
"""写入 dim_package_coupon"""
def upsert_packages(self, records: list) -> tuple:
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.dim_package_coupon (
store_id,
package_id,
package_code,
package_name,
table_area_id,
table_area_name,
selling_price,
duration_seconds,
start_time,
end_time,
type,
is_enabled,
is_delete,
usable_count,
creator_name,
date_type,
group_type,
coupon_money,
area_tag_type,
system_group_type,
card_type_ids,
raw_data
)
VALUES (
%(store_id)s,
%(package_id)s,
%(package_code)s,
%(package_name)s,
%(table_area_id)s,
%(table_area_name)s,
%(selling_price)s,
%(duration_seconds)s,
%(start_time)s,
%(end_time)s,
%(type)s,
%(is_enabled)s,
%(is_delete)s,
%(usable_count)s,
%(creator_name)s,
%(date_type)s,
%(group_type)s,
%(coupon_money)s,
%(area_tag_type)s,
%(system_group_type)s,
%(card_type_ids)s,
%(raw_data)s
)
ON CONFLICT (store_id, package_id) DO UPDATE SET
package_code = EXCLUDED.package_code,
package_name = EXCLUDED.package_name,
table_area_id = EXCLUDED.table_area_id,
table_area_name = EXCLUDED.table_area_name,
selling_price = EXCLUDED.selling_price,
duration_seconds = EXCLUDED.duration_seconds,
start_time = EXCLUDED.start_time,
end_time = EXCLUDED.end_time,
type = EXCLUDED.type,
is_enabled = EXCLUDED.is_enabled,
is_delete = EXCLUDED.is_delete,
usable_count = EXCLUDED.usable_count,
creator_name = EXCLUDED.creator_name,
date_type = EXCLUDED.date_type,
group_type = EXCLUDED.group_type,
coupon_money = EXCLUDED.coupon_money,
area_tag_type = EXCLUDED.area_tag_type,
system_group_type = EXCLUDED.system_group_type,
card_type_ids = EXCLUDED.card_type_ids,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql, records, page_size=self._batch_size()
)
return (inserted, updated, 0)

134
loaders/dimensions/product.py Normal file
View File

@@ -0,0 +1,134 @@
# -*- coding: utf-8 -*-
"""商品维度 + 价格SCD2 加载器"""
from ..base_loader import BaseLoader
from scd.scd2_handler import SCD2Handler
class ProductLoader(BaseLoader):
"""商品维度加载器（dim_product + dim_product_price_scd）"""
def __init__(self, db_ops):
super().__init__(db_ops)
# SCD2 处理器，复用通用逻辑
self.scd_handler = SCD2Handler(db_ops)
def upsert_products(self, records: list, store_id: int) -> tuple:
"""
加载商品维度及价格SCD
返回: (inserted_count, updated_count, skipped_count)
"""
if not records:
return (0, 0, 0)
# 1) 维度主表：billiards.dim_product
sql_base = """
INSERT INTO billiards.dim_product (
store_id,
product_id,
site_product_id,
product_name,
category_id,
category_name,
second_category_id,
unit,
cost_price,
sale_price,
allow_discount,
status,
supplier_id,
barcode,
is_combo,
created_time,
updated_time,
raw_data
)
VALUES (
%(store_id)s,
%(product_id)s,
%(site_product_id)s,
%(product_name)s,
%(category_id)s,
%(category_name)s,
%(second_category_id)s,
%(unit)s,
%(cost_price)s,
%(sale_price)s,
%(allow_discount)s,
%(status)s,
%(supplier_id)s,
%(barcode)s,
%(is_combo)s,
%(created_time)s,
%(updated_time)s,
%(raw_data)s
)
ON CONFLICT (store_id, product_id) DO UPDATE SET
site_product_id = EXCLUDED.site_product_id,
product_name = EXCLUDED.product_name,
category_id = EXCLUDED.category_id,
category_name = EXCLUDED.category_name,
second_category_id = EXCLUDED.second_category_id,
unit = EXCLUDED.unit,
cost_price = EXCLUDED.cost_price,
sale_price = EXCLUDED.sale_price,
allow_discount = EXCLUDED.allow_discount,
status = EXCLUDED.status,
supplier_id = EXCLUDED.supplier_id,
barcode = EXCLUDED.barcode,
is_combo = EXCLUDED.is_combo,
updated_time = COALESCE(EXCLUDED.updated_time, now()),
raw_data = EXCLUDED.raw_data
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql_base,
records,
page_size=self._batch_size(),
)
# 2) 价格 SCD2：billiards.dim_product_price_scd
# 只追踪 price + 类目 + 名称等字段的历史
tracked_fields = [
"product_name",
"category_id",
"category_name",
"second_category_id",
"cost_price",
"sale_price",
"allow_discount",
"status",
]
natural_key = ["store_id", "product_id"]
for rec in records:
effective_date = rec.get("updated_time") or rec.get("created_time")
scd_record = {
"store_id": rec["store_id"],
"product_id": rec["product_id"],
"product_name": rec.get("product_name"),
"category_id": rec.get("category_id"),
"category_name": rec.get("category_name"),
"second_category_id": rec.get("second_category_id"),
"cost_price": rec.get("cost_price"),
"sale_price": rec.get("sale_price"),
"allow_discount": rec.get("allow_discount"),
"status": rec.get("status"),
# 原表中有 raw_data jsonb 字段，这里直接复用 task 传入的 raw_data
"raw_data": rec.get("raw_data"),
}
# 这里我们不强行区分 INSERT/UPDATE/SKIP，对 ETL 统计来说意义不大
self.scd_handler.upsert(
table_name="billiards.dim_product_price_scd",
natural_key=natural_key,
tracked_fields=tracked_fields,
record=scd_record,
effective_date=effective_date,
)
# skipped_count 统一按 0 返回（真正被丢弃的记录在 Task 端已经过滤）
return (inserted, updated, 0)

80
loaders/dimensions/table.py Normal file
View File

@@ -0,0 +1,80 @@
# -*- coding: utf-8 -*-
"""台桌维度加载器"""
from ..base_loader import BaseLoader
class TableLoader(BaseLoader):
"""将台桌档案写入 dim_table"""
def upsert_tables(self, records: list) -> tuple:
"""批量写入台桌档案"""
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.dim_table (
store_id,
table_id,
site_id,
area_id,
area_name,
table_name,
table_price,
table_status,
table_status_name,
light_status,
is_rest_area,
show_status,
virtual_table,
charge_free,
only_allow_groupon,
is_online_reservation,
created_time,
raw_data
)
VALUES (
%(store_id)s,
%(table_id)s,
%(site_id)s,
%(area_id)s,
%(area_name)s,
%(table_name)s,
%(table_price)s,
%(table_status)s,
%(table_status_name)s,
%(light_status)s,
%(is_rest_area)s,
%(show_status)s,
%(virtual_table)s,
%(charge_free)s,
%(only_allow_groupon)s,
%(is_online_reservation)s,
%(created_time)s,
%(raw_data)s
)
ON CONFLICT (store_id, table_id) DO UPDATE SET
site_id = EXCLUDED.site_id,
area_id = EXCLUDED.area_id,
area_name = EXCLUDED.area_name,
table_name = EXCLUDED.table_name,
table_price = EXCLUDED.table_price,
table_status = EXCLUDED.table_status,
table_status_name = EXCLUDED.table_status_name,
light_status = EXCLUDED.light_status,
is_rest_area = EXCLUDED.is_rest_area,
show_status = EXCLUDED.show_status,
virtual_table = EXCLUDED.virtual_table,
charge_free = EXCLUDED.charge_free,
only_allow_groupon = EXCLUDED.only_allow_groupon,
is_online_reservation = EXCLUDED.is_online_reservation,
created_time = COALESCE(EXCLUDED.created_time, dim_table.created_time),
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql, records, page_size=self._batch_size()
)
return (inserted, updated, 0)

0
loaders/facts/__init__.py Normal file
View File

64
loaders/facts/assistant_abolish.py Normal file
View File

@@ -0,0 +1,64 @@
# -*- coding: utf-8 -*-
"""助教作废事实表"""
from ..base_loader import BaseLoader
class AssistantAbolishLoader(BaseLoader):
"""写入 fact_assistant_abolish"""
def upsert_records(self, records: list) -> tuple:
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.fact_assistant_abolish (
store_id,
abolish_id,
table_id,
table_name,
table_area_id,
table_area,
assistant_no,
assistant_name,
charge_minutes,
abolish_amount,
create_time,
trash_reason,
raw_data
)
VALUES (
%(store_id)s,
%(abolish_id)s,
%(table_id)s,
%(table_name)s,
%(table_area_id)s,
%(table_area)s,
%(assistant_no)s,
%(assistant_name)s,
%(charge_minutes)s,
%(abolish_amount)s,
%(create_time)s,
%(trash_reason)s,
%(raw_data)s
)
ON CONFLICT (store_id, abolish_id) DO UPDATE SET
table_id = EXCLUDED.table_id,
table_name = EXCLUDED.table_name,
table_area_id = EXCLUDED.table_area_id,
table_area = EXCLUDED.table_area,
assistant_no = EXCLUDED.assistant_no,
assistant_name = EXCLUDED.assistant_name,
charge_minutes = EXCLUDED.charge_minutes,
abolish_amount = EXCLUDED.abolish_amount,
create_time = EXCLUDED.create_time,
trash_reason = EXCLUDED.trash_reason,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql, records, page_size=self._batch_size()
)
return (inserted, updated, 0)

136
loaders/facts/assistant_ledger.py Normal file
View File

@@ -0,0 +1,136 @@
# -*- coding: utf-8 -*-
"""助教流水事实表"""
from ..base_loader import BaseLoader
class AssistantLedgerLoader(BaseLoader):
"""写入 fact_assistant_ledger"""
def upsert_ledgers(self, records: list) -> tuple:
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.fact_assistant_ledger (
store_id,
ledger_id,
assistant_no,
assistant_name,
nickname,
level_name,
table_name,
ledger_unit_price,
ledger_count,
ledger_amount,
projected_income,
service_money,
member_discount_amount,
manual_discount_amount,
coupon_deduct_money,
order_trade_no,
order_settle_id,
operator_id,
operator_name,
assistant_team_id,
assistant_level,
site_table_id,
order_assistant_id,
site_assistant_id,
user_id,
ledger_start_time,
ledger_end_time,
start_use_time,
last_use_time,
income_seconds,
real_use_seconds,
is_trash,
trash_reason,
is_confirm,
ledger_status,
create_time,
raw_data
)
VALUES (
%(store_id)s,
%(ledger_id)s,
%(assistant_no)s,
%(assistant_name)s,
%(nickname)s,
%(level_name)s,
%(table_name)s,
%(ledger_unit_price)s,
%(ledger_count)s,
%(ledger_amount)s,
%(projected_income)s,
%(service_money)s,
%(member_discount_amount)s,
%(manual_discount_amount)s,
%(coupon_deduct_money)s,
%(order_trade_no)s,
%(order_settle_id)s,
%(operator_id)s,
%(operator_name)s,
%(assistant_team_id)s,
%(assistant_level)s,
%(site_table_id)s,
%(order_assistant_id)s,
%(site_assistant_id)s,
%(user_id)s,
%(ledger_start_time)s,
%(ledger_end_time)s,
%(start_use_time)s,
%(last_use_time)s,
%(income_seconds)s,
%(real_use_seconds)s,
%(is_trash)s,
%(trash_reason)s,
%(is_confirm)s,
%(ledger_status)s,
%(create_time)s,
%(raw_data)s
)
ON CONFLICT (store_id, ledger_id) DO UPDATE SET
assistant_no = EXCLUDED.assistant_no,
assistant_name = EXCLUDED.assistant_name,
nickname = EXCLUDED.nickname,
level_name = EXCLUDED.level_name,
table_name = EXCLUDED.table_name,
ledger_unit_price = EXCLUDED.ledger_unit_price,
ledger_count = EXCLUDED.ledger_count,
ledger_amount = EXCLUDED.ledger_amount,
projected_income = EXCLUDED.projected_income,
service_money = EXCLUDED.service_money,
member_discount_amount = EXCLUDED.member_discount_amount,
manual_discount_amount = EXCLUDED.manual_discount_amount,
coupon_deduct_money = EXCLUDED.coupon_deduct_money,
order_trade_no = EXCLUDED.order_trade_no,
order_settle_id = EXCLUDED.order_settle_id,
operator_id = EXCLUDED.operator_id,
operator_name = EXCLUDED.operator_name,
assistant_team_id = EXCLUDED.assistant_team_id,
assistant_level = EXCLUDED.assistant_level,
site_table_id = EXCLUDED.site_table_id,
order_assistant_id = EXCLUDED.order_assistant_id,
site_assistant_id = EXCLUDED.site_assistant_id,
user_id = EXCLUDED.user_id,
ledger_start_time = EXCLUDED.ledger_start_time,
ledger_end_time = EXCLUDED.ledger_end_time,
start_use_time = EXCLUDED.start_use_time,
last_use_time = EXCLUDED.last_use_time,
income_seconds = EXCLUDED.income_seconds,
real_use_seconds = EXCLUDED.real_use_seconds,
is_trash = EXCLUDED.is_trash,
trash_reason = EXCLUDED.trash_reason,
is_confirm = EXCLUDED.is_confirm,
ledger_status = EXCLUDED.ledger_status,
create_time = EXCLUDED.create_time,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql, records, page_size=self._batch_size()
)
return (inserted, updated, 0)

91
loaders/facts/coupon_usage.py Normal file
View File

@@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
"""券核销事实表"""
from ..base_loader import BaseLoader
class CouponUsageLoader(BaseLoader):
"""写入 fact_coupon_usage"""
def upsert_coupon_usage(self, records: list) -> tuple:
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.fact_coupon_usage (
store_id,
usage_id,
coupon_code,
coupon_channel,
coupon_name,
sale_price,
coupon_money,
coupon_free_time,
use_status,
create_time,
consume_time,
operator_id,
operator_name,
table_id,
site_order_id,
group_package_id,
coupon_remark,
deal_id,
certificate_id,
verify_id,
is_delete,
raw_data
)
VALUES (
%(store_id)s,
%(usage_id)s,
%(coupon_code)s,
%(coupon_channel)s,
%(coupon_name)s,
%(sale_price)s,
%(coupon_money)s,
%(coupon_free_time)s,
%(use_status)s,
%(create_time)s,
%(consume_time)s,
%(operator_id)s,
%(operator_name)s,
%(table_id)s,
%(site_order_id)s,
%(group_package_id)s,
%(coupon_remark)s,
%(deal_id)s,
%(certificate_id)s,
%(verify_id)s,
%(is_delete)s,
%(raw_data)s
)
ON CONFLICT (store_id, usage_id) DO UPDATE SET
coupon_code = EXCLUDED.coupon_code,
coupon_channel = EXCLUDED.coupon_channel,
coupon_name = EXCLUDED.coupon_name,
sale_price = EXCLUDED.sale_price,
coupon_money = EXCLUDED.coupon_money,
coupon_free_time = EXCLUDED.coupon_free_time,
use_status = EXCLUDED.use_status,
create_time = EXCLUDED.create_time,
consume_time = EXCLUDED.consume_time,
operator_id = EXCLUDED.operator_id,
operator_name = EXCLUDED.operator_name,
table_id = EXCLUDED.table_id,
site_order_id = EXCLUDED.site_order_id,
group_package_id = EXCLUDED.group_package_id,
coupon_remark = EXCLUDED.coupon_remark,
deal_id = EXCLUDED.deal_id,
certificate_id = EXCLUDED.certificate_id,
verify_id = EXCLUDED.verify_id,
is_delete = EXCLUDED.is_delete,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql, records, page_size=self._batch_size()
)
return (inserted, updated, 0)

73
loaders/facts/inventory_change.py Normal file
View File

@@ -0,0 +1,73 @@
# -*- coding: utf-8 -*-
"""库存变动事实表"""
from ..base_loader import BaseLoader
class InventoryChangeLoader(BaseLoader):
"""写入 fact_inventory_change"""
def upsert_changes(self, records: list) -> tuple:
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.fact_inventory_change (
store_id,
change_id,
site_goods_id,
stock_type,
goods_name,
change_time,
start_qty,
end_qty,
change_qty,
unit,
price,
operator_name,
remark,
goods_category_id,
goods_second_category_id,
raw_data
)
VALUES (
%(store_id)s,
%(change_id)s,
%(site_goods_id)s,
%(stock_type)s,
%(goods_name)s,
%(change_time)s,
%(start_qty)s,
%(end_qty)s,
%(change_qty)s,
%(unit)s,
%(price)s,
%(operator_name)s,
%(remark)s,
%(goods_category_id)s,
%(goods_second_category_id)s,
%(raw_data)s
)
ON CONFLICT (store_id, change_id) DO UPDATE SET
site_goods_id = EXCLUDED.site_goods_id,
stock_type = EXCLUDED.stock_type,
goods_name = EXCLUDED.goods_name,
change_time = EXCLUDED.change_time,
start_qty = EXCLUDED.start_qty,
end_qty = EXCLUDED.end_qty,
change_qty = EXCLUDED.change_qty,
unit = EXCLUDED.unit,
price = EXCLUDED.price,
operator_name = EXCLUDED.operator_name,
remark = EXCLUDED.remark,
goods_category_id = EXCLUDED.goods_category_id,
goods_second_category_id = EXCLUDED.goods_second_category_id,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql, records, page_size=self._batch_size()
)
return (inserted, updated, 0)

42
loaders/facts/order.py Normal file
View File

@@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
"""订单事实表加载器"""
from ..base_loader import BaseLoader
class OrderLoader(BaseLoader):
"""订单数据加载器"""
def upsert_orders(self, records: list, store_id: int) -> tuple:
"""加载订单数据"""
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.fact_order (
store_id, order_id, order_no, member_id, table_id,
order_time, end_time, total_amount, discount_amount,
final_amount, pay_status, order_status, remark, raw_data
)
VALUES (
%(store_id)s, %(order_id)s, %(order_no)s, %(member_id)s, %(table_id)s,
%(order_time)s, %(end_time)s, %(total_amount)s, %(discount_amount)s,
%(final_amount)s, %(pay_status)s, %(order_status)s, %(remark)s, %(raw_data)s
)
ON CONFLICT (store_id, order_id) DO UPDATE SET
order_no = EXCLUDED.order_no,
member_id = EXCLUDED.member_id,
table_id = EXCLUDED.table_id,
order_time = EXCLUDED.order_time,
end_time = EXCLUDED.end_time,
total_amount = EXCLUDED.total_amount,
discount_amount = EXCLUDED.discount_amount,
final_amount = EXCLUDED.final_amount,
pay_status = EXCLUDED.pay_status,
order_status = EXCLUDED.order_status,
remark = EXCLUDED.remark,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(sql, records, page_size=self._batch_size())
return (inserted, updated, 0)

61
loaders/facts/payment.py Normal file
View File

@@ -0,0 +1,61 @@
# -*- coding: utf-8 -*-
"""支付事实表加载器"""
from ..base_loader import BaseLoader
class PaymentLoader(BaseLoader):
"""支付数据加载器"""
def upsert_payments(self, records: list, store_id: int) -> tuple:
"""加载支付数据"""
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.fact_payment (
store_id, pay_id, order_id,
site_id, tenant_id,
order_settle_id, order_trade_no,
relate_type, relate_id,
create_time, pay_time,
pay_amount, fee_amount, discount_amount,
payment_method, pay_type,
online_pay_channel, pay_terminal,
pay_status, remark, raw_data
)
VALUES (
%(store_id)s, %(pay_id)s, %(order_id)s,
%(site_id)s, %(tenant_id)s,
%(order_settle_id)s, %(order_trade_no)s,
%(relate_type)s, %(relate_id)s,
%(create_time)s, %(pay_time)s,
%(pay_amount)s, %(fee_amount)s, %(discount_amount)s,
%(payment_method)s, %(pay_type)s,
%(online_pay_channel)s, %(pay_terminal)s,
%(pay_status)s, %(remark)s, %(raw_data)s
)
ON CONFLICT (store_id, pay_id) DO UPDATE SET
order_settle_id = EXCLUDED.order_settle_id,
order_trade_no = EXCLUDED.order_trade_no,
relate_type = EXCLUDED.relate_type,
relate_id = EXCLUDED.relate_id,
order_id = EXCLUDED.order_id,
site_id = EXCLUDED.site_id,
tenant_id = EXCLUDED.tenant_id,
create_time = EXCLUDED.create_time,
pay_time = EXCLUDED.pay_time,
pay_amount = EXCLUDED.pay_amount,
fee_amount = EXCLUDED.fee_amount,
discount_amount = EXCLUDED.discount_amount,
payment_method = EXCLUDED.payment_method,
pay_type = EXCLUDED.pay_type,
online_pay_channel = EXCLUDED.online_pay_channel,
pay_terminal = EXCLUDED.pay_terminal,
pay_status = EXCLUDED.pay_status,
remark = EXCLUDED.remark,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(sql, records, page_size=self._batch_size())
return (inserted, updated, 0)

88
loaders/facts/refund.py Normal file
View File

@@ -0,0 +1,88 @@
# -*- coding: utf-8 -*-
"""退款事实表加载器"""
from ..base_loader import BaseLoader
class RefundLoader(BaseLoader):
"""写入 fact_refund"""
def upsert_refunds(self, records: list) -> tuple:
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.fact_refund (
store_id,
refund_id,
site_id,
tenant_id,
pay_amount,
pay_status,
pay_time,
create_time,
relate_type,
relate_id,
payment_method,
refund_amount,
action_type,
pay_terminal,
operator_id,
channel_pay_no,
channel_fee,
is_delete,
member_id,
member_card_id,
raw_data
)
VALUES (
%(store_id)s,
%(refund_id)s,
%(site_id)s,
%(tenant_id)s,
%(pay_amount)s,
%(pay_status)s,
%(pay_time)s,
%(create_time)s,
%(relate_type)s,
%(relate_id)s,
%(payment_method)s,
%(refund_amount)s,
%(action_type)s,
%(pay_terminal)s,
%(operator_id)s,
%(channel_pay_no)s,
%(channel_fee)s,
%(is_delete)s,
%(member_id)s,
%(member_card_id)s,
%(raw_data)s
)
ON CONFLICT (store_id, refund_id) DO UPDATE SET
site_id = EXCLUDED.site_id,
tenant_id = EXCLUDED.tenant_id,
pay_amount = EXCLUDED.pay_amount,
pay_status = EXCLUDED.pay_status,
pay_time = EXCLUDED.pay_time,
create_time = EXCLUDED.create_time,
relate_type = EXCLUDED.relate_type,
relate_id = EXCLUDED.relate_id,
payment_method = EXCLUDED.payment_method,
refund_amount = EXCLUDED.refund_amount,
action_type = EXCLUDED.action_type,
pay_terminal = EXCLUDED.pay_terminal,
operator_id = EXCLUDED.operator_id,
channel_pay_no = EXCLUDED.channel_pay_no,
channel_fee = EXCLUDED.channel_fee,
is_delete = EXCLUDED.is_delete,
member_id = EXCLUDED.member_id,
member_card_id = EXCLUDED.member_card_id,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql, records, page_size=self._batch_size()
)
return (inserted, updated, 0)

82
loaders/facts/table_discount.py Normal file
View File

@@ -0,0 +1,82 @@
# -*- coding: utf-8 -*-
"""台费打折事实表"""
from ..base_loader import BaseLoader
class TableDiscountLoader(BaseLoader):
"""写入 fact_table_discount"""
def upsert_discounts(self, records: list) -> tuple:
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.fact_table_discount (
store_id,
discount_id,
adjust_type,
applicant_id,
applicant_name,
operator_id,
operator_name,
ledger_amount,
ledger_count,
ledger_name,
ledger_status,
order_settle_id,
order_trade_no,
site_table_id,
table_area_id,
table_area_name,
create_time,
is_delete,
raw_data
)
VALUES (
%(store_id)s,
%(discount_id)s,
%(adjust_type)s,
%(applicant_id)s,
%(applicant_name)s,
%(operator_id)s,
%(operator_name)s,
%(ledger_amount)s,
%(ledger_count)s,
%(ledger_name)s,
%(ledger_status)s,
%(order_settle_id)s,
%(order_trade_no)s,
%(site_table_id)s,
%(table_area_id)s,
%(table_area_name)s,
%(create_time)s,
%(is_delete)s,
%(raw_data)s
)
ON CONFLICT (store_id, discount_id) DO UPDATE SET
adjust_type = EXCLUDED.adjust_type,
applicant_id = EXCLUDED.applicant_id,
applicant_name = EXCLUDED.applicant_name,
operator_id = EXCLUDED.operator_id,
operator_name = EXCLUDED.operator_name,
ledger_amount = EXCLUDED.ledger_amount,
ledger_count = EXCLUDED.ledger_count,
ledger_name = EXCLUDED.ledger_name,
ledger_status = EXCLUDED.ledger_status,
order_settle_id = EXCLUDED.order_settle_id,
order_trade_no = EXCLUDED.order_trade_no,
site_table_id = EXCLUDED.site_table_id,
table_area_id = EXCLUDED.table_area_id,
table_area_name = EXCLUDED.table_area_name,
create_time = EXCLUDED.create_time,
is_delete = EXCLUDED.is_delete,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql, records, page_size=self._batch_size()
)
return (inserted, updated, 0)

188
loaders/facts/ticket.py Normal file
View File

@@ -0,0 +1,188 @@
# -*- coding: utf-8 -*-
"""小票详情加载器"""
from ..base_loader import BaseLoader
import json
class TicketLoader(BaseLoader):
"""
小票详情 JSON 解析加载器，写入 DWD 事实表。
处理：
- fact_order（订单头）
- fact_order_goods（商品项）
- fact_table_usage（台桌使用）
- fact_assistant_service（助教服务）
"""
def process_tickets(self, tickets: list, store_id: int) -> tuple:
"""
批量处理小票 JSON。
返回 (插入数, 错误数)
"""
inserted_count = 0
error_count = 0
# 准备批量数据列表
orders = []
goods_list = []
table_usages = []
assistant_services = []
for ticket in tickets:
try:
# 1. 解析订单头部 (fact_order)
root_data = ticket.get("data", {}).get("data", {})
if not root_data:
continue
order_settle_id = root_data.get("orderSettleId")
if not order_settle_id:
continue
orders.append({
"store_id": store_id,
"order_settle_id": order_settle_id,
"order_trade_no": 0,
"order_no": str(root_data.get("orderSettleNumber", "")),
"member_id": 0,
"pay_time": root_data.get("payTime"),
"total_amount": root_data.get("consumeMoney", 0),
"pay_amount": root_data.get("actualPayment", 0),
"discount_amount": root_data.get("memberOfferAmount", 0),
"coupon_amount": root_data.get("couponAmount", 0),
"status": "PAID",
"cashier_name": root_data.get("cashierName", ""),
"remark": root_data.get("orderRemark", ""),
"raw_data": json.dumps(ticket, ensure_ascii=False)
})
# 2. 解析订单项 (orderItem 列表)
order_items = root_data.get("orderItem", [])
for item in order_items:
order_trade_no = item.get("siteOrderId")
# 2.1 台桌流水
table_ledger = item.get("tableLedger")
if table_ledger:
table_usages.append({
"store_id": store_id,
"order_ledger_id": table_ledger.get("orderTableLedgerId"),
"order_settle_id": order_settle_id,
"table_id": table_ledger.get("siteTableId"),
"table_name": table_ledger.get("tableName"),
"start_time": table_ledger.get("chargeStartTime"),
"end_time": table_ledger.get("chargeEndTime"),
"duration_minutes": table_ledger.get("useDuration", 0),
"total_amount": table_ledger.get("consumptionAmount", 0),
"pay_amount": table_ledger.get("consumptionAmount", 0) - table_ledger.get("memberDiscountAmount", 0)
})
# 2.2 商品流水
goods_ledgers = item.get("goodsLedgers", [])
for g in goods_ledgers:
goods_list.append({
"store_id": store_id,
"order_goods_id": g.get("orderGoodsLedgerId"),
"order_settle_id": order_settle_id,
"order_trade_no": order_trade_no,
"goods_id": g.get("siteGoodsId"),
"goods_name": g.get("goodsName"),
"quantity": g.get("goodsCount", 0),
"unit_price": g.get("goodsPrice", 0),
"total_amount": g.get("ledgerAmount", 0),
"pay_amount": g.get("realGoodsMoney", 0)
})
# 2.3 助教服务
assistant_ledgers = item.get("assistantPlayWith", [])
for a in assistant_ledgers:
assistant_services.append({
"store_id": store_id,
"ledger_id": a.get("orderAssistantLedgerId"),
"order_settle_id": order_settle_id,
"assistant_id": a.get("assistantId"),
"assistant_name": a.get("ledgerName"),
"service_type": a.get("skillName", "Play"),
"start_time": a.get("ledgerStartTime"),
"end_time": a.get("ledgerEndTime"),
"duration_minutes": int(a.get("ledgerCount", 0) / 60) if a.get("ledgerCount") else 0,
"total_amount": a.get("ledgerAmount", 0),
"pay_amount": a.get("ledgerAmount", 0)
})
inserted_count += 1
except Exception as e:
self.logger.error(f"Error parsing ticket: {e}", exc_info=True)
error_count += 1
# 3. 批量插入/更新
if orders:
self._upsert_orders(orders)
if goods_list:
self._upsert_goods(goods_list)
if table_usages:
self._upsert_table_usages(table_usages)
if assistant_services:
self._upsert_assistant_services(assistant_services)
return inserted_count, error_count
def _upsert_orders(self, rows):
sql = """
INSERT INTO billiards.fact_order (
store_id, order_settle_id, order_trade_no, order_no, member_id,
pay_time, total_amount, pay_amount, discount_amount, coupon_amount,
status, cashier_name, remark, raw_data
) VALUES (
%(store_id)s, %(order_settle_id)s, %(order_trade_no)s, %(order_no)s, %(member_id)s,
%(pay_time)s, %(total_amount)s, %(pay_amount)s, %(discount_amount)s, %(coupon_amount)s,
%(status)s, %(cashier_name)s, %(remark)s, %(raw_data)s
)
ON CONFLICT (store_id, order_settle_id) DO UPDATE SET
pay_time = EXCLUDED.pay_time,
pay_amount = EXCLUDED.pay_amount,
updated_at = now()
"""
self.db.batch_execute(sql, rows)
def _upsert_goods(self, rows):
sql = """
INSERT INTO billiards.fact_order_goods (
store_id, order_goods_id, order_settle_id, order_trade_no,
goods_id, goods_name, quantity, unit_price, total_amount, pay_amount
) VALUES (
%(store_id)s, %(order_goods_id)s, %(order_settle_id)s, %(order_trade_no)s,
%(goods_id)s, %(goods_name)s, %(quantity)s, %(unit_price)s, %(total_amount)s, %(pay_amount)s
)
ON CONFLICT (store_id, order_goods_id) DO UPDATE SET
pay_amount = EXCLUDED.pay_amount
"""
self.db.batch_execute(sql, rows)
def _upsert_table_usages(self, rows):
sql = """
INSERT INTO billiards.fact_table_usage (
store_id, order_ledger_id, order_settle_id, table_id, table_name,
start_time, end_time, duration_minutes, total_amount, pay_amount
) VALUES (
%(store_id)s, %(order_ledger_id)s, %(order_settle_id)s, %(table_id)s, %(table_name)s,
%(start_time)s, %(end_time)s, %(duration_minutes)s, %(total_amount)s, %(pay_amount)s
)
ON CONFLICT (store_id, order_ledger_id) DO UPDATE SET
pay_amount = EXCLUDED.pay_amount
"""
self.db.batch_execute(sql, rows)
def _upsert_assistant_services(self, rows):
sql = """
INSERT INTO billiards.fact_assistant_service (
store_id, ledger_id, order_settle_id, assistant_id, assistant_name,
service_type, start_time, end_time, duration_minutes, total_amount, pay_amount
) VALUES (
%(store_id)s, %(ledger_id)s, %(order_settle_id)s, %(assistant_id)s, %(assistant_name)s,
%(service_type)s, %(start_time)s, %(end_time)s, %(duration_minutes)s, %(total_amount)s, %(pay_amount)s
)
ON CONFLICT (store_id, ledger_id) DO UPDATE SET
pay_amount = EXCLUDED.pay_amount
"""
self.db.batch_execute(sql, rows)

118
loaders/facts/topup.py Normal file
View File

@@ -0,0 +1,118 @@
# -*- coding: utf-8 -*-
"""充值记录事实表"""
from ..base_loader import BaseLoader
class TopupLoader(BaseLoader):
"""写入 fact_topup"""
def upsert_topups(self, records: list) -> tuple:
if not records:
return (0, 0, 0)
sql = """
INSERT INTO billiards.fact_topup (
store_id,
topup_id,
member_id,
member_name,
member_phone,
card_id,
card_type_name,
pay_amount,
consume_money,
settle_status,
settle_type,
settle_name,
settle_relate_id,
pay_time,
create_time,
operator_id,
operator_name,
payment_method,
refund_amount,
cash_amount,
card_amount,
balance_amount,
online_amount,
rounding_amount,
adjust_amount,
goods_money,
table_charge_money,
service_money,
coupon_amount,
order_remark,
raw_data
)
VALUES (
%(store_id)s,
%(topup_id)s,
%(member_id)s,
%(member_name)s,
%(member_phone)s,
%(card_id)s,
%(card_type_name)s,
%(pay_amount)s,
%(consume_money)s,
%(settle_status)s,
%(settle_type)s,
%(settle_name)s,
%(settle_relate_id)s,
%(pay_time)s,
%(create_time)s,
%(operator_id)s,
%(operator_name)s,
%(payment_method)s,
%(refund_amount)s,
%(cash_amount)s,
%(card_amount)s,
%(balance_amount)s,
%(online_amount)s,
%(rounding_amount)s,
%(adjust_amount)s,
%(goods_money)s,
%(table_charge_money)s,
%(service_money)s,
%(coupon_amount)s,
%(order_remark)s,
%(raw_data)s
)
ON CONFLICT (store_id, topup_id) DO UPDATE SET
member_id = EXCLUDED.member_id,
member_name = EXCLUDED.member_name,
member_phone = EXCLUDED.member_phone,
card_id = EXCLUDED.card_id,
card_type_name = EXCLUDED.card_type_name,
pay_amount = EXCLUDED.pay_amount,
consume_money = EXCLUDED.consume_money,
settle_status = EXCLUDED.settle_status,
settle_type = EXCLUDED.settle_type,
settle_name = EXCLUDED.settle_name,
settle_relate_id = EXCLUDED.settle_relate_id,
pay_time = EXCLUDED.pay_time,
create_time = EXCLUDED.create_time,
operator_id = EXCLUDED.operator_id,
operator_name = EXCLUDED.operator_name,
payment_method = EXCLUDED.payment_method,
refund_amount = EXCLUDED.refund_amount,
cash_amount = EXCLUDED.cash_amount,
card_amount = EXCLUDED.card_amount,
balance_amount = EXCLUDED.balance_amount,
online_amount = EXCLUDED.online_amount,
rounding_amount = EXCLUDED.rounding_amount,
adjust_amount = EXCLUDED.adjust_amount,
goods_money = EXCLUDED.goods_money,
table_charge_money = EXCLUDED.table_charge_money,
service_money = EXCLUDED.service_money,
coupon_amount = EXCLUDED.coupon_amount,
order_remark = EXCLUDED.order_remark,
raw_data = EXCLUDED.raw_data,
updated_at = now()
RETURNING (xmax = 0) AS inserted
"""
inserted, updated = self.db.batch_upsert_with_returning(
sql, records, page_size=self._batch_size()
)
return (inserted, updated, 0)

6
loaders/ods/__init__.py Normal file
View File

@@ -0,0 +1,6 @@
# -*- coding: utf-8 -*-
"""ODS loader helpers."""
from .generic import GenericODSLoader
__all__ = ["GenericODSLoader"]

67
loaders/ods/generic.py Normal file
View File

@@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
"""Generic ODS loader that keeps raw payload + primary keys."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Iterable, Sequence
from ..base_loader import BaseLoader
class GenericODSLoader(BaseLoader):
"""Insert/update helper for ODS tables that share the same pattern."""
def __init__(
self,
db_ops,
table_name: str,
columns: Sequence[str],
conflict_columns: Sequence[str],
):
super().__init__(db_ops)
if not conflict_columns:
raise ValueError("conflict_columns must not be empty for ODS loader")
self.table_name = table_name
self.columns = list(columns)
self.conflict_columns = list(conflict_columns)
self._sql = self._build_sql()
def upsert_rows(self, rows: Iterable[dict]) -> tuple[int, int, int]:
"""Insert/update the provided iterable of dictionaries."""
rows = list(rows)
if not rows:
return (0, 0, 0)
normalized = [self._normalize_row(row) for row in rows]
inserted, updated = self.db.batch_upsert_with_returning(
self._sql, normalized, page_size=self._batch_size()
)
return inserted, updated, 0
def _build_sql(self) -> str:
col_list = ", ".join(self.columns)
placeholders = ", ".join(f"%({col})s" for col in self.columns)
conflict_clause = ", ".join(self.conflict_columns)
update_columns = [c for c in self.columns if c not in self.conflict_columns]
set_clause = ", ".join(f"{col} = EXCLUDED.{col}" for col in update_columns)
return (
f"INSERT INTO {self.table_name} ({col_list}) "
f"VALUES ({placeholders}) "
f"ON CONFLICT ({conflict_clause}) DO UPDATE SET {set_clause} "
f"RETURNING (xmax = 0) AS inserted"
)
def _normalize_row(self, row: dict) -> dict:
normalized = {}
for col in self.columns:
value = row.get(col)
if col == "payload" and value is not None and not isinstance(value, str):
normalized[col] = json.dumps(value, ensure_ascii=False)
else:
normalized[col] = value
if "fetched_at" in normalized and normalized["fetched_at"] is None:
normalized["fetched_at"] = datetime.now(timezone.utc)
return normalized