初始提交：飞球 ETL 系统全量代码
This commit is contained in:
0
loaders/dimensions/__init__.py
Normal file
0
loaders/dimensions/__init__.py
Normal file
114
loaders/dimensions/assistant.py
Normal file
114
loaders/dimensions/assistant.py
Normal file
@@ -0,0 +1,114 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""助教维度加载器"""
|
||||
|
||||
from ..base_loader import BaseLoader
|
||||
|
||||
|
||||
class AssistantLoader(BaseLoader):
|
||||
"""写入 dim_assistant"""
|
||||
|
||||
def upsert_assistants(self, records: list) -> tuple:
|
||||
if not records:
|
||||
return (0, 0, 0)
|
||||
|
||||
sql = """
|
||||
INSERT INTO billiards.dim_assistant (
|
||||
store_id,
|
||||
assistant_id,
|
||||
assistant_no,
|
||||
nickname,
|
||||
real_name,
|
||||
gender,
|
||||
mobile,
|
||||
level,
|
||||
team_id,
|
||||
team_name,
|
||||
assistant_status,
|
||||
work_status,
|
||||
entry_time,
|
||||
resign_time,
|
||||
start_time,
|
||||
end_time,
|
||||
create_time,
|
||||
update_time,
|
||||
system_role_id,
|
||||
online_status,
|
||||
allow_cx,
|
||||
charge_way,
|
||||
pd_unit_price,
|
||||
cx_unit_price,
|
||||
is_guaranteed,
|
||||
is_team_leader,
|
||||
serial_number,
|
||||
show_sort,
|
||||
is_delete,
|
||||
raw_data
|
||||
)
|
||||
VALUES (
|
||||
%(store_id)s,
|
||||
%(assistant_id)s,
|
||||
%(assistant_no)s,
|
||||
%(nickname)s,
|
||||
%(real_name)s,
|
||||
%(gender)s,
|
||||
%(mobile)s,
|
||||
%(level)s,
|
||||
%(team_id)s,
|
||||
%(team_name)s,
|
||||
%(assistant_status)s,
|
||||
%(work_status)s,
|
||||
%(entry_time)s,
|
||||
%(resign_time)s,
|
||||
%(start_time)s,
|
||||
%(end_time)s,
|
||||
%(create_time)s,
|
||||
%(update_time)s,
|
||||
%(system_role_id)s,
|
||||
%(online_status)s,
|
||||
%(allow_cx)s,
|
||||
%(charge_way)s,
|
||||
%(pd_unit_price)s,
|
||||
%(cx_unit_price)s,
|
||||
%(is_guaranteed)s,
|
||||
%(is_team_leader)s,
|
||||
%(serial_number)s,
|
||||
%(show_sort)s,
|
||||
%(is_delete)s,
|
||||
%(raw_data)s
|
||||
)
|
||||
ON CONFLICT (store_id, assistant_id) DO UPDATE SET
|
||||
assistant_no = EXCLUDED.assistant_no,
|
||||
nickname = EXCLUDED.nickname,
|
||||
real_name = EXCLUDED.real_name,
|
||||
gender = EXCLUDED.gender,
|
||||
mobile = EXCLUDED.mobile,
|
||||
level = EXCLUDED.level,
|
||||
team_id = EXCLUDED.team_id,
|
||||
team_name = EXCLUDED.team_name,
|
||||
assistant_status= EXCLUDED.assistant_status,
|
||||
work_status = EXCLUDED.work_status,
|
||||
entry_time = EXCLUDED.entry_time,
|
||||
resign_time = EXCLUDED.resign_time,
|
||||
start_time = EXCLUDED.start_time,
|
||||
end_time = EXCLUDED.end_time,
|
||||
update_time = COALESCE(EXCLUDED.update_time, now()),
|
||||
system_role_id = EXCLUDED.system_role_id,
|
||||
online_status = EXCLUDED.online_status,
|
||||
allow_cx = EXCLUDED.allow_cx,
|
||||
charge_way = EXCLUDED.charge_way,
|
||||
pd_unit_price = EXCLUDED.pd_unit_price,
|
||||
cx_unit_price = EXCLUDED.cx_unit_price,
|
||||
is_guaranteed = EXCLUDED.is_guaranteed,
|
||||
is_team_leader = EXCLUDED.is_team_leader,
|
||||
serial_number = EXCLUDED.serial_number,
|
||||
show_sort = EXCLUDED.show_sort,
|
||||
is_delete = EXCLUDED.is_delete,
|
||||
raw_data = EXCLUDED.raw_data,
|
||||
updated_at = now()
|
||||
RETURNING (xmax = 0) AS inserted
|
||||
"""
|
||||
|
||||
inserted, updated = self.db.batch_upsert_with_returning(
|
||||
sql, records, page_size=self._batch_size()
|
||||
)
|
||||
return (inserted, updated, 0)
|
||||
34
loaders/dimensions/member.py
Normal file
34
loaders/dimensions/member.py
Normal file
@@ -0,0 +1,34 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""会员维度表加载器"""
|
||||
from ..base_loader import BaseLoader
|
||||
|
||||
class MemberLoader(BaseLoader):
|
||||
"""会员维度加载器"""
|
||||
|
||||
def upsert_members(self, records: list, store_id: int) -> tuple:
|
||||
"""加载会员数据"""
|
||||
if not records:
|
||||
return (0, 0, 0)
|
||||
|
||||
sql = """
|
||||
INSERT INTO billiards.dim_member (
|
||||
store_id, member_id, member_name, phone, balance,
|
||||
status, register_time, raw_data
|
||||
)
|
||||
VALUES (
|
||||
%(store_id)s, %(member_id)s, %(member_name)s, %(phone)s, %(balance)s,
|
||||
%(status)s, %(register_time)s, %(raw_data)s
|
||||
)
|
||||
ON CONFLICT (store_id, member_id) DO UPDATE SET
|
||||
member_name = EXCLUDED.member_name,
|
||||
phone = EXCLUDED.phone,
|
||||
balance = EXCLUDED.balance,
|
||||
status = EXCLUDED.status,
|
||||
register_time = EXCLUDED.register_time,
|
||||
raw_data = EXCLUDED.raw_data,
|
||||
updated_at = now()
|
||||
RETURNING (xmax = 0) AS inserted
|
||||
"""
|
||||
|
||||
inserted, updated = self.db.batch_upsert_with_returning(sql, records, page_size=self._batch_size())
|
||||
return (inserted, updated, 0)
|
||||
91
loaders/dimensions/package.py
Normal file
91
loaders/dimensions/package.py
Normal file
@@ -0,0 +1,91 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""团购/套餐定义加载器"""
|
||||
|
||||
from ..base_loader import BaseLoader
|
||||
|
||||
|
||||
class PackageDefinitionLoader(BaseLoader):
|
||||
"""写入 dim_package_coupon"""
|
||||
|
||||
def upsert_packages(self, records: list) -> tuple:
|
||||
if not records:
|
||||
return (0, 0, 0)
|
||||
|
||||
sql = """
|
||||
INSERT INTO billiards.dim_package_coupon (
|
||||
store_id,
|
||||
package_id,
|
||||
package_code,
|
||||
package_name,
|
||||
table_area_id,
|
||||
table_area_name,
|
||||
selling_price,
|
||||
duration_seconds,
|
||||
start_time,
|
||||
end_time,
|
||||
type,
|
||||
is_enabled,
|
||||
is_delete,
|
||||
usable_count,
|
||||
creator_name,
|
||||
date_type,
|
||||
group_type,
|
||||
coupon_money,
|
||||
area_tag_type,
|
||||
system_group_type,
|
||||
card_type_ids,
|
||||
raw_data
|
||||
)
|
||||
VALUES (
|
||||
%(store_id)s,
|
||||
%(package_id)s,
|
||||
%(package_code)s,
|
||||
%(package_name)s,
|
||||
%(table_area_id)s,
|
||||
%(table_area_name)s,
|
||||
%(selling_price)s,
|
||||
%(duration_seconds)s,
|
||||
%(start_time)s,
|
||||
%(end_time)s,
|
||||
%(type)s,
|
||||
%(is_enabled)s,
|
||||
%(is_delete)s,
|
||||
%(usable_count)s,
|
||||
%(creator_name)s,
|
||||
%(date_type)s,
|
||||
%(group_type)s,
|
||||
%(coupon_money)s,
|
||||
%(area_tag_type)s,
|
||||
%(system_group_type)s,
|
||||
%(card_type_ids)s,
|
||||
%(raw_data)s
|
||||
)
|
||||
ON CONFLICT (store_id, package_id) DO UPDATE SET
|
||||
package_code = EXCLUDED.package_code,
|
||||
package_name = EXCLUDED.package_name,
|
||||
table_area_id = EXCLUDED.table_area_id,
|
||||
table_area_name = EXCLUDED.table_area_name,
|
||||
selling_price = EXCLUDED.selling_price,
|
||||
duration_seconds = EXCLUDED.duration_seconds,
|
||||
start_time = EXCLUDED.start_time,
|
||||
end_time = EXCLUDED.end_time,
|
||||
type = EXCLUDED.type,
|
||||
is_enabled = EXCLUDED.is_enabled,
|
||||
is_delete = EXCLUDED.is_delete,
|
||||
usable_count = EXCLUDED.usable_count,
|
||||
creator_name = EXCLUDED.creator_name,
|
||||
date_type = EXCLUDED.date_type,
|
||||
group_type = EXCLUDED.group_type,
|
||||
coupon_money = EXCLUDED.coupon_money,
|
||||
area_tag_type = EXCLUDED.area_tag_type,
|
||||
system_group_type = EXCLUDED.system_group_type,
|
||||
card_type_ids = EXCLUDED.card_type_ids,
|
||||
raw_data = EXCLUDED.raw_data,
|
||||
updated_at = now()
|
||||
RETURNING (xmax = 0) AS inserted
|
||||
"""
|
||||
|
||||
inserted, updated = self.db.batch_upsert_with_returning(
|
||||
sql, records, page_size=self._batch_size()
|
||||
)
|
||||
return (inserted, updated, 0)
|
||||
134
loaders/dimensions/product.py
Normal file
134
loaders/dimensions/product.py
Normal file
@@ -0,0 +1,134 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""商品维度 + 价格SCD2 加载器"""
|
||||
|
||||
from ..base_loader import BaseLoader
|
||||
from scd.scd2_handler import SCD2Handler
|
||||
|
||||
|
||||
class ProductLoader(BaseLoader):
|
||||
"""商品维度加载器（dim_product + dim_product_price_scd）"""
|
||||
|
||||
def __init__(self, db_ops):
|
||||
super().__init__(db_ops)
|
||||
# SCD2 处理器，复用通用逻辑
|
||||
self.scd_handler = SCD2Handler(db_ops)
|
||||
|
||||
def upsert_products(self, records: list, store_id: int) -> tuple:
|
||||
"""
|
||||
加载商品维度及价格SCD
|
||||
|
||||
返回: (inserted_count, updated_count, skipped_count)
|
||||
"""
|
||||
if not records:
|
||||
return (0, 0, 0)
|
||||
|
||||
# 1) 维度主表：billiards.dim_product
|
||||
sql_base = """
|
||||
INSERT INTO billiards.dim_product (
|
||||
store_id,
|
||||
product_id,
|
||||
site_product_id,
|
||||
product_name,
|
||||
category_id,
|
||||
category_name,
|
||||
second_category_id,
|
||||
unit,
|
||||
cost_price,
|
||||
sale_price,
|
||||
allow_discount,
|
||||
status,
|
||||
supplier_id,
|
||||
barcode,
|
||||
is_combo,
|
||||
created_time,
|
||||
updated_time,
|
||||
raw_data
|
||||
)
|
||||
VALUES (
|
||||
%(store_id)s,
|
||||
%(product_id)s,
|
||||
%(site_product_id)s,
|
||||
%(product_name)s,
|
||||
%(category_id)s,
|
||||
%(category_name)s,
|
||||
%(second_category_id)s,
|
||||
%(unit)s,
|
||||
%(cost_price)s,
|
||||
%(sale_price)s,
|
||||
%(allow_discount)s,
|
||||
%(status)s,
|
||||
%(supplier_id)s,
|
||||
%(barcode)s,
|
||||
%(is_combo)s,
|
||||
%(created_time)s,
|
||||
%(updated_time)s,
|
||||
%(raw_data)s
|
||||
)
|
||||
ON CONFLICT (store_id, product_id) DO UPDATE SET
|
||||
site_product_id = EXCLUDED.site_product_id,
|
||||
product_name = EXCLUDED.product_name,
|
||||
category_id = EXCLUDED.category_id,
|
||||
category_name = EXCLUDED.category_name,
|
||||
second_category_id = EXCLUDED.second_category_id,
|
||||
unit = EXCLUDED.unit,
|
||||
cost_price = EXCLUDED.cost_price,
|
||||
sale_price = EXCLUDED.sale_price,
|
||||
allow_discount = EXCLUDED.allow_discount,
|
||||
status = EXCLUDED.status,
|
||||
supplier_id = EXCLUDED.supplier_id,
|
||||
barcode = EXCLUDED.barcode,
|
||||
is_combo = EXCLUDED.is_combo,
|
||||
updated_time = COALESCE(EXCLUDED.updated_time, now()),
|
||||
raw_data = EXCLUDED.raw_data
|
||||
RETURNING (xmax = 0) AS inserted
|
||||
"""
|
||||
|
||||
inserted, updated = self.db.batch_upsert_with_returning(
|
||||
sql_base,
|
||||
records,
|
||||
page_size=self._batch_size(),
|
||||
)
|
||||
|
||||
# 2) 价格 SCD2：billiards.dim_product_price_scd
|
||||
# 只追踪 price + 类目 + 名称等字段的历史
|
||||
tracked_fields = [
|
||||
"product_name",
|
||||
"category_id",
|
||||
"category_name",
|
||||
"second_category_id",
|
||||
"cost_price",
|
||||
"sale_price",
|
||||
"allow_discount",
|
||||
"status",
|
||||
]
|
||||
natural_key = ["store_id", "product_id"]
|
||||
|
||||
for rec in records:
|
||||
effective_date = rec.get("updated_time") or rec.get("created_time")
|
||||
|
||||
scd_record = {
|
||||
"store_id": rec["store_id"],
|
||||
"product_id": rec["product_id"],
|
||||
"product_name": rec.get("product_name"),
|
||||
"category_id": rec.get("category_id"),
|
||||
"category_name": rec.get("category_name"),
|
||||
"second_category_id": rec.get("second_category_id"),
|
||||
"cost_price": rec.get("cost_price"),
|
||||
"sale_price": rec.get("sale_price"),
|
||||
"allow_discount": rec.get("allow_discount"),
|
||||
"status": rec.get("status"),
|
||||
# 原表中有 raw_data jsonb 字段，这里直接复用 task 传入的 raw_data
|
||||
"raw_data": rec.get("raw_data"),
|
||||
}
|
||||
|
||||
# 这里我们不强行区分 INSERT/UPDATE/SKIP，对 ETL 统计来说意义不大
|
||||
self.scd_handler.upsert(
|
||||
table_name="billiards.dim_product_price_scd",
|
||||
natural_key=natural_key,
|
||||
tracked_fields=tracked_fields,
|
||||
record=scd_record,
|
||||
effective_date=effective_date,
|
||||
)
|
||||
|
||||
# skipped_count 统一按 0 返回（真正被丢弃的记录在 Task 端已经过滤）
|
||||
return (inserted, updated, 0)
|
||||
80
loaders/dimensions/table.py
Normal file
80
loaders/dimensions/table.py
Normal file
@@ -0,0 +1,80 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""台桌维度加载器"""
|
||||
|
||||
from ..base_loader import BaseLoader
|
||||
|
||||
|
||||
class TableLoader(BaseLoader):
|
||||
"""将台桌档案写入 dim_table"""
|
||||
|
||||
def upsert_tables(self, records: list) -> tuple:
|
||||
"""批量写入台桌档案"""
|
||||
if not records:
|
||||
return (0, 0, 0)
|
||||
|
||||
sql = """
|
||||
INSERT INTO billiards.dim_table (
|
||||
store_id,
|
||||
table_id,
|
||||
site_id,
|
||||
area_id,
|
||||
area_name,
|
||||
table_name,
|
||||
table_price,
|
||||
table_status,
|
||||
table_status_name,
|
||||
light_status,
|
||||
is_rest_area,
|
||||
show_status,
|
||||
virtual_table,
|
||||
charge_free,
|
||||
only_allow_groupon,
|
||||
is_online_reservation,
|
||||
created_time,
|
||||
raw_data
|
||||
)
|
||||
VALUES (
|
||||
%(store_id)s,
|
||||
%(table_id)s,
|
||||
%(site_id)s,
|
||||
%(area_id)s,
|
||||
%(area_name)s,
|
||||
%(table_name)s,
|
||||
%(table_price)s,
|
||||
%(table_status)s,
|
||||
%(table_status_name)s,
|
||||
%(light_status)s,
|
||||
%(is_rest_area)s,
|
||||
%(show_status)s,
|
||||
%(virtual_table)s,
|
||||
%(charge_free)s,
|
||||
%(only_allow_groupon)s,
|
||||
%(is_online_reservation)s,
|
||||
%(created_time)s,
|
||||
%(raw_data)s
|
||||
)
|
||||
ON CONFLICT (store_id, table_id) DO UPDATE SET
|
||||
site_id = EXCLUDED.site_id,
|
||||
area_id = EXCLUDED.area_id,
|
||||
area_name = EXCLUDED.area_name,
|
||||
table_name = EXCLUDED.table_name,
|
||||
table_price = EXCLUDED.table_price,
|
||||
table_status = EXCLUDED.table_status,
|
||||
table_status_name = EXCLUDED.table_status_name,
|
||||
light_status = EXCLUDED.light_status,
|
||||
is_rest_area = EXCLUDED.is_rest_area,
|
||||
show_status = EXCLUDED.show_status,
|
||||
virtual_table = EXCLUDED.virtual_table,
|
||||
charge_free = EXCLUDED.charge_free,
|
||||
only_allow_groupon = EXCLUDED.only_allow_groupon,
|
||||
is_online_reservation = EXCLUDED.is_online_reservation,
|
||||
created_time = COALESCE(EXCLUDED.created_time, dim_table.created_time),
|
||||
raw_data = EXCLUDED.raw_data,
|
||||
updated_at = now()
|
||||
RETURNING (xmax = 0) AS inserted
|
||||
"""
|
||||
|
||||
inserted, updated = self.db.batch_upsert_with_returning(
|
||||
sql, records, page_size=self._batch_size()
|
||||
)
|
||||
return (inserted, updated, 0)
|
||||
Reference in New Issue
Block a user