root/feiqiu-ETL
Template
1
0
Fork 0
Code Pull Requests Packages Releases Wiki Activity

DWD完成

Browse Source
This commit is contained in:
Neo
2025-12-09 04:57:05 +08:00
parent f301cc1fd5
commit 561c640700
46 changed files with 26181 additions and 3540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1886
etl_billiards/database/schema_ODS_doc copy.sql Normal file
View File

File diff suppressed because it is too large Load Diff

1907
etl_billiards/database/schema_ODS_doc.sql Normal file
View File

File diff suppressed because it is too large Load Diff

1801
etl_billiards/database/schema_dwd.sql
View File

File diff suppressed because it is too large Load Diff

184
etl_billiards/database/schema_dwd_doc.sql
View File

@@ -2,22 +2,117 @@
CREATE SCHEMA IF NOT EXISTS billiards_dwd;
SET search_path TO billiards_dwd;
-- SCD2 字段统一默认值、中文注释、唯一性（业务键 + 时间段不重叠）控制
CREATE EXTENSION IF NOT EXISTS btree_gist;
DO $$
DECLARE
rec RECORD;
BEGIN
-- 统一 SCD2 默认值与注释，避免后续手工遗漏
FOR rec IN
SELECT table_name
FROM information_schema.columns
WHERE table_schema = 'billiards_dwd'
AND column_name = 'scd2_start_time'
LOOP
EXECUTE format('ALTER TABLE billiards_dwd.%I ALTER COLUMN scd2_start_time SET DEFAULT now()', rec.table_name);
EXECUTE format('ALTER TABLE billiards_dwd.%I ALTER COLUMN scd2_end_time SET DEFAULT ''9999-12-31''', rec.table_name);
EXECUTE format('ALTER TABLE billiards_dwd.%I ALTER COLUMN scd2_is_current SET DEFAULT 1', rec.table_name);
EXECUTE format('ALTER TABLE billiards_dwd.%I ALTER COLUMN scd2_version SET DEFAULT 1', rec.table_name);
EXECUTE format('COMMENT ON COLUMN billiards_dwd.%I.scd2_start_time IS ''SCD2 开始时间（版本生效起点）''', rec.table_name);
EXECUTE format('COMMENT ON COLUMN billiards_dwd.%I.scd2_end_time IS ''SCD2 结束时间（默认 9999-12-31，表示当前版本仍有效）''', rec.table_name);
EXECUTE format('COMMENT ON COLUMN billiards_dwd.%I.scd2_is_current IS ''SCD2 当前版本标记：1=当前版本，0=历史版本''', rec.table_name);
EXECUTE format('COMMENT ON COLUMN billiards_dwd.%I.scd2_version IS ''SCD2 版本号，自增，配合时间段避免重叠''', rec.table_name);
END LOOP;
-- 约束：同一业务键时间段不重叠，且仅有一条当前版本
FOR rec IN (
SELECT tc.table_name,
string_agg(format('%I WITH =', kcu.column_name), ', ' ORDER BY kcu.ordinal_position) AS pk_eq_expr,
string_agg(format('%I', kcu.column_name), ', ' ORDER BY kcu.ordinal_position) AS pk_cols
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.table_schema = kcu.table_schema
AND tc.table_name = kcu.table_name
AND tc.constraint_name = kcu.constraint_name
WHERE tc.table_schema = 'billiards_dwd'
AND tc.constraint_type = 'PRIMARY KEY'
AND EXISTS (
SELECT 1 FROM information_schema.columns c
WHERE c.table_schema = 'billiards_dwd'
AND c.table_name = tc.table_name
AND c.column_name = 'scd2_start_time'
)
GROUP BY tc.table_name
)
LOOP
IF NOT EXISTS (
SELECT 1 FROM pg_constraint
WHERE conname = format('%s_scd2_no_overlap', rec.table_name)
AND conrelid = format('billiards_dwd.%s', rec.table_name)::regclass
) THEN
EXECUTE format(
'ALTER TABLE billiards_dwd.%I ADD CONSTRAINT %I EXCLUDE USING gist (%s, tstzrange(scd2_start_time, scd2_end_time) WITH &&) WHERE (scd2_is_current = 1);',
rec.table_name,
rec.table_name || '_scd2_no_overlap',
rec.pk_eq_expr
);
END IF;
IF to_regclass(format('billiards_dwd.%s_scd2_current_unique_idx', rec.table_name)) IS NULL THEN
EXECUTE format(
'CREATE UNIQUE INDEX %I ON billiards_dwd.%I (%s) WHERE (scd2_is_current = 1);',
rec.table_name || '_scd2_current_unique_idx',
rec.table_name,
rec.pk_cols
);
END IF;
END LOOP;
END
$$;
-- SCD2 统一约定（DIM 表使用）：
-- SCD2_start_time TIMESTAMPTZ DEFAULT now() -- 版本开始时间
-- SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31' -- 版本结束时间
-- SCD2_is_current INT DEFAULT 1 -- 当前版本标记（1当前/0历史）
-- SCD2_version INT DEFAULT 1 -- 版本号，自增
-- dim_site
CREATE TABLE IF NOT EXISTS dim_site (
site_id BIGINT,
org_id BIGINT,
shop_name TEXT,
business_tel TEXT,
full_address TEXT,
tenant_id BIGINT,
shop_name TEXT,
site_label TEXT,
full_address TEXT,
address TEXT,
longitude NUMERIC(10,6),
latitude NUMERIC(10,6),
tenant_site_region_id BIGINT,
business_tel TEXT,
site_type INTEGER,
shop_status INTEGER,
SCD2_start_time TIMESTAMPTZ DEFAULT now(),
SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31',
SCD2_is_current INT DEFAULT 1,
SCD2_version INT DEFAULT 1,
PRIMARY KEY (site_id)
);
COMMENT ON COLUMN dim_site.site_id IS '门店主键 ID，唯一标识一家门店。与所有事实表中的 site_id 对应。 | 来源: siteProfile.id | 角色: 主键';
COMMENT ON COLUMN dim_site.org_id IS '上级组织 ID，用于区域组织划分。 | 来源: siteProfile.org_id | 角色: 外键';
COMMENT ON COLUMN dim_site.shop_name IS '门店名称，展示用。 | 来源: siteProfile.shop_name';
COMMENT ON COLUMN dim_site.business_tel IS '门店电话。 | 来源: siteProfile.business_tel';
COMMENT ON COLUMN dim_site.full_address IS '门店完整地址。 | 来源: siteProfile.full_address';
COMMENT ON COLUMN dim_site.tenant_id IS '租户 ID。与其它表 tenant_id 对应。 | 来源: siteProfile.tenant_id | 角色: 外键';
COMMENT ON COLUMN dim_site.site_id IS '???? ID?????????????????? site_id ??? | ??: siteProfile.id | ??: ??';
COMMENT ON COLUMN dim_site.org_id IS '???? ID?????????? | ??: siteProfile.org_id | ??: ??';
COMMENT ON COLUMN dim_site.tenant_id IS '?? ID????? tenant_id ??? | ??: siteProfile.tenant_id | ??: ??';
COMMENT ON COLUMN dim_site.shop_name IS '????????? | ??: siteProfile.shop_name';
COMMENT ON COLUMN dim_site.site_label IS '???????????????? | ??: siteProfile.site_label';
COMMENT ON COLUMN dim_site.full_address IS '??????? | ??: siteProfile.full_address';
COMMENT ON COLUMN dim_site.address IS '???????????? | ??: siteProfile.address';
COMMENT ON COLUMN dim_site.longitude IS '???????? | ??: siteProfile.longitude';
COMMENT ON COLUMN dim_site.latitude IS '???????? | ??: siteProfile.latitude';
COMMENT ON COLUMN dim_site.tenant_site_region_id IS '????/?????????? | ??: siteProfile.tenant_site_region_id';
COMMENT ON COLUMN dim_site.business_tel IS '????? | ??: siteProfile.business_tel';
COMMENT ON COLUMN dim_site.site_type IS '??????????????????? | ??: siteProfile.site_type';
COMMENT ON COLUMN dim_site.shop_status IS '??????????????????? | ??: siteProfile.shop_status';
-- dim_site_Ex
CREATE TABLE IF NOT EXISTS dim_site_Ex (
@@ -42,6 +137,10 @@ CREATE TABLE IF NOT EXISTS dim_site_Ex (
shop_status INTEGER,
create_time TIMESTAMPTZ,
update_time TIMESTAMPTZ,
SCD2_start_time TIMESTAMPTZ DEFAULT now(),
SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31',
SCD2_is_current INT DEFAULT 1,
SCD2_version INT DEFAULT 1,
PRIMARY KEY (site_id)
);
COMMENT ON COLUMN dim_site_Ex.site_id IS '门店主键 ID，唯一标识一家门店。与所有事实表中的 site_id 对应。 | 来源: siteProfile.id | 角色: 主键';
@@ -69,17 +168,19 @@ COMMENT ON COLUMN dim_site_Ex.update_time IS '门店最近更新时间。 | 来
-- dim_table
CREATE TABLE IF NOT EXISTS dim_table (
table_id BIGINT,
tenant_id BIGINT,
site_id BIGINT,
table_name TEXT,
site_table_area_id BIGINT,
site_table_area_name TEXT,
tenant_table_area_id BIGINT,
table_price NUMERIC(18,2),
SCD2_start_time TIMESTAMPTZ DEFAULT now(),
SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31',
SCD2_is_current INT DEFAULT 1,
SCD2_version INT DEFAULT 1,
PRIMARY KEY (table_id)
);
COMMENT ON COLUMN dim_table.table_id IS '台桌主键，唯一标识一张台或包厢。 | 来源: id | 角色: 主键';
COMMENT ON COLUMN dim_table.tenant_id IS '租户 ID。 | 来源: tenantId | 角色: 外键';
COMMENT ON COLUMN dim_table.site_id IS '门店 ID。 | 来源: siteId | 角色: 外键';
COMMENT ON COLUMN dim_table.table_name IS '台桌名称/编号，如 A17、888。 | 来源: tableName';
COMMENT ON COLUMN dim_table.site_table_area_id IS '门店区 ID，用于区分 A区/B区/补时区等。 | 来源: siteTableAreaId | 角色: 外键';
@@ -95,8 +196,10 @@ CREATE TABLE IF NOT EXISTS dim_table_Ex (
table_cloth_use_time INTEGER,
table_cloth_use_cycle INTEGER,
table_status INTEGER,
last_maintenance_time TIMESTAMPTZ,
remark TEXT,
SCD2_start_time TIMESTAMPTZ DEFAULT now(),
SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31',
SCD2_is_current INT DEFAULT 1,
SCD2_version INT DEFAULT 1,
PRIMARY KEY (table_id)
);
COMMENT ON COLUMN dim_table_Ex.table_id IS '台桌主键，唯一标识一张台或包厢。 | 来源: id | 角色: 主键';
@@ -105,8 +208,6 @@ COMMENT ON COLUMN dim_table_Ex.is_online_reservation IS '是否可线上预约
COMMENT ON COLUMN dim_table_Ex.table_cloth_use_time IS '已使用台呢时长（秒）。 | 来源: tableClothUseTime';
COMMENT ON COLUMN dim_table_Ex.table_cloth_use_cycle IS '台呢更换周期阈值（秒）。 | 来源: tableClothUseCycle';
COMMENT ON COLUMN dim_table_Ex.table_status IS '当前台桌状态：1=空闲，2=使用中，3=暂停中，4=锁定。 | 来源: tableStatus';
COMMENT ON COLUMN dim_table_Ex.last_maintenance_time IS '最近保养时间（未在 JSON 中出现）。 | 来源: lastMaintenanceTime';
COMMENT ON COLUMN dim_table_Ex.remark IS '备注信息。 | 来源: remark';
-- dim_assistant
CREATE TABLE IF NOT EXISTS dim_assistant (
@@ -125,6 +226,10 @@ CREATE TABLE IF NOT EXISTS dim_assistant (
resign_time TIMESTAMPTZ,
leave_status INTEGER,
assistant_status INTEGER,
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (assistant_id)
);
COMMENT ON COLUMN dim_assistant.assistant_id IS '助教账号 ID，关联助教服务流水表。 | 来源: id | 角色: 主键';
@@ -189,6 +294,10 @@ CREATE TABLE IF NOT EXISTS dim_assistant_Ex (
light_status INTEGER,
is_team_leader INTEGER,
serial_number BIGINT,
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (assistant_id)
);
COMMENT ON COLUMN dim_assistant_Ex.assistant_id IS '助教账号 ID，关联助教服务流水表。 | 来源: id | 角色: 主键';
@@ -248,6 +357,10 @@ CREATE TABLE IF NOT EXISTS dim_member (
member_card_grade_name TEXT,
create_time TIMESTAMPTZ,
update_time TIMESTAMPTZ,
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (member_id)
);
COMMENT ON COLUMN dim_member.member_id IS '租户内会员主键。 | 来源: id | 角色: 主键';
@@ -259,7 +372,6 @@ COMMENT ON COLUMN dim_member.nickname IS '昵称（未必是真实姓名）。 |
COMMENT ON COLUMN dim_member.member_card_grade_code IS '会员等级代码：1=金卡？2=银卡？3=钻石卡？4=黑卡？（按照 MD 文档枚举）。 | 来源: member_card_grade_code';
COMMENT ON COLUMN dim_member.member_card_grade_name IS '等级名称，中文描述。 | 来源: member_card_grade_name';
COMMENT ON COLUMN dim_member.create_time IS '会员档案创建时间。 | 来源: create_time';
COMMENT ON COLUMN dim_member.update_time IS '最近更新时间。 | 来源: update_time';
-- dim_member_Ex
CREATE TABLE IF NOT EXISTS dim_member_Ex (
@@ -270,6 +382,10 @@ CREATE TABLE IF NOT EXISTS dim_member_Ex (
growth_value NUMERIC(18,2),
user_status INTEGER,
status INTEGER,
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (member_id)
);
COMMENT ON COLUMN dim_member_Ex.member_id IS '租户内会员主键。 | 来源: id | 角色: 主键';
@@ -299,6 +415,10 @@ CREATE TABLE IF NOT EXISTS dim_member_card_account (
last_consume_time TIMESTAMPTZ,
status INTEGER,
is_delete INTEGER,
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (member_card_id)
);
COMMENT ON COLUMN dim_member_card_account.member_card_id IS '会员卡账户主键，唯一标识一张具体卡。 | 来源: id | 角色: 主键';
@@ -373,6 +493,10 @@ CREATE TABLE IF NOT EXISTS dim_member_card_account_Ex (
goodsCategoryId TEXT,
pdAssisnatLevel TEXT,
cxAssisnatLevel TEXT,
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (member_card_id)
);
COMMENT ON COLUMN dim_member_card_account_Ex.member_card_id IS '会员卡账户主键，唯一标识一张具体卡。 | 来源: id | 角色: 主键';
@@ -444,6 +568,10 @@ CREATE TABLE IF NOT EXISTS dim_tenant_goods (
create_time TIMESTAMPTZ,
update_time TIMESTAMPTZ,
is_delete INTEGER,
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (tenant_goods_id)
);
COMMENT ON COLUMN dim_tenant_goods.tenant_goods_id IS '租户级商品档案主键 ID，唯一标识一条商品档案。所有业务事实表（销售、库存等）中引用租户级商品时应指向此字段。 | 来源: id | 角色: 主键';
@@ -481,6 +609,10 @@ CREATE TABLE IF NOT EXISTS dim_tenant_goods_Ex (
common_sale_royalty INTEGER,
point_sale_royalty INTEGER,
out_goods_id BIGINT,
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (tenant_goods_id)
);
COMMENT ON COLUMN dim_tenant_goods_Ex.tenant_goods_id IS '租户级商品档案主键 ID，唯一标识一条商品档案。所有业务事实表（销售、库存等）中引用租户级商品时应指向此字段。 | 来源: id | 角色: 主键';
@@ -524,6 +656,10 @@ CREATE TABLE IF NOT EXISTS dim_store_goods (
enable_status INTEGER,
send_state INTEGER,
is_delete INTEGER,
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (site_goods_id)
);
COMMENT ON COLUMN dim_store_goods.site_goods_id IS '门店级商品 ID，本表主键；其它业务表中的 site_goods_id 与此对应，用于库存、销售等关联。 | 来源: id | 角色: 主键';
@@ -575,6 +711,10 @@ CREATE TABLE IF NOT EXISTS dim_store_goods_Ex (
option_required INTEGER,
remark TEXT,
sort_order INTEGER,
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (site_goods_id)
);
COMMENT ON COLUMN dim_store_goods_Ex.site_goods_id IS '门店级商品 ID，本表主键；其它业务表中的 site_goods_id 与此对应，用于库存、销售等关联。 | 来源: id | 角色: 主键';
@@ -618,6 +758,10 @@ CREATE TABLE IF NOT EXISTS dim_goods_category (
open_salesman INTEGER,
sort_order INTEGER,
is_warehousing INTEGER,
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (category_id)
);
COMMENT ON COLUMN dim_goods_category.category_id IS '分类节点主键。来自分类树节点的 id，在整个商品分类维度内唯一。用于在事实表中作为商品分类外键引用。 | 来源: id | 角色: 主键';
@@ -651,6 +795,10 @@ CREATE TABLE IF NOT EXISTS dim_groupbuy_package (
create_time TIMESTAMPTZ,
tenant_table_area_id_list VARCHAR(512),
card_type_ids VARCHAR(255),
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (groupbuy_package_id)
);
COMMENT ON COLUMN dim_groupbuy_package.groupbuy_package_id IS '门店侧团购套餐主键。每条记录一个套餐定义，供团购券核销记录指向。平台验券记录中的 group_package_id 通常指向这里。 | 来源: id | 角色: 主键';
@@ -692,6 +840,10 @@ CREATE TABLE IF NOT EXISTS dim_groupbuy_package_Ex (
effective_status INTEGER,
max_selectable_categories INTEGER,
creator_name VARCHAR(100),
SCD2_start_time TIMESTAMPTZ,
SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT,
SCD2_version INT,
PRIMARY KEY (groupbuy_package_id)
);
COMMENT ON COLUMN dim_groupbuy_package_Ex.groupbuy_package_id IS '门店侧团购套餐主键。每条记录一个套餐定义，供团购券核销记录指向。平台验券记录中的 group_package_id 通常指向这里。 | 来源: id | 角色: 主键';

105
etl_billiards/database/schema_etl_admin.sql Normal file
View File

@@ -0,0 +1,105 @@
-- 文件说明：etl_admin 调度元数据 DDL（独立文件，便于初始化任务单独执行）。
-- 包含任务注册表、游标表、运行记录表；字段注释使用中文。
CREATE SCHEMA IF NOT EXISTS etl_admin;
CREATE TABLE IF NOT EXISTS etl_admin.etl_task (
task_id BIGSERIAL PRIMARY KEY,
task_code TEXT NOT NULL,
store_id BIGINT NOT NULL,
enabled BOOLEAN DEFAULT TRUE,
cursor_field TEXT,
window_minutes_default INT DEFAULT 30,
overlap_seconds INT DEFAULT 120,
page_size INT DEFAULT 200,
retry_max INT DEFAULT 3,
params JSONB DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now(),
UNIQUE (task_code, store_id)
);
COMMENT ON TABLE etl_admin.etl_task IS '任务注册表：调度依据的任务清单（与 task_registry 中的任务码对应）。';
COMMENT ON COLUMN etl_admin.etl_task.task_code IS '任务编码，需与代码中的任务码一致。';
COMMENT ON COLUMN etl_admin.etl_task.store_id IS '门店/租户粒度，区分多门店执行。';
COMMENT ON COLUMN etl_admin.etl_task.enabled IS '是否启用此任务。';
COMMENT ON COLUMN etl_admin.etl_task.cursor_field IS '增量游标字段名（可选）。';
COMMENT ON COLUMN etl_admin.etl_task.window_minutes_default IS '默认时间窗口（分钟）。';
COMMENT ON COLUMN etl_admin.etl_task.overlap_seconds IS '窗口重叠秒数，用于防止遗漏。';
COMMENT ON COLUMN etl_admin.etl_task.page_size IS '默认分页大小。';
COMMENT ON COLUMN etl_admin.etl_task.retry_max IS 'API重试次数上限。';
COMMENT ON COLUMN etl_admin.etl_task.params IS '任务级自定义参数 JSON。';
COMMENT ON COLUMN etl_admin.etl_task.created_at IS '创建时间。';
COMMENT ON COLUMN etl_admin.etl_task.updated_at IS '更新时间。';
CREATE TABLE IF NOT EXISTS etl_admin.etl_cursor (
cursor_id BIGSERIAL PRIMARY KEY,
task_id BIGINT NOT NULL REFERENCES etl_admin.etl_task(task_id) ON DELETE CASCADE,
store_id BIGINT NOT NULL,
last_start TIMESTAMPTZ,
last_end TIMESTAMPTZ,
last_id BIGINT,
last_run_id BIGINT,
extra JSONB DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now(),
UNIQUE (task_id, store_id)
);
COMMENT ON TABLE etl_admin.etl_cursor IS '任务游标表：记录每个任务/门店的增量窗口及最后 run。';
COMMENT ON COLUMN etl_admin.etl_cursor.task_id IS '关联 etl_task.task_id。';
COMMENT ON COLUMN etl_admin.etl_cursor.store_id IS '门店/租户粒度。';
COMMENT ON COLUMN etl_admin.etl_cursor.last_start IS '上次窗口开始时间（含重叠偏移）。';
COMMENT ON COLUMN etl_admin.etl_cursor.last_end IS '上次窗口结束时间。';
COMMENT ON COLUMN etl_admin.etl_cursor.last_id IS '上次处理的最大主键/游标值（可选）。';
COMMENT ON COLUMN etl_admin.etl_cursor.last_run_id IS '上次运行ID，对应 etl_run.run_id。';
COMMENT ON COLUMN etl_admin.etl_cursor.extra IS '附加游标信息 JSON。';
COMMENT ON COLUMN etl_admin.etl_cursor.created_at IS '创建时间。';
COMMENT ON COLUMN etl_admin.etl_cursor.updated_at IS '更新时间。';
CREATE TABLE IF NOT EXISTS etl_admin.etl_run (
run_id BIGSERIAL PRIMARY KEY,
run_uuid TEXT NOT NULL,
task_id BIGINT NOT NULL REFERENCES etl_admin.etl_task(task_id) ON DELETE CASCADE,
store_id BIGINT NOT NULL,
status TEXT NOT NULL,
started_at TIMESTAMPTZ DEFAULT now(),
ended_at TIMESTAMPTZ,
window_start TIMESTAMPTZ,
window_end TIMESTAMPTZ,
window_minutes INT,
overlap_seconds INT,
fetched_count INT DEFAULT 0,
loaded_count INT DEFAULT 0,
updated_count INT DEFAULT 0,
skipped_count INT DEFAULT 0,
error_count INT DEFAULT 0,
unknown_fields INT DEFAULT 0,
export_dir TEXT,
log_path TEXT,
request_params JSONB DEFAULT '{}'::jsonb,
manifest JSONB DEFAULT '{}'::jsonb,
error_message TEXT,
extra JSONB DEFAULT '{}'::jsonb
);
COMMENT ON TABLE etl_admin.etl_run IS '运行记录表：记录每次任务执行的窗口、状态、计数与日志路径。';
COMMENT ON COLUMN etl_admin.etl_run.run_uuid IS '本次调度的唯一标识。';
COMMENT ON COLUMN etl_admin.etl_run.task_id IS '关联 etl_task.task_id。';
COMMENT ON COLUMN etl_admin.etl_run.store_id IS '门店/租户粒度。';
COMMENT ON COLUMN etl_admin.etl_run.status IS '运行状态（SUCC/FAIL/PARTIAL 等）。';
COMMENT ON COLUMN etl_admin.etl_run.started_at IS '开始时间。';
COMMENT ON COLUMN etl_admin.etl_run.ended_at IS '结束时间。';
COMMENT ON COLUMN etl_admin.etl_run.window_start IS '本次窗口开始时间。';
COMMENT ON COLUMN etl_admin.etl_run.window_end IS '本次窗口结束时间。';
COMMENT ON COLUMN etl_admin.etl_run.window_minutes IS '窗口跨度（分钟）。';
COMMENT ON COLUMN etl_admin.etl_run.overlap_seconds IS '窗口重叠秒数。';
COMMENT ON COLUMN etl_admin.etl_run.fetched_count IS '抓取/读取的记录数。';
COMMENT ON COLUMN etl_admin.etl_run.loaded_count IS '插入的记录数。';
COMMENT ON COLUMN etl_admin.etl_run.updated_count IS '更新的记录数。';
COMMENT ON COLUMN etl_admin.etl_run.skipped_count IS '跳过的记录数。';
COMMENT ON COLUMN etl_admin.etl_run.error_count IS '错误记录数。';
COMMENT ON COLUMN etl_admin.etl_run.unknown_fields IS '未知字段计数（清洗阶段）。';
COMMENT ON COLUMN etl_admin.etl_run.export_dir IS '抓取/导出目录。';
COMMENT ON COLUMN etl_admin.etl_run.log_path IS '日志路径。';
COMMENT ON COLUMN etl_admin.etl_run.request_params IS '请求参数 JSON。';
COMMENT ON COLUMN etl_admin.etl_run.manifest IS '运行产出清单/统计 JSON。';
COMMENT ON COLUMN etl_admin.etl_run.error_message IS '错误信息（若失败）。';
COMMENT ON COLUMN etl_admin.etl_run.extra IS '附加字段，保留扩展。';

1128
etl_billiards/database/schema_v2.sql
View File

File diff suppressed because it is too large Load Diff

28
etl_billiards/database/seed_ods_tasks.sql
View File

@@ -1,34 +1,35 @@
-- 将新的 ODS 任务注册到 etl_admin.etl_task（根据需要替换 store_id）
-- 使用方式（示例）：
-- 灏嗘柊鐨?ODS 浠诲姟娉ㄥ唽鍒?etl_admin.etl_task锛堟牴鎹渶瑕佹浛鎹?store_id锛?
-- 浣跨敤鏂瑰紡锛堢ず渚嬶級锛?
-- psql "$PG_DSN" -f etl_billiards/database/seed_ods_tasks.sql
-- 或者在 psql 中执行本文件内容。
-- 鎴栬€呭湪 psql 涓墽琛屾湰鏂囦欢鍐呭銆?
WITH target_store AS (
SELECT 2790685415443269::bigint AS store_id -- TODO: 替换为实际 store_id
SELECT 2790685415443269::bigint AS store_id -- TODO: 鏇挎崲涓哄疄闄?store_id
),
task_codes AS (
SELECT unnest(ARRAY[
'ODS_ASSISTANT_ACCOUNTS',
'ODS_ASSISTANT_LEDGER',
'ODS_ASSISTANT_ABOLISH',
'ODS_INVENTORY_CHANGE',
'assistant_accounts_masterS',
'assistant_service_records',
'assistant_cancellation_records',
'goods_stock_movements',
'ODS_INVENTORY_STOCK',
'ODS_PACKAGE',
'ODS_GROUP_BUY_REDEMPTION',
'ODS_MEMBER',
'ODS_MEMBER_BALANCE',
'ODS_MEMBER_CARD',
'member_stored_value_cards',
'ODS_PAYMENT',
'ODS_REFUND',
'ODS_COUPON_VERIFY',
'ODS_RECHARGE_SETTLE',
'platform_coupon_redemption_records',
'recharge_settlements',
'ODS_TABLES',
'ODS_GOODS_CATEGORY',
'ODS_STORE_GOODS',
'ODS_TABLE_DISCOUNT',
'table_fee_discount_records',
'ODS_TENANT_GOODS',
'ODS_SETTLEMENT_TICKET',
'ODS_ORDER_SETTLE'
'settlement_records',
'INIT_ODS_SCHEMA'
]) AS task_code
)
INSERT INTO etl_admin.etl_task (task_code, store_id, enabled)
@@ -37,3 +38,4 @@ FROM task_codes t CROSS JOIN target_store s
ON CONFLICT (task_code, store_id) DO UPDATE
SET enabled = EXCLUDED.enabled;