init: 项目初始提交 - NeoZQYY Monorepo 完整代码
This commit is contained in:
0
db/etl_feiqiu/schemas/.gitkeep
Normal file
0
db/etl_feiqiu/schemas/.gitkeep
Normal file
214
db/etl_feiqiu/schemas/app.sql
Normal file
214
db/etl_feiqiu/schemas/app.sql
Normal file
@@ -0,0 +1,214 @@
|
||||
-- =============================================================================
|
||||
-- app schema DDL — 面向外部访问的视图/函数 + RLS 策略
|
||||
-- 说明：以视图封装 DWS/Core 层数据，所有视图启用 RLS，以 site_id 过滤
|
||||
-- 不存储实际数据，仅做访问层
|
||||
-- =============================================================================
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS app;
|
||||
SET search_path TO app;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 应用角色（供 FDW 和外部应用使用）
|
||||
-- -----------------------------------------------------------------------------
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app_reader') THEN
|
||||
CREATE ROLE app_reader;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
GRANT USAGE ON SCHEMA app TO app_reader;
|
||||
GRANT USAGE ON SCHEMA core TO app_reader;
|
||||
GRANT USAGE ON SCHEMA dws TO app_reader;
|
||||
|
||||
-- =============================================================================
|
||||
-- 第一部分：基于 Core 层的视图
|
||||
-- =============================================================================
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 1. v_site — 门店视图
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE OR REPLACE VIEW app.v_site AS
|
||||
SELECT
|
||||
s.site_id,
|
||||
s.tenant_id,
|
||||
s.shop_name,
|
||||
s.site_label,
|
||||
s.shop_status
|
||||
FROM core.dim_site s;
|
||||
|
||||
COMMENT ON VIEW app.v_site IS '门店视图：封装 core.dim_site，供外部应用访问。';
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 2. v_member — 会员视图
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE OR REPLACE VIEW app.v_member AS
|
||||
SELECT
|
||||
m.member_id,
|
||||
m.system_member_id,
|
||||
m.tenant_id,
|
||||
m.register_site_id AS site_id,
|
||||
m.mobile,
|
||||
m.nickname,
|
||||
m.member_card_grade_name,
|
||||
m.status
|
||||
FROM core.dim_member m;
|
||||
|
||||
COMMENT ON VIEW app.v_member IS '会员视图：封装 core.dim_member，以 register_site_id 作为 site_id。';
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 3. v_assistant — 助教视图
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE OR REPLACE VIEW app.v_assistant AS
|
||||
SELECT
|
||||
a.assistant_id,
|
||||
a.tenant_id,
|
||||
a.site_id,
|
||||
a.real_name,
|
||||
a.nickname,
|
||||
a.mobile,
|
||||
a.level,
|
||||
a.assistant_status,
|
||||
a.leave_status
|
||||
FROM core.dim_assistant a;
|
||||
|
||||
COMMENT ON VIEW app.v_assistant IS '助教视图：封装 core.dim_assistant。';
|
||||
|
||||
-- =============================================================================
|
||||
-- 第二部分：基于 DWS 层的汇总视图
|
||||
-- =============================================================================
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 4. v_assistant_daily — 助教日明细视图
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE OR REPLACE VIEW app.v_assistant_daily AS
|
||||
SELECT
|
||||
d.id,
|
||||
d.site_id,
|
||||
d.assistant_id,
|
||||
d.stat_date,
|
||||
d.total_service_hours,
|
||||
d.total_service_count,
|
||||
d.total_revenue,
|
||||
d.basic_hours,
|
||||
d.extra_hours,
|
||||
d.tip_hours,
|
||||
d.created_at
|
||||
FROM dws.dws_assistant_daily_detail d;
|
||||
|
||||
COMMENT ON VIEW app.v_assistant_daily IS '助教日明细视图：封装 dws.dws_assistant_daily_detail。';
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 5. v_finance_daily — 财务日报视图
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE OR REPLACE VIEW app.v_finance_daily AS
|
||||
SELECT
|
||||
f.id,
|
||||
f.site_id,
|
||||
f.stat_date,
|
||||
f.total_revenue,
|
||||
f.table_fee_revenue,
|
||||
f.goods_revenue,
|
||||
f.assistant_revenue,
|
||||
f.recharge_revenue,
|
||||
f.total_orders,
|
||||
f.total_customers,
|
||||
f.created_at
|
||||
FROM dws.dws_finance_daily_summary f;
|
||||
|
||||
COMMENT ON VIEW app.v_finance_daily IS '财务日报视图：封装 dws.dws_finance_daily_summary。';
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 6. v_member_consumption — 会员消费汇总视图
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE OR REPLACE VIEW app.v_member_consumption AS
|
||||
SELECT
|
||||
mc.id,
|
||||
mc.site_id,
|
||||
mc.member_id,
|
||||
mc.total_visits,
|
||||
mc.total_consumption,
|
||||
mc.last_visit_date,
|
||||
mc.first_visit_date,
|
||||
mc.avg_consumption,
|
||||
mc.created_at
|
||||
FROM dws.dws_member_consumption_summary mc;
|
||||
|
||||
COMMENT ON VIEW app.v_member_consumption IS '会员消费汇总视图：封装 dws.dws_member_consumption_summary。';
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 7. v_order_summary — 订单汇总视图
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE OR REPLACE VIEW app.v_order_summary AS
|
||||
SELECT
|
||||
os.site_id,
|
||||
os.order_settle_id,
|
||||
os.order_trade_no,
|
||||
os.member_id,
|
||||
os.total_amount,
|
||||
os.actual_amount,
|
||||
os.settle_time,
|
||||
os.pay_status,
|
||||
os.created_at
|
||||
FROM dws.dws_order_summary os;
|
||||
|
||||
COMMENT ON VIEW app.v_order_summary IS '订单汇总视图：封装 dws.dws_order_summary。';
|
||||
|
||||
-- =============================================================================
|
||||
-- 第三部分：RLS 策略
|
||||
-- 说明：所有视图基于 site_id 隔离，通过会话变量 app.current_site_id 过滤
|
||||
-- 使用方式：SET app.current_site_id = '2790685415443269';
|
||||
-- =============================================================================
|
||||
|
||||
-- 对视图底层表启用 RLS（视图本身不支持 RLS，需在底层表上设置）
|
||||
-- 由于 app schema 的视图引用 core/dws 表，RLS 需在源表上配置
|
||||
|
||||
-- core 层 RLS
|
||||
ALTER TABLE core.dim_site ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE core.dim_member ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE core.dim_assistant ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE core.dim_table ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE core.fact_settlement ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE core.fact_payment ENABLE ROW LEVEL SECURITY;
|
||||
|
||||
-- core 层策略
|
||||
CREATE POLICY site_isolation_dim_site ON core.dim_site
|
||||
FOR SELECT TO app_reader
|
||||
USING (site_id = current_setting('app.current_site_id')::bigint);
|
||||
|
||||
CREATE POLICY site_isolation_dim_member ON core.dim_member
|
||||
FOR SELECT TO app_reader
|
||||
USING (register_site_id = current_setting('app.current_site_id')::bigint);
|
||||
|
||||
CREATE POLICY site_isolation_dim_assistant ON core.dim_assistant
|
||||
FOR SELECT TO app_reader
|
||||
USING (site_id = current_setting('app.current_site_id')::bigint);
|
||||
|
||||
CREATE POLICY site_isolation_dim_table ON core.dim_table
|
||||
FOR SELECT TO app_reader
|
||||
USING (site_id = current_setting('app.current_site_id')::bigint);
|
||||
|
||||
CREATE POLICY site_isolation_fact_settlement ON core.fact_settlement
|
||||
FOR SELECT TO app_reader
|
||||
USING (site_id = current_setting('app.current_site_id')::bigint);
|
||||
|
||||
CREATE POLICY site_isolation_fact_payment ON core.fact_payment
|
||||
FOR SELECT TO app_reader
|
||||
USING (site_id = current_setting('app.current_site_id')::bigint);
|
||||
|
||||
-- =============================================================================
|
||||
-- 第四部分：授权
|
||||
-- =============================================================================
|
||||
|
||||
-- 授予 app_reader 对 core 表的 SELECT 权限
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA core TO app_reader;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA app TO app_reader;
|
||||
|
||||
-- 授予对 dws 表的 SELECT 权限（视图需要）
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA dws TO app_reader;
|
||||
|
||||
-- 设置默认权限（未来新建的表自动授权）
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA core GRANT SELECT ON TABLES TO app_reader;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA app GRANT SELECT ON TABLES TO app_reader;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA dws GRANT SELECT ON TABLES TO app_reader;
|
||||
161
db/etl_feiqiu/schemas/core.sql
Normal file
161
db/etl_feiqiu/schemas/core.sql
Normal file
@@ -0,0 +1,161 @@
|
||||
-- =============================================================================
|
||||
-- core schema DDL — 统一维度/事实最小字段集层
|
||||
-- 说明：从 DWD 维度表和事实表提取跨系统共享的核心字段
|
||||
-- 仅包含 ID、名称、状态、site_id 等最小字段集
|
||||
-- 第一版保持精简，后续按需扩展
|
||||
-- =============================================================================
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS core;
|
||||
SET search_path TO core;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 1. dim_site — 门店维度（核心字段）
|
||||
-- 来源：dwd.dim_site（14 字段 → 6 字段）
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS dim_site (
|
||||
site_id BIGINT PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
shop_name TEXT NOT NULL,
|
||||
site_label TEXT,
|
||||
shop_status INTEGER,
|
||||
site_id_alias BIGINT GENERATED ALWAYS AS (site_id) STORED
|
||||
);
|
||||
COMMENT ON TABLE core.dim_site IS '门店维度核心表：仅保留跨系统共享的最小字段集。';
|
||||
COMMENT ON COLUMN core.dim_site.site_id IS '门店 ID（主键）。';
|
||||
COMMENT ON COLUMN core.dim_site.tenant_id IS '租户/品牌 ID。';
|
||||
COMMENT ON COLUMN core.dim_site.shop_name IS '门店名称。';
|
||||
COMMENT ON COLUMN core.dim_site.site_label IS '门店标签（如 A/B 店）。';
|
||||
COMMENT ON COLUMN core.dim_site.shop_status IS '门店状态枚举。';
|
||||
-- site_id_alias 用于 RLS 策略中统一使用 site_id 过滤
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 2. dim_member — 会员维度（核心字段）
|
||||
-- 来源：dwd.dim_member（12 字段 → 8 字段）
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS dim_member (
|
||||
member_id BIGINT PRIMARY KEY,
|
||||
system_member_id BIGINT,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
register_site_id BIGINT NOT NULL,
|
||||
mobile TEXT,
|
||||
nickname TEXT,
|
||||
member_card_grade_name TEXT,
|
||||
status INTEGER
|
||||
);
|
||||
COMMENT ON TABLE core.dim_member IS '会员维度核心表：仅保留跨系统共享的最小字段集。';
|
||||
COMMENT ON COLUMN core.dim_member.member_id IS '会员 ID（主键）。';
|
||||
COMMENT ON COLUMN core.dim_member.system_member_id IS '系统级会员 ID（跨门店统一）。';
|
||||
COMMENT ON COLUMN core.dim_member.tenant_id IS '租户/品牌 ID。';
|
||||
COMMENT ON COLUMN core.dim_member.register_site_id IS '注册门店 ID（等同 site_id）。';
|
||||
COMMENT ON COLUMN core.dim_member.mobile IS '手机号。';
|
||||
COMMENT ON COLUMN core.dim_member.nickname IS '昵称/姓名。';
|
||||
COMMENT ON COLUMN core.dim_member.member_card_grade_name IS '会员卡等级名称。';
|
||||
COMMENT ON COLUMN core.dim_member.status IS '会员状态。';
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 3. dim_assistant — 助教维度（核心字段）
|
||||
-- 来源：dwd.dim_assistant（16 字段 → 9 字段）
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS dim_assistant (
|
||||
assistant_id BIGINT PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
site_id BIGINT NOT NULL,
|
||||
real_name TEXT NOT NULL,
|
||||
nickname TEXT,
|
||||
mobile TEXT,
|
||||
level INTEGER,
|
||||
assistant_status INTEGER,
|
||||
leave_status INTEGER
|
||||
);
|
||||
COMMENT ON TABLE core.dim_assistant IS '助教维度核心表：仅保留跨系统共享的最小字段集。';
|
||||
COMMENT ON COLUMN core.dim_assistant.assistant_id IS '助教 ID（主键）。';
|
||||
COMMENT ON COLUMN core.dim_assistant.tenant_id IS '租户/品牌 ID。';
|
||||
COMMENT ON COLUMN core.dim_assistant.site_id IS '门店 ID。';
|
||||
COMMENT ON COLUMN core.dim_assistant.real_name IS '真实姓名。';
|
||||
COMMENT ON COLUMN core.dim_assistant.nickname IS '昵称。';
|
||||
COMMENT ON COLUMN core.dim_assistant.mobile IS '手机号。';
|
||||
COMMENT ON COLUMN core.dim_assistant.level IS '助教等级。';
|
||||
COMMENT ON COLUMN core.dim_assistant.assistant_status IS '助教状态。';
|
||||
COMMENT ON COLUMN core.dim_assistant.leave_status IS '离职状态。';
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 4. dim_table — 台桌维度（核心字段）
|
||||
-- 来源：dwd.dim_table（8 字段 → 5 字段）
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS dim_table (
|
||||
table_id BIGINT PRIMARY KEY,
|
||||
site_id BIGINT NOT NULL,
|
||||
table_name TEXT NOT NULL,
|
||||
site_table_area_name TEXT,
|
||||
table_price NUMERIC(18,2)
|
||||
);
|
||||
COMMENT ON TABLE core.dim_table IS '台桌维度核心表：仅保留跨系统共享的最小字段集。';
|
||||
COMMENT ON COLUMN core.dim_table.table_id IS '台桌 ID（主键）。';
|
||||
COMMENT ON COLUMN core.dim_table.site_id IS '门店 ID。';
|
||||
COMMENT ON COLUMN core.dim_table.table_name IS '台桌名称。';
|
||||
COMMENT ON COLUMN core.dim_table.site_table_area_name IS '区域名称。';
|
||||
COMMENT ON COLUMN core.dim_table.table_price IS '台桌单价。';
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 5. dim_goods_category — 商品分类维度（核心字段）
|
||||
-- 来源：dwd.dim_goods_category（10 字段 → 5 字段）
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS dim_goods_category (
|
||||
category_id BIGINT PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
category_name TEXT NOT NULL,
|
||||
parent_id BIGINT,
|
||||
level INTEGER
|
||||
);
|
||||
COMMENT ON TABLE core.dim_goods_category IS '商品分类维度核心表。';
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 6. fact_settlement — 结算事实（核心字段）
|
||||
-- 来源：dwd.dwd_settlement_head（约 30 字段 → 12 字段）
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS fact_settlement (
|
||||
order_settle_id BIGINT PRIMARY KEY,
|
||||
site_id BIGINT NOT NULL,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
order_trade_no BIGINT,
|
||||
member_id BIGINT,
|
||||
total_amount NUMERIC(18,2),
|
||||
actual_amount NUMERIC(18,2),
|
||||
discount_amount NUMERIC(18,2),
|
||||
pay_status INTEGER,
|
||||
settle_time TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ
|
||||
);
|
||||
COMMENT ON TABLE core.fact_settlement IS '结算事实核心表：仅保留跨系统共享的最小字段集。';
|
||||
COMMENT ON COLUMN core.fact_settlement.order_settle_id IS '结账单 ID（主键）。';
|
||||
COMMENT ON COLUMN core.fact_settlement.site_id IS '门店 ID。';
|
||||
COMMENT ON COLUMN core.fact_settlement.tenant_id IS '租户 ID。';
|
||||
COMMENT ON COLUMN core.fact_settlement.total_amount IS '应收总额。';
|
||||
COMMENT ON COLUMN core.fact_settlement.actual_amount IS '实收金额。';
|
||||
COMMENT ON COLUMN core.fact_settlement.discount_amount IS '优惠金额。';
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 7. fact_payment — 支付事实（核心字段）
|
||||
-- 来源：dwd.dwd_payment（约 12 字段 → 7 字段）
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS fact_payment (
|
||||
payment_id BIGINT PRIMARY KEY,
|
||||
site_id BIGINT NOT NULL,
|
||||
order_settle_id BIGINT,
|
||||
pay_type INTEGER,
|
||||
pay_amount NUMERIC(18,2),
|
||||
pay_time TIMESTAMPTZ,
|
||||
status INTEGER
|
||||
);
|
||||
COMMENT ON TABLE core.fact_payment IS '支付事实核心表。';
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 索引
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE INDEX IF NOT EXISTS idx_core_member_site ON core.dim_member (register_site_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_core_assistant_site ON core.dim_assistant (site_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_core_table_site ON core.dim_table (site_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_core_settlement_site ON core.fact_settlement (site_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_core_settlement_time ON core.fact_settlement (settle_time);
|
||||
CREATE INDEX IF NOT EXISTS idx_core_payment_site ON core.fact_payment (site_id);
|
||||
2089
db/etl_feiqiu/schemas/dwd.sql
Normal file
2089
db/etl_feiqiu/schemas/dwd.sql
Normal file
File diff suppressed because it is too large
Load Diff
1675
db/etl_feiqiu/schemas/dws.sql
Normal file
1675
db/etl_feiqiu/schemas/dws.sql
Normal file
File diff suppressed because it is too large
Load Diff
105
db/etl_feiqiu/schemas/meta.sql
Normal file
105
db/etl_feiqiu/schemas/meta.sql
Normal file
@@ -0,0 +1,105 @@
|
||||
-- 文件说明：meta 调度元数据 DDL（独立文件，便于初始化任务单独执行）。
|
||||
-- 包含任务注册表、游标表、运行记录表；字段注释使用中文。
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS meta;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta.etl_task (
|
||||
task_id BIGSERIAL PRIMARY KEY,
|
||||
task_code TEXT NOT NULL,
|
||||
store_id BIGINT NOT NULL,
|
||||
enabled BOOLEAN DEFAULT TRUE,
|
||||
cursor_field TEXT,
|
||||
window_minutes_default INT DEFAULT 30,
|
||||
overlap_seconds INT DEFAULT 600,
|
||||
page_size INT DEFAULT 200,
|
||||
retry_max INT DEFAULT 3,
|
||||
params JSONB DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ DEFAULT now(),
|
||||
UNIQUE (task_code, store_id)
|
||||
);
|
||||
COMMENT ON TABLE meta.etl_task IS '任务注册表：调度依据的任务清单（与 task_registry 中的任务码对应）。';
|
||||
COMMENT ON COLUMN meta.etl_task.task_code IS '任务编码，需与代码中的任务码一致。';
|
||||
COMMENT ON COLUMN meta.etl_task.store_id IS '门店/租户粒度，区分多门店执行。';
|
||||
COMMENT ON COLUMN meta.etl_task.enabled IS '是否启用此任务。';
|
||||
COMMENT ON COLUMN meta.etl_task.cursor_field IS '增量游标字段名（可选）。';
|
||||
COMMENT ON COLUMN meta.etl_task.window_minutes_default IS '默认时间窗口（分钟）。';
|
||||
COMMENT ON COLUMN meta.etl_task.overlap_seconds IS '窗口重叠秒数，用于防止遗漏。';
|
||||
COMMENT ON COLUMN meta.etl_task.page_size IS '默认分页大小。';
|
||||
COMMENT ON COLUMN meta.etl_task.retry_max IS 'API重试次数上限。';
|
||||
COMMENT ON COLUMN meta.etl_task.params IS '任务级自定义参数 JSON。';
|
||||
COMMENT ON COLUMN meta.etl_task.created_at IS '创建时间。';
|
||||
COMMENT ON COLUMN meta.etl_task.updated_at IS '更新时间。';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta.etl_cursor (
|
||||
cursor_id BIGSERIAL PRIMARY KEY,
|
||||
task_id BIGINT NOT NULL REFERENCES meta.etl_task(task_id) ON DELETE CASCADE,
|
||||
store_id BIGINT NOT NULL,
|
||||
last_start TIMESTAMPTZ,
|
||||
last_end TIMESTAMPTZ,
|
||||
last_id BIGINT,
|
||||
last_run_id BIGINT,
|
||||
extra JSONB DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ DEFAULT now(),
|
||||
UNIQUE (task_id, store_id)
|
||||
);
|
||||
COMMENT ON TABLE meta.etl_cursor IS '任务游标表：记录每个任务/门店的增量窗口及最后 run。';
|
||||
COMMENT ON COLUMN meta.etl_cursor.task_id IS '关联 etl_task.task_id。';
|
||||
COMMENT ON COLUMN meta.etl_cursor.store_id IS '门店/租户粒度。';
|
||||
COMMENT ON COLUMN meta.etl_cursor.last_start IS '上次窗口开始时间（含重叠偏移）。';
|
||||
COMMENT ON COLUMN meta.etl_cursor.last_end IS '上次窗口结束时间。';
|
||||
COMMENT ON COLUMN meta.etl_cursor.last_id IS '上次处理的最大主键/游标值（可选）。';
|
||||
COMMENT ON COLUMN meta.etl_cursor.last_run_id IS '上次运行ID，对应 etl_run.run_id。';
|
||||
COMMENT ON COLUMN meta.etl_cursor.extra IS '附加游标信息 JSON。';
|
||||
COMMENT ON COLUMN meta.etl_cursor.created_at IS '创建时间。';
|
||||
COMMENT ON COLUMN meta.etl_cursor.updated_at IS '更新时间。';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta.etl_run (
|
||||
run_id BIGSERIAL PRIMARY KEY,
|
||||
run_uuid TEXT NOT NULL,
|
||||
task_id BIGINT NOT NULL REFERENCES meta.etl_task(task_id) ON DELETE CASCADE,
|
||||
store_id BIGINT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
started_at TIMESTAMPTZ DEFAULT now(),
|
||||
ended_at TIMESTAMPTZ,
|
||||
window_start TIMESTAMPTZ,
|
||||
window_end TIMESTAMPTZ,
|
||||
window_minutes INT,
|
||||
overlap_seconds INT,
|
||||
fetched_count INT DEFAULT 0,
|
||||
loaded_count INT DEFAULT 0,
|
||||
updated_count INT DEFAULT 0,
|
||||
skipped_count INT DEFAULT 0,
|
||||
error_count INT DEFAULT 0,
|
||||
unknown_fields INT DEFAULT 0,
|
||||
export_dir TEXT,
|
||||
log_path TEXT,
|
||||
request_params JSONB DEFAULT '{}'::jsonb,
|
||||
manifest JSONB DEFAULT '{}'::jsonb,
|
||||
error_message TEXT,
|
||||
extra JSONB DEFAULT '{}'::jsonb
|
||||
);
|
||||
COMMENT ON TABLE meta.etl_run IS '运行记录表：记录每次任务执行的窗口、状态、计数与日志路径。';
|
||||
COMMENT ON COLUMN meta.etl_run.run_uuid IS '本次调度的唯一标识。';
|
||||
COMMENT ON COLUMN meta.etl_run.task_id IS '关联 etl_task.task_id。';
|
||||
COMMENT ON COLUMN meta.etl_run.store_id IS '门店/租户粒度。';
|
||||
COMMENT ON COLUMN meta.etl_run.status IS '运行状态（SUCC/FAIL/PARTIAL 等）。';
|
||||
COMMENT ON COLUMN meta.etl_run.started_at IS '开始时间。';
|
||||
COMMENT ON COLUMN meta.etl_run.ended_at IS '结束时间。';
|
||||
COMMENT ON COLUMN meta.etl_run.window_start IS '本次窗口开始时间。';
|
||||
COMMENT ON COLUMN meta.etl_run.window_end IS '本次窗口结束时间。';
|
||||
COMMENT ON COLUMN meta.etl_run.window_minutes IS '窗口跨度（分钟）。';
|
||||
COMMENT ON COLUMN meta.etl_run.overlap_seconds IS '窗口重叠秒数。';
|
||||
COMMENT ON COLUMN meta.etl_run.fetched_count IS '抓取/读取的记录数。';
|
||||
COMMENT ON COLUMN meta.etl_run.loaded_count IS '插入的记录数。';
|
||||
COMMENT ON COLUMN meta.etl_run.updated_count IS '更新的记录数。';
|
||||
COMMENT ON COLUMN meta.etl_run.skipped_count IS '跳过的记录数。';
|
||||
COMMENT ON COLUMN meta.etl_run.error_count IS '错误记录数。';
|
||||
COMMENT ON COLUMN meta.etl_run.unknown_fields IS '未知字段计数（清洗阶段）。';
|
||||
COMMENT ON COLUMN meta.etl_run.export_dir IS '抓取/导出目录。';
|
||||
COMMENT ON COLUMN meta.etl_run.log_path IS '日志路径。';
|
||||
COMMENT ON COLUMN meta.etl_run.request_params IS '请求参数 JSON。';
|
||||
COMMENT ON COLUMN meta.etl_run.manifest IS '运行产出清单/统计 JSON。';
|
||||
COMMENT ON COLUMN meta.etl_run.error_message IS '错误信息（若失败）。';
|
||||
COMMENT ON COLUMN meta.etl_run.extra IS '附加字段，保留扩展。';
|
||||
2057
db/etl_feiqiu/schemas/ods.sql
Normal file
2057
db/etl_feiqiu/schemas/ods.sql
Normal file
File diff suppressed because it is too large
Load Diff
2057
db/etl_feiqiu/schemas/schema_ODS_doc.sql
Normal file
2057
db/etl_feiqiu/schemas/schema_ODS_doc.sql
Normal file
File diff suppressed because it is too large
Load Diff
2089
db/etl_feiqiu/schemas/schema_dwd_doc.sql
Normal file
2089
db/etl_feiqiu/schemas/schema_dwd_doc.sql
Normal file
File diff suppressed because it is too large
Load Diff
1675
db/etl_feiqiu/schemas/schema_dws.sql
Normal file
1675
db/etl_feiqiu/schemas/schema_dws.sql
Normal file
File diff suppressed because it is too large
Load Diff
105
db/etl_feiqiu/schemas/schema_etl_admin.sql
Normal file
105
db/etl_feiqiu/schemas/schema_etl_admin.sql
Normal file
@@ -0,0 +1,105 @@
|
||||
-- 文件说明：etl_admin 调度元数据 DDL（独立文件，便于初始化任务单独执行）。
|
||||
-- 包含任务注册表、游标表、运行记录表；字段注释使用中文。
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS etl_admin;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS etl_admin.etl_task (
|
||||
task_id BIGSERIAL PRIMARY KEY,
|
||||
task_code TEXT NOT NULL,
|
||||
store_id BIGINT NOT NULL,
|
||||
enabled BOOLEAN DEFAULT TRUE,
|
||||
cursor_field TEXT,
|
||||
window_minutes_default INT DEFAULT 30,
|
||||
overlap_seconds INT DEFAULT 600,
|
||||
page_size INT DEFAULT 200,
|
||||
retry_max INT DEFAULT 3,
|
||||
params JSONB DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ DEFAULT now(),
|
||||
UNIQUE (task_code, store_id)
|
||||
);
|
||||
COMMENT ON TABLE etl_admin.etl_task IS '任务注册表：调度依据的任务清单（与 task_registry 中的任务码对应）。';
|
||||
COMMENT ON COLUMN etl_admin.etl_task.task_code IS '任务编码，需与代码中的任务码一致。';
|
||||
COMMENT ON COLUMN etl_admin.etl_task.store_id IS '门店/租户粒度，区分多门店执行。';
|
||||
COMMENT ON COLUMN etl_admin.etl_task.enabled IS '是否启用此任务。';
|
||||
COMMENT ON COLUMN etl_admin.etl_task.cursor_field IS '增量游标字段名（可选）。';
|
||||
COMMENT ON COLUMN etl_admin.etl_task.window_minutes_default IS '默认时间窗口（分钟）。';
|
||||
COMMENT ON COLUMN etl_admin.etl_task.overlap_seconds IS '窗口重叠秒数，用于防止遗漏。';
|
||||
COMMENT ON COLUMN etl_admin.etl_task.page_size IS '默认分页大小。';
|
||||
COMMENT ON COLUMN etl_admin.etl_task.retry_max IS 'API重试次数上限。';
|
||||
COMMENT ON COLUMN etl_admin.etl_task.params IS '任务级自定义参数 JSON。';
|
||||
COMMENT ON COLUMN etl_admin.etl_task.created_at IS '创建时间。';
|
||||
COMMENT ON COLUMN etl_admin.etl_task.updated_at IS '更新时间。';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS etl_admin.etl_cursor (
|
||||
cursor_id BIGSERIAL PRIMARY KEY,
|
||||
task_id BIGINT NOT NULL REFERENCES etl_admin.etl_task(task_id) ON DELETE CASCADE,
|
||||
store_id BIGINT NOT NULL,
|
||||
last_start TIMESTAMPTZ,
|
||||
last_end TIMESTAMPTZ,
|
||||
last_id BIGINT,
|
||||
last_run_id BIGINT,
|
||||
extra JSONB DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ DEFAULT now(),
|
||||
UNIQUE (task_id, store_id)
|
||||
);
|
||||
COMMENT ON TABLE etl_admin.etl_cursor IS '任务游标表：记录每个任务/门店的增量窗口及最后 run。';
|
||||
COMMENT ON COLUMN etl_admin.etl_cursor.task_id IS '关联 etl_task.task_id。';
|
||||
COMMENT ON COLUMN etl_admin.etl_cursor.store_id IS '门店/租户粒度。';
|
||||
COMMENT ON COLUMN etl_admin.etl_cursor.last_start IS '上次窗口开始时间（含重叠偏移）。';
|
||||
COMMENT ON COLUMN etl_admin.etl_cursor.last_end IS '上次窗口结束时间。';
|
||||
COMMENT ON COLUMN etl_admin.etl_cursor.last_id IS '上次处理的最大主键/游标值（可选）。';
|
||||
COMMENT ON COLUMN etl_admin.etl_cursor.last_run_id IS '上次运行ID，对应 etl_run.run_id。';
|
||||
COMMENT ON COLUMN etl_admin.etl_cursor.extra IS '附加游标信息 JSON。';
|
||||
COMMENT ON COLUMN etl_admin.etl_cursor.created_at IS '创建时间。';
|
||||
COMMENT ON COLUMN etl_admin.etl_cursor.updated_at IS '更新时间。';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS etl_admin.etl_run (
|
||||
run_id BIGSERIAL PRIMARY KEY,
|
||||
run_uuid TEXT NOT NULL,
|
||||
task_id BIGINT NOT NULL REFERENCES etl_admin.etl_task(task_id) ON DELETE CASCADE,
|
||||
store_id BIGINT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
started_at TIMESTAMPTZ DEFAULT now(),
|
||||
ended_at TIMESTAMPTZ,
|
||||
window_start TIMESTAMPTZ,
|
||||
window_end TIMESTAMPTZ,
|
||||
window_minutes INT,
|
||||
overlap_seconds INT,
|
||||
fetched_count INT DEFAULT 0,
|
||||
loaded_count INT DEFAULT 0,
|
||||
updated_count INT DEFAULT 0,
|
||||
skipped_count INT DEFAULT 0,
|
||||
error_count INT DEFAULT 0,
|
||||
unknown_fields INT DEFAULT 0,
|
||||
export_dir TEXT,
|
||||
log_path TEXT,
|
||||
request_params JSONB DEFAULT '{}'::jsonb,
|
||||
manifest JSONB DEFAULT '{}'::jsonb,
|
||||
error_message TEXT,
|
||||
extra JSONB DEFAULT '{}'::jsonb
|
||||
);
|
||||
COMMENT ON TABLE etl_admin.etl_run IS '运行记录表：记录每次任务执行的窗口、状态、计数与日志路径。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.run_uuid IS '本次调度的唯一标识。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.task_id IS '关联 etl_task.task_id。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.store_id IS '门店/租户粒度。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.status IS '运行状态（SUCC/FAIL/PARTIAL 等）。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.started_at IS '开始时间。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.ended_at IS '结束时间。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.window_start IS '本次窗口开始时间。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.window_end IS '本次窗口结束时间。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.window_minutes IS '窗口跨度（分钟）。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.overlap_seconds IS '窗口重叠秒数。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.fetched_count IS '抓取/读取的记录数。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.loaded_count IS '插入的记录数。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.updated_count IS '更新的记录数。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.skipped_count IS '跳过的记录数。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.error_count IS '错误记录数。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.unknown_fields IS '未知字段计数（清洗阶段）。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.export_dir IS '抓取/导出目录。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.log_path IS '日志路径。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.request_params IS '请求参数 JSON。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.manifest IS '运行产出清单/统计 JSON。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.error_message IS '错误信息（若失败）。';
|
||||
COMMENT ON COLUMN etl_admin.etl_run.extra IS '附加字段，保留扩展。';
|
||||
173
db/etl_feiqiu/schemas/schema_verify_perf_indexes.sql
Normal file
173
db/etl_feiqiu/schemas/schema_verify_perf_indexes.sql
Normal file
@@ -0,0 +1,173 @@
|
||||
SET client_encoding TO "UTF8";
|
||||
|
||||
-- ============================================================================
|
||||
-- 校验性能索引（ODS / DWD）
|
||||
-- ----------------------------------------------------------------------------
|
||||
-- 用途:
|
||||
-- 1) 加速校验查询（主键查找、窗口扫描、当前版本扫描）。
|
||||
-- 2) 保持数据语义不变（仅添加索引 + ANALYZE，不改写业务数据）。
|
||||
--
|
||||
-- 注意事项:
|
||||
-- 1) 本脚本具有幂等性（`CREATE INDEX IF NOT EXISTS`）。
|
||||
-- 2) 如有严格的在线 DDL 要求，请手动使用 `CREATE INDEX CONCURRENTLY`
|
||||
-- 在维护安全模式下执行（不可在事务块内运行）。
|
||||
-- ============================================================================
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
rec RECORD;
|
||||
pk_cols TEXT[];
|
||||
pk_cols_sql TEXT;
|
||||
idx_name TEXT;
|
||||
BEGIN
|
||||
FOR rec IN
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'billiards_ods'
|
||||
AND table_type = 'BASE TABLE'
|
||||
LOOP
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'billiards_ods'
|
||||
AND table_name = rec.table_name
|
||||
AND column_name = 'fetched_at'
|
||||
) THEN
|
||||
idx_name := left(format('idx_%s_vfy_fetched_at', rec.table_name), 50)
|
||||
|| '_' || substr(md5(rec.table_name || '_vfy_fetched_at'), 1, 8);
|
||||
EXECUTE format(
|
||||
'CREATE INDEX IF NOT EXISTS %I ON billiards_ods.%I (fetched_at)',
|
||||
idx_name, rec.table_name
|
||||
);
|
||||
|
||||
SELECT array_agg(kcu.column_name ORDER BY kcu.ordinal_position)
|
||||
INTO pk_cols
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.table_schema = kcu.table_schema
|
||||
AND tc.table_name = kcu.table_name
|
||||
AND tc.constraint_name = kcu.constraint_name
|
||||
WHERE tc.table_schema = 'billiards_ods'
|
||||
AND tc.table_name = rec.table_name
|
||||
AND tc.constraint_type = 'PRIMARY KEY';
|
||||
|
||||
IF pk_cols IS NOT NULL AND coalesce(array_length(pk_cols, 1), 0) <= 3 THEN
|
||||
SELECT string_agg(format('%I', c), ', ')
|
||||
INTO pk_cols_sql
|
||||
FROM unnest(pk_cols) AS c;
|
||||
|
||||
idx_name := left(format('idx_%s_vfy_fetched_pk', rec.table_name), 50)
|
||||
|| '_' || substr(md5(rec.table_name || '_vfy_fetched_pk'), 1, 8);
|
||||
EXECUTE format(
|
||||
'CREATE INDEX IF NOT EXISTS %I ON billiards_ods.%I (fetched_at, %s)',
|
||||
idx_name, rec.table_name, pk_cols_sql
|
||||
);
|
||||
END IF;
|
||||
END IF;
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
rec RECORD;
|
||||
tcol TEXT;
|
||||
pk_cols TEXT[];
|
||||
pk_cols_sql TEXT;
|
||||
idx_name TEXT;
|
||||
time_candidates TEXT[] := ARRAY[
|
||||
'pay_time',
|
||||
'create_time',
|
||||
'start_use_time',
|
||||
'scd2_start_time',
|
||||
'calc_time',
|
||||
'order_date',
|
||||
'fetched_at'
|
||||
];
|
||||
BEGIN
|
||||
FOR rec IN
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'billiards_dwd'
|
||||
AND table_type = 'BASE TABLE'
|
||||
LOOP
|
||||
SELECT array_agg(kcu.column_name ORDER BY kcu.ordinal_position)
|
||||
INTO pk_cols
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.table_schema = kcu.table_schema
|
||||
AND tc.table_name = kcu.table_name
|
||||
AND tc.constraint_name = kcu.constraint_name
|
||||
WHERE tc.table_schema = 'billiards_dwd'
|
||||
AND tc.table_name = rec.table_name
|
||||
AND tc.constraint_type = 'PRIMARY KEY';
|
||||
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'billiards_dwd'
|
||||
AND table_name = rec.table_name
|
||||
AND column_name = 'scd2_is_current'
|
||||
) AND pk_cols IS NOT NULL
|
||||
AND coalesce(array_length(pk_cols, 1), 0) BETWEEN 1 AND 4 THEN
|
||||
SELECT string_agg(format('%I', c), ', ')
|
||||
INTO pk_cols_sql
|
||||
FROM unnest(pk_cols) AS c;
|
||||
|
||||
idx_name := left(format('idx_%s_vfy_pk_current', rec.table_name), 50)
|
||||
|| '_' || substr(md5(rec.table_name || '_vfy_pk_current'), 1, 8);
|
||||
EXECUTE format(
|
||||
'CREATE INDEX IF NOT EXISTS %I ON billiards_dwd.%I (%s, scd2_is_current)',
|
||||
idx_name, rec.table_name, pk_cols_sql
|
||||
);
|
||||
END IF;
|
||||
|
||||
FOREACH tcol IN ARRAY time_candidates
|
||||
LOOP
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'billiards_dwd'
|
||||
AND table_name = rec.table_name
|
||||
AND column_name = tcol
|
||||
) THEN
|
||||
idx_name := left(format('idx_%s_vfy_%s', rec.table_name, tcol), 50)
|
||||
|| '_' || substr(md5(rec.table_name || '_vfy_' || tcol), 1, 8);
|
||||
EXECUTE format(
|
||||
'CREATE INDEX IF NOT EXISTS %I ON billiards_dwd.%I (%I)',
|
||||
idx_name, rec.table_name, tcol
|
||||
);
|
||||
|
||||
IF pk_cols IS NOT NULL AND coalesce(array_length(pk_cols, 1), 0) <= 3 THEN
|
||||
SELECT string_agg(format('%I', c), ', ')
|
||||
INTO pk_cols_sql
|
||||
FROM unnest(pk_cols) AS c;
|
||||
|
||||
idx_name := left(format('idx_%s_vfy_%s_pk', rec.table_name, tcol), 50)
|
||||
|| '_' || substr(md5(rec.table_name || '_vfy_' || tcol || '_pk'), 1, 8);
|
||||
EXECUTE format(
|
||||
'CREATE INDEX IF NOT EXISTS %I ON billiards_dwd.%I (%I, %s)',
|
||||
idx_name, rec.table_name, tcol, pk_cols_sql
|
||||
);
|
||||
END IF;
|
||||
END IF;
|
||||
END LOOP;
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
rec RECORD;
|
||||
BEGIN
|
||||
FOR rec IN
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema IN ('billiards_ods', 'billiards_dwd')
|
||||
AND table_type = 'BASE TABLE'
|
||||
LOOP
|
||||
EXECUTE format('ANALYZE %I.%I', rec.table_schema, rec.table_name);
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
|
||||
Reference in New Issue
Block a user