Add one-skill
This commit is contained in:
@@ -0,0 +1,176 @@
|
||||
-- =====================================================================
|
||||
-- @SparkSqlName: PAIMONA-D-SQL-{表名}-CREATE
|
||||
-- @Version: 1.0
|
||||
-- @Desc: 建表模板(CREATE TABLE)
|
||||
-- @TargetTables: {新表名}
|
||||
-- @TargetDatabase: Paimon
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景1:基础表创建(非分区)
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS ${db_eda_env}.basic_table (
|
||||
-- 主键/标识字段
|
||||
id BIGINT COMMENT '主键ID',
|
||||
|
||||
-- 业务字段
|
||||
name STRING COMMENT '名称',
|
||||
category STRING COMMENT '类别',
|
||||
amount DECIMAL(18,2) COMMENT '金额',
|
||||
status STRING COMMENT '状态',
|
||||
|
||||
-- 时间字段
|
||||
created_at TIMESTAMP COMMENT '创建时间',
|
||||
updated_at TIMESTAMP COMMENT '更新时间',
|
||||
|
||||
-- 技术字段
|
||||
etl_time TIMESTAMP COMMENT '数据加工时间',
|
||||
etl_remark STRING COMMENT '备注信息'
|
||||
)
|
||||
COMMENT '基础业务表'
|
||||
STORED AS PARQUET; -- 存储格式
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景2:分区表创建(单分区字段)
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS ${db_eda_env}.partitioned_table (
|
||||
-- 主键/标识字段
|
||||
id BIGINT COMMENT '主键ID',
|
||||
|
||||
-- 业务字段
|
||||
user_id STRING COMMENT '用户ID',
|
||||
user_name STRING COMMENT '用户姓名',
|
||||
order_count BIGINT COMMENT '订单数',
|
||||
total_amount DECIMAL(18,2) COMMENT '总金额',
|
||||
|
||||
-- 维度字段
|
||||
department STRING COMMENT '部门',
|
||||
region STRING COMMENT '地区',
|
||||
|
||||
-- 技术字段
|
||||
etl_time TIMESTAMP COMMENT '数据加工时间'
|
||||
)
|
||||
COMMENT '按日分区的统计表'
|
||||
PARTITIONED BY (day_id STRING COMMENT '统计日期,格式yyyy-MM-dd')
|
||||
STORED AS PARQUET;
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景3:多分区字段表
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS ${db_eda_env}.multi_partition_table (
|
||||
id BIGINT COMMENT '主键ID',
|
||||
name STRING COMMENT '名称',
|
||||
amount DECIMAL(18,2) COMMENT '金额',
|
||||
etl_time TIMESTAMP COMMENT '数据加工时间'
|
||||
)
|
||||
COMMENT '多分区字段示例表'
|
||||
PARTITIONED BY (
|
||||
year_id STRING COMMENT '年份',
|
||||
month_id STRING COMMENT '月份'
|
||||
)
|
||||
STORED AS PARQUET;
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景4:带表属性配置
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS ${db_eda_env}.configured_table (
|
||||
id BIGINT COMMENT '主键ID',
|
||||
name STRING COMMENT '名称',
|
||||
amount DECIMAL(18,2) COMMENT '金额',
|
||||
etl_time TIMESTAMP COMMENT '数据加工时间'
|
||||
)
|
||||
COMMENT '带属性配置的表'
|
||||
PARTITIONED BY (day_id STRING)
|
||||
STORED AS PARQUET
|
||||
TBLPROPERTIES (
|
||||
'parquet.compression' = 'SNAPPY', -- 压缩格式
|
||||
'spark.sql.partitionOverwriteMode' = 'dynamic' -- 动态分区覆盖模式
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景5:Paimon 表创建(主键表)
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS ${db_eda_env}.paimon_pk_table (
|
||||
-- 主键字段(Paimon 主键表必须包含所有主键字段)
|
||||
id BIGINT COMMENT '主键ID',
|
||||
day_id STRING COMMENT '分区日期',
|
||||
|
||||
-- 业务字段
|
||||
name STRING COMMENT '名称',
|
||||
amount DECIMAL(18,2) COMMENT '金额',
|
||||
status STRING COMMENT '状态',
|
||||
|
||||
-- 技术字段
|
||||
etl_time TIMESTAMP COMMENT '数据加工时间'
|
||||
)
|
||||
COMMENT 'Paimon 主键表(支持 MERGE INTO)'
|
||||
PARTITIONED BY (day_id)
|
||||
TBLPROPERTIES (
|
||||
'primary-key' = 'id,day_id', -- 主键定义
|
||||
'bucket' = '4', -- 分桶数
|
||||
'changelog-producer' = 'input' -- 变更日志生产
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景6:临时表创建
|
||||
-- ============================================================================
|
||||
CREATE TEMPORARY TABLE tmp_processing_table (
|
||||
id BIGINT,
|
||||
name STRING,
|
||||
amount DECIMAL(18,2)
|
||||
);
|
||||
|
||||
-- 或使用 AS 创建临时表
|
||||
CREATE TEMPORARY TABLE tmp_source AS
|
||||
SELECT id, name, amount
|
||||
FROM source_table
|
||||
WHERE day_id = '${day_id}';
|
||||
|
||||
-- ============================================================================
|
||||
-- 字段类型速查
|
||||
-- ============================================================================
|
||||
/*
|
||||
| 类型 | 说明 | 适用场景 |
|
||||
|---------------|----------------|------------------------|
|
||||
| STRING | 字符串 | 名称、编码、描述 |
|
||||
| INT | 整数 | 数量、等级、标志 |
|
||||
| BIGINT | 大整数 | ID、计数、金额(整数) |
|
||||
| DECIMAL(p,s) | 定点数 | 金额、比例、精度数值 |
|
||||
| DOUBLE | 浮点数 | 科学计算(慎用于金额) |
|
||||
| BOOLEAN | 布尔 | 状态标志 |
|
||||
| DATE | 日期 | 日期字段 |
|
||||
| TIMESTAMP | 时间戳 | 时间字段 |
|
||||
| ARRAY<type> | 数组 | 多值字段 |
|
||||
| MAP<k,v> | 映射 | 属性字典 |
|
||||
*/
|
||||
|
||||
-- ============================================================================
|
||||
-- 建表规范说明
|
||||
-- ============================================================================
|
||||
/*
|
||||
1. 字段命名规范
|
||||
- 使用 snake_case:user_id, total_amount
|
||||
- 主键:id 或 {业务}_id
|
||||
- 技术字段:etl_time, etl_remark
|
||||
|
||||
2. COMMENT 必须添加
|
||||
- 每个字段必须有 COMMENT
|
||||
- 表必须有 COMMENT
|
||||
|
||||
3. 分区字段选择
|
||||
- 按时间分区:day_id, month_id
|
||||
- 分区粒度:日分区最常用
|
||||
|
||||
4. 存储格式
|
||||
- 推荐:PARQUET(列存储,压缩好)
|
||||
- 可选:ORC、AVRO
|
||||
|
||||
5. Paimon 表特性
|
||||
- primary-key:主键字段列表
|
||||
- bucket:分桶数(影响并发)
|
||||
- 支持 MERGE INTO 操作
|
||||
|
||||
6. 表属性配置
|
||||
- 压缩格式:SNAPPY(推荐)、GZIP、LZ4
|
||||
- 动态分区模式:dynamic(推荐)
|
||||
*/
|
||||
@@ -0,0 +1,148 @@
|
||||
-- =====================================================================
|
||||
-- @SparkSqlName: PAIMONA-D-SQL-{表名}-ETL
|
||||
-- @Version: 1.0
|
||||
-- @Desc: ETL 数据处理模板(临时表链式处理)
|
||||
-- @TargetTables: ${db_eda_env}.{目标表名}
|
||||
-- @SourceTables: {源表列表}
|
||||
-- @TargetDatabase: Paimon
|
||||
-- @SourceDatabase: Paimon
|
||||
-- @任务调度频度: {日/周/月}
|
||||
-- @修改记录:
|
||||
-- 版本号 更新时间 更新人员 更新内容
|
||||
-- V1.0 {日期} {人员} 创建脚本
|
||||
-- @数据处理步骤:
|
||||
-- Step01: {步骤描述}
|
||||
-- Step02: {步骤描述}
|
||||
-- Step03: {步骤描述}
|
||||
-- 参数说明
|
||||
-- 账期参数:
|
||||
-- ${day_id} 日账期,格式:20250101
|
||||
-- 环境变量:
|
||||
-- 变量名 测试环境值 生产环境值
|
||||
-- ${db_tmp_env} {库名} {库名}
|
||||
-- ${db_eda_env} {库名} {库名}
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- Step01: 基础清洗与过滤
|
||||
-- ============================================================================
|
||||
-- 说明:从源表读取数据,进行基础过滤和清洗
|
||||
-- 输入:{源表名}
|
||||
-- 输出:${db_tmp_env}.tmp_{表名}_01
|
||||
|
||||
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_01;
|
||||
CREATE TABLE ${db_tmp_env}.tmp_xxx_01 AS
|
||||
SELECT
|
||||
id,
|
||||
name,
|
||||
amount,
|
||||
status,
|
||||
created_at,
|
||||
day_id
|
||||
FROM source_table
|
||||
WHERE day_id = '${day_id}' -- 分区过滤(必须)
|
||||
AND status IN ('active', 'valid') -- 业务过滤
|
||||
AND amount > 0 -- 数据质量过滤
|
||||
AND id IS NOT NULL -- NULL过滤;
|
||||
|
||||
-- ============================================================================
|
||||
-- Step02: 多表关联与维度补全
|
||||
-- ============================================================================
|
||||
-- 说明:关联维度表,补全业务属性字段
|
||||
-- 输入:${db_tmp_env}.tmp_xxx_01, {维度表1}, {维度表2}
|
||||
-- 输出:${db_tmp_env}.tmp_xxx_02
|
||||
|
||||
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_02;
|
||||
CREATE TABLE ${db_tmp_env}.tmp_xxx_02 AS
|
||||
SELECT
|
||||
a.id,
|
||||
a.name,
|
||||
a.amount,
|
||||
a.status,
|
||||
b.category_name, -- 维度补全:类别名称
|
||||
c.department_name, -- 维度补全:部门名称
|
||||
a.created_at,
|
||||
a.day_id
|
||||
FROM ${db_tmp_env}.tmp_xxx_01 a
|
||||
LEFT JOIN dim_category b
|
||||
ON a.category_id = b.id
|
||||
AND b.day_id = '${day_id}' -- 维度表分区过滤
|
||||
LEFT JOIN dim_department c
|
||||
ON a.department_id = c.id
|
||||
AND c.day_id = '${day_id}'; -- 维度表分区过滤
|
||||
|
||||
-- ============================================================================
|
||||
-- Step03: 聚合计算与指标生成
|
||||
-- ============================================================================
|
||||
-- 说明:按业务维度聚合,计算统计指标
|
||||
-- 输入:${db_tmp_env}.tmp_xxx_02
|
||||
-- 输出:${db_tmp_env}.tmp_xxx_03
|
||||
|
||||
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_03;
|
||||
CREATE TABLE ${db_tmp_env}.tmp_xxx_03 AS
|
||||
SELECT
|
||||
day_id,
|
||||
category_name,
|
||||
department_name,
|
||||
COUNT(*) AS record_count, -- 记录数
|
||||
COUNT(DISTINCT id) AS unique_count, -- 唯一计数
|
||||
SUM(amount) AS total_amount, -- 总金额
|
||||
AVG(amount) AS avg_amount, -- 平均金额
|
||||
MAX(amount) AS max_amount, -- 最大金额
|
||||
MIN(amount) AS min_amount -- 最小金额
|
||||
FROM ${db_tmp_env}.tmp_xxx_02
|
||||
GROUP BY day_id, category_name, department_name;
|
||||
|
||||
-- ============================================================================
|
||||
-- Step04: 最终输出写入目标表
|
||||
-- ============================================================================
|
||||
-- 说明:补全目标表标准字段,写入结果表
|
||||
-- 输入:${db_tmp_env}.tmp_xxx_03
|
||||
-- 输出:${db_eda_env}.{目标表名}
|
||||
|
||||
INSERT OVERWRITE TABLE ${db_eda_env}.target_table
|
||||
PARTITION (day_id = '${day_id}')
|
||||
SELECT
|
||||
-- 业务字段
|
||||
category_name,
|
||||
department_name,
|
||||
record_count,
|
||||
unique_count,
|
||||
total_amount,
|
||||
avg_amount,
|
||||
max_amount,
|
||||
min_amount,
|
||||
|
||||
-- 技术字段
|
||||
current_timestamp() AS etl_time, -- 数据加工时间
|
||||
'${day_id}' AS stat_date -- 统计日期;
|
||||
|
||||
-- ============================================================================
|
||||
-- 关键规则说明
|
||||
-- ============================================================================
|
||||
/*
|
||||
1. 禁止使用 CTE (WITH 子句)
|
||||
- 每个步骤必须物化为临时表
|
||||
- 原因:避免内存溢出,便于调试和断点续跑
|
||||
|
||||
2. 先 DROP 再 CREATE
|
||||
- 每个临时表创建前必须先 DROP
|
||||
- 原因:防止表已存在导致失败
|
||||
|
||||
3. 分区过滤必须前置
|
||||
- 所有源表和维度表查询必须带 day_id 过滤
|
||||
- 原因:避免全表扫描,提升性能
|
||||
|
||||
4. JOIN 条件下推
|
||||
- 维度表关联时带上分区过滤条件
|
||||
- 原因:减少关联数据量
|
||||
|
||||
5. 临时表命名规范
|
||||
- 格式:tmp_{业务简称}_{步骤序号}
|
||||
- 示例:tmp_order_stats_01, tmp_order_stats_02
|
||||
|
||||
6. 目标表写入规范
|
||||
- 使用 INSERT OVERWRITE(覆盖写入)
|
||||
- 明确指定分区
|
||||
- 补全技术字段(etl_time 等)
|
||||
*/
|
||||
@@ -0,0 +1,131 @@
|
||||
-- =====================================================================
|
||||
-- @SparkSqlName: PAIMONA-D-SQL-{表名}-INSERT
|
||||
-- @Version: 1.0
|
||||
-- @Desc: 数据插入模板(INSERT OVERWRITE)
|
||||
-- @TargetTables: ${db_eda_env}.{目标表名}
|
||||
-- @SourceTables: {源表列表}
|
||||
-- @TargetDatabase: Paimon
|
||||
-- @SourceDatabase: Paimon
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景1:分区表覆盖写入
|
||||
-- ============================================================================
|
||||
-- 适用:每日/每周/每月增量写入分区表
|
||||
|
||||
INSERT OVERWRITE TABLE ${db_eda_env}.target_table
|
||||
PARTITION (day_id = '${day_id}')
|
||||
SELECT
|
||||
field1,
|
||||
field2,
|
||||
field3,
|
||||
current_timestamp() AS etl_time
|
||||
FROM source_table
|
||||
WHERE day_id = '${day_id}';
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景2:动态分区写入
|
||||
-- ============================================================================
|
||||
-- 适用:多分区字段,数据中包含分区值
|
||||
|
||||
INSERT OVERWRITE TABLE ${db_eda_env}.target_table
|
||||
PARTITION (day_id, region) -- 动态分区字段
|
||||
SELECT
|
||||
field1,
|
||||
field2,
|
||||
field3,
|
||||
day_id, -- 分区字段1(数据中包含)
|
||||
region, -- 分区字段2(数据中包含)
|
||||
current_timestamp() AS etl_time
|
||||
FROM source_table
|
||||
WHERE day_id BETWEEN '${start_day}' AND '${end_day}';
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景3:全表覆盖写入
|
||||
-- ============================================================================
|
||||
-- 适用:全量刷新、初始化数据
|
||||
|
||||
INSERT OVERWRITE TABLE ${db_eda_env}.target_table
|
||||
SELECT
|
||||
field1,
|
||||
field2,
|
||||
field3,
|
||||
current_timestamp() AS etl_time
|
||||
FROM source_table;
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景4:追加写入(慎用)
|
||||
-- ============================================================================
|
||||
-- 适用:日志表、流水表(无分区或允许重复)
|
||||
|
||||
INSERT INTO TABLE ${db_eda_env}.target_table
|
||||
SELECT
|
||||
field1,
|
||||
field2,
|
||||
field3,
|
||||
current_timestamp() AS etl_time
|
||||
FROM source_table
|
||||
WHERE day_id = '${day_id}';
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景5:从临时表写入目标表
|
||||
-- ============================================================================
|
||||
-- 适用:ETL 流程最后一步
|
||||
|
||||
INSERT OVERWRITE TABLE ${db_eda_env}.target_table
|
||||
PARTITION (day_id = '${day_id}')
|
||||
SELECT
|
||||
-- 业务字段(与目标表字段顺序一致)
|
||||
user_id,
|
||||
user_name,
|
||||
order_count,
|
||||
total_amount,
|
||||
|
||||
-- 技术字段
|
||||
current_timestamp() AS etl_time,
|
||||
'${day_id}' AS stat_date
|
||||
FROM ${db_tmp_env}.tmp_xxx_final;
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景6:MERGE INTO(更新插入)
|
||||
-- ============================================================================
|
||||
-- 适用:增量更新、修正历史数据
|
||||
|
||||
MERGE INTO ${db_eda_env}.target_table t
|
||||
USING ${db_tmp_env}.tmp_xxx_source s
|
||||
ON t.id = s.id AND t.day_id = s.day_id
|
||||
WHEN MATCHED THEN
|
||||
UPDATE SET
|
||||
t.name = s.name,
|
||||
t.amount = s.amount,
|
||||
t.etl_time = current_timestamp()
|
||||
WHEN NOT MATCHED THEN
|
||||
INSERT (id, day_id, name, amount, etl_time)
|
||||
VALUES (s.id, s.day_id, s.name, s.amount, current_timestamp());
|
||||
|
||||
-- ============================================================================
|
||||
-- 关键规则说明
|
||||
-- ============================================================================
|
||||
/*
|
||||
1. INSERT OVERWRITE vs INSERT INTO
|
||||
- INSERT OVERWRITE:覆盖写入(推荐)
|
||||
- INSERT INTO:追加写入(可能导致重复数据)
|
||||
|
||||
2. 分区表写入必须指定分区
|
||||
- 避免全表覆盖导致历史数据丢失
|
||||
- 格式:PARTITION (day_id = '${day_id}')
|
||||
|
||||
3. 字段顺序必须与目标表一致
|
||||
- 目标表字段顺序:业务字段 → 技术字段 → 分区字段
|
||||
- SELECT 字段顺序必须匹配
|
||||
|
||||
4. 技术字段补全
|
||||
- etl_time:数据写入时间
|
||||
- stat_date:统计日期(可选)
|
||||
- etl_remark:备注信息(可选)
|
||||
|
||||
5. MERGE INTO 注意事项
|
||||
- Spark 3.x+ 支持
|
||||
- 目标表必须支持事务(如 Paimon/Delta)
|
||||
- 关联字段必须唯一(避免多条匹配)
|
||||
*/
|
||||
@@ -0,0 +1,179 @@
|
||||
-- =====================================================================
|
||||
-- @SparkSqlName: PAIMONA-D-SQL-{表名}-PARTITION
|
||||
-- @Version: 1.0
|
||||
-- @Desc: 分区表操作模板
|
||||
-- @TargetTables: {分区表名}
|
||||
-- @TargetDatabase: Paimon
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- 分区表创建
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ${db_eda_env}.daily_partition_table (
|
||||
id BIGINT COMMENT '主键ID',
|
||||
user_id STRING COMMENT '用户ID',
|
||||
amount DECIMAL(18,2) COMMENT '金额',
|
||||
etl_time TIMESTAMP COMMENT '数据加工时间'
|
||||
)
|
||||
COMMENT '按日分区表'
|
||||
PARTITIONED BY (day_id STRING COMMENT '统计日期')
|
||||
STORED AS PARQUET;
|
||||
|
||||
-- ============================================================================
|
||||
-- 分区写入操作
|
||||
-- ============================================================================
|
||||
|
||||
-- 1. 静态分区写入(指定分区值)
|
||||
INSERT OVERWRITE TABLE ${db_eda_env}.daily_partition_table
|
||||
PARTITION (day_id = '2026-05-09')
|
||||
SELECT
|
||||
id,
|
||||
user_id,
|
||||
amount,
|
||||
current_timestamp() AS etl_time
|
||||
FROM source_table
|
||||
WHERE day_id = '${day_id}';
|
||||
|
||||
-- 2. 动态分区写入(数据中包含分区值)
|
||||
-- 需要先设置动态分区模式
|
||||
SET spark.sql.partitionOverwriteMode = dynamic;
|
||||
|
||||
INSERT OVERWRITE TABLE ${db_eda_env}.daily_partition_table
|
||||
PARTITION (day_id) -- 动态分区字段
|
||||
SELECT
|
||||
id,
|
||||
user_id,
|
||||
amount,
|
||||
current_timestamp() AS etl_time,
|
||||
day_id -- 数据中包含分区值
|
||||
FROM source_table
|
||||
WHERE day_id BETWEEN '2026-05-01' AND '2026-05-09';
|
||||
|
||||
-- ============================================================================
|
||||
-- 分区查询操作
|
||||
-- ============================================================================
|
||||
|
||||
-- 3. 单分区查询
|
||||
SELECT *
|
||||
FROM ${db_eda_env}.daily_partition_table
|
||||
WHERE day_id = '2026-05-09';
|
||||
|
||||
-- 4. 多分区查询
|
||||
SELECT *
|
||||
FROM ${db_eda_env}.daily_partition_table
|
||||
WHERE day_id IN ('2026-05-01', '2026-05-02', '2026-05-03');
|
||||
|
||||
-- 5. 分区范围查询
|
||||
SELECT *
|
||||
FROM ${db_eda_env}.daily_partition_table
|
||||
WHERE day_id >= '2026-05-01'
|
||||
AND day_id <= '2026-05-09';
|
||||
|
||||
-- 6. 最近 N 天分区查询(动态计算)
|
||||
SELECT *
|
||||
FROM ${db_eda_env}.daily_partition_table
|
||||
WHERE day_id >= date_format(date_sub(current_date(), 30), 'yyyy-MM-dd');
|
||||
|
||||
-- ============================================================================
|
||||
-- 分区管理操作
|
||||
-- ============================================================================
|
||||
|
||||
-- 7. 查看分区列表
|
||||
SHOW PARTITIONS ${db_eda_env}.daily_partition_table;
|
||||
|
||||
-- 8. 查看特定分区详情
|
||||
DESCRIBE EXTENDED ${db_eda_env}.daily_partition_table PARTITION (day_id = '2026-05-09');
|
||||
|
||||
-- 9. 添加分区(手动创建空分区,部分表类型支持)
|
||||
ALTER TABLE ${db_eda_env}.daily_partition_table
|
||||
ADD IF NOT EXISTS PARTITION (day_id = '2026-05-10');
|
||||
|
||||
-- 10. 删除分区(清理历史数据)
|
||||
ALTER TABLE ${db_eda_env}.daily_partition_table
|
||||
DROP IF EXISTS PARTITION (day_id = '2026-01-01');
|
||||
|
||||
-- ============================================================================
|
||||
-- 多分区字段操作
|
||||
-- ============================================================================
|
||||
|
||||
-- 11. 多分区字段表创建
|
||||
CREATE TABLE IF NOT EXISTS ${db_eda_env}.multi_partition_table (
|
||||
id BIGINT,
|
||||
name STRING,
|
||||
amount DECIMAL(18,2),
|
||||
etl_time TIMESTAMP
|
||||
)
|
||||
PARTITIONED BY (year_id STRING, month_id STRING)
|
||||
STORED AS PARQUET;
|
||||
|
||||
-- 12. 多分区字段写入
|
||||
INSERT OVERWRITE TABLE ${db_eda_env}.multi_partition_table
|
||||
PARTITION (year_id = '2026', month_id = '05')
|
||||
SELECT
|
||||
id,
|
||||
name,
|
||||
amount,
|
||||
current_timestamp() AS etl_time
|
||||
FROM source_table
|
||||
WHERE year_id = '2026' AND month_id = '05';
|
||||
|
||||
-- 13. 多分区字段动态写入
|
||||
SET spark.sql.partitionOverwriteMode = dynamic;
|
||||
|
||||
INSERT OVERWRITE TABLE ${db_eda_env}.multi_partition_table
|
||||
PARTITION (year_id, month_id)
|
||||
SELECT
|
||||
id,
|
||||
name,
|
||||
amount,
|
||||
current_timestamp() AS etl_time,
|
||||
year_id,
|
||||
month_id
|
||||
FROM source_table;
|
||||
|
||||
-- ============================================================================
|
||||
-- 分区数据清理
|
||||
-- ============================================================================
|
||||
|
||||
-- 14. 清理指定分区数据
|
||||
INSERT OVERWRITE TABLE ${db_eda_env}.daily_partition_table
|
||||
PARTITION (day_id = '2026-05-09')
|
||||
SELECT * FROM ${db_eda_env}.daily_partition_table
|
||||
WHERE day_id = '2026-05-09'
|
||||
AND status = 'valid'; -- 只保留有效数据
|
||||
|
||||
-- 15. 清理 N 天前分区(批量)
|
||||
-- 使用脚本或程序循环执行
|
||||
-- ALTER TABLE xxx DROP PARTITION (day_id = '历史分区')
|
||||
|
||||
-- ============================================================================
|
||||
-- 分区最佳实践
|
||||
-- ============================================================================
|
||||
/*
|
||||
1. 分区字段选择原则
|
||||
- 查询高频过滤字段
|
||||
- 数据量分布均匀的字段
|
||||
- 时间字段最常用(day_id, month_id)
|
||||
|
||||
2. 分区粒度选择
|
||||
- 日增量数据 → day_id 分区
|
||||
- 月增量数据 → month_id 分区
|
||||
- 大数据量 → 可细分到 hour_id
|
||||
|
||||
3. 分区数量控制
|
||||
- 单表分区数建议 < 10000
|
||||
- 过多分区影响元数据性能
|
||||
|
||||
4. 查询必须带分区过滤
|
||||
- 避免:SELECT * FROM table(全表扫描)
|
||||
- 推荐:SELECT * FROM table WHERE day_id = '${day_id}'
|
||||
|
||||
5. 动态分区写入设置
|
||||
- SET spark.sql.partitionOverwriteMode = dynamic;
|
||||
- 避免误覆盖其他分区
|
||||
|
||||
6. 分区数据清理
|
||||
- 定期清理历史分区(如保留近90天)
|
||||
- 使用 ALTER TABLE DROP PARTITION
|
||||
*/
|
||||
@@ -0,0 +1,160 @@
|
||||
-- =====================================================================
|
||||
-- @SparkSqlName: PAIMONA-D-SQL-{表名}-QUERY
|
||||
-- @Version: 1.0
|
||||
-- @Desc: 标准 SELECT 查询模板
|
||||
-- @TargetTables: 无(查询输出)
|
||||
-- @SourceTables: {源表列表}
|
||||
-- @TargetDatabase: Paimon
|
||||
-- @SourceDatabase: Paimon
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- 基础查询示例
|
||||
-- ============================================================================
|
||||
|
||||
-- 1. 单表查询
|
||||
SELECT
|
||||
id,
|
||||
name,
|
||||
amount,
|
||||
created_at
|
||||
FROM source_table
|
||||
WHERE day_id = '${day_id}' -- 分区过滤
|
||||
AND status = 'active' -- 业务过滤
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1000;
|
||||
|
||||
-- ============================================================================
|
||||
-- JOIN 查询示例
|
||||
-- ============================================================================
|
||||
|
||||
-- 2. 两表 JOIN
|
||||
SELECT
|
||||
a.id,
|
||||
a.name,
|
||||
b.category_name
|
||||
FROM main_table a
|
||||
JOIN dim_table b ON a.category_id = b.id
|
||||
WHERE a.day_id = '${day_id}'
|
||||
AND b.is_active = true;
|
||||
|
||||
-- 3. 多表 JOIN(带别名)
|
||||
SELECT
|
||||
o.order_id,
|
||||
u.user_name,
|
||||
p.product_name,
|
||||
oi.quantity,
|
||||
oi.unit_price
|
||||
FROM orders o
|
||||
JOIN users u ON o.user_id = u.id
|
||||
JOIN order_items oi ON o.order_id = oi.order_id
|
||||
JOIN products p ON oi.product_id = p.id
|
||||
WHERE o.day_id = '${day_id}'
|
||||
AND o.status IN ('completed', 'shipped');
|
||||
|
||||
-- ============================================================================
|
||||
-- 聚合查询示例
|
||||
-- ============================================================================
|
||||
|
||||
-- 4. GROUP BY 聚合
|
||||
SELECT
|
||||
department,
|
||||
COUNT(*) AS employee_count,
|
||||
SUM(salary) AS total_salary,
|
||||
AVG(salary) AS avg_salary,
|
||||
MAX(salary) AS max_salary,
|
||||
MIN(salary) AS min_salary
|
||||
FROM employees
|
||||
WHERE day_id = '${day_id}'
|
||||
GROUP BY department
|
||||
HAVING COUNT(*) >= 5
|
||||
ORDER BY total_salary DESC;
|
||||
|
||||
-- 5. 多字段分组 + 去重计数
|
||||
SELECT
|
||||
date,
|
||||
region,
|
||||
COUNT(*) AS order_count,
|
||||
COUNT(DISTINCT user_id) AS unique_users,
|
||||
SUM(amount) AS total_amount
|
||||
FROM orders
|
||||
WHERE day_id = '${day_id}'
|
||||
GROUP BY date, region;
|
||||
|
||||
-- ============================================================================
|
||||
-- 窗口函数示例
|
||||
-- ============================================================================
|
||||
|
||||
-- 6. ROW_NUMBER(分组取Top N)
|
||||
SELECT *
|
||||
FROM (
|
||||
SELECT
|
||||
department,
|
||||
name,
|
||||
salary,
|
||||
ROW_NUMBER() OVER (PARTITION BY department ORDER BY salary DESC) AS rn
|
||||
FROM employees
|
||||
WHERE day_id = '${day_id}'
|
||||
) t
|
||||
WHERE rn <= 3; -- 每个部门薪资前3名
|
||||
|
||||
-- 7. 累计聚合
|
||||
SELECT
|
||||
date,
|
||||
amount,
|
||||
SUM(amount) OVER (ORDER BY date) AS cumulative_amount,
|
||||
AVG(amount) OVER (
|
||||
ORDER BY date
|
||||
ROWS BETWEEN 6 PRECEDING AND CURRENT ROW
|
||||
) AS moving_avg_7d
|
||||
FROM daily_sales
|
||||
WHERE day_id = '${day_id}';
|
||||
|
||||
-- 8. LAG/LEAD(环比计算)
|
||||
SELECT
|
||||
date,
|
||||
amount,
|
||||
LAG(amount, 1) OVER (ORDER BY date) AS prev_amount,
|
||||
amount - LAG(amount, 1) OVER (ORDER BY date) AS daily_change,
|
||||
ROUND((amount - LAG(amount, 1) OVER (ORDER BY date))
|
||||
/ LAG(amount, 1) OVER (ORDER BY date) * 100, 2) AS growth_rate_pct
|
||||
FROM daily_sales
|
||||
WHERE day_id = '${day_id}';
|
||||
|
||||
-- ============================================================================
|
||||
-- 子查询示例
|
||||
-- ============================================================================
|
||||
|
||||
-- 9. IN 子查询
|
||||
SELECT *
|
||||
FROM orders
|
||||
WHERE user_id IN (
|
||||
SELECT id FROM users WHERE vip_level >= 3
|
||||
)
|
||||
AND day_id = '${day_id}';
|
||||
|
||||
-- 10. EXISTS 子查询
|
||||
SELECT *
|
||||
FROM products p
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM inventory i
|
||||
WHERE i.product_id = p.id
|
||||
AND i.quantity > 0
|
||||
)
|
||||
AND p.day_id = '${day_id}';
|
||||
|
||||
-- ============================================================================
|
||||
-- 条件聚合示例
|
||||
-- ============================================================================
|
||||
|
||||
-- 11. CASE WHEN + 聚合
|
||||
SELECT
|
||||
date,
|
||||
COUNT(*) AS total_orders,
|
||||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed_count,
|
||||
SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) AS cancelled_count,
|
||||
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending_count,
|
||||
SUM(CASE WHEN amount > 1000 THEN amount ELSE 0 END) AS high_value_amount
|
||||
FROM orders
|
||||
WHERE day_id = '${day_id}'
|
||||
GROUP BY date;
|
||||
Reference in New Issue
Block a user