Add one-skill

This commit is contained in:
Xin Wang
2026-05-13 11:03:00 +08:00
parent a4c8b29176
commit f9e36ef92d
34 changed files with 7656 additions and 0 deletions

View File

@@ -0,0 +1,211 @@
-- =====================================================================
-- @Name: KUDU-D-SQL-{表名}-CREATE
-- @Version: 1.0
-- @Desc: Kudu (via Impala) 建表模板
-- @TargetDatabase: Apache Kudu (via Impala)
-- @说明: Kudu 通过 Impala 访问,使用 Impala DDL 操作 Kudu 表
-- =====================================================================
-- ============================================================================
-- 场景1基础表创建Hash 分区)
-- ============================================================================
-- 适用:按主键 Hash 分布数据,写入和点查性能好
CREATE TABLE IF NOT EXISTS db_name.kudu_basic (
-- 主键列Kudu 表必须有主键)
id BIGINT NOT NULL COMMENT '主键ID',
-- 业务字段
name STRING COMMENT '名称',
category STRING COMMENT '类别',
amount DECIMAL(18,2) COMMENT '金额',
status STRING COMMENT '状态',
created_at TIMESTAMP COMMENT '创建时间',
updated_at TIMESTAMP COMMENT '更新时间'
)
PRIMARY KEY (id)
PARTITION BY HASH(id) PARTITIONS 8
STORED AS KUDU
TBLPROPERTIES (
'kudu.num_tablet_replicas' = '3'
);
-- ============================================================================
-- 场景2Hash + Range 组合分区
-- ============================================================================
-- 适用:按时间范围 + Hash 组合,兼顾范围查询和写入性能
CREATE TABLE IF NOT EXISTS db_name.kudu_range_hash (
-- 主键列(必须包含分区列)
id BIGINT NOT NULL COMMENT '主键ID',
stat_date STRING NOT NULL COMMENT '统计日期 yyyy-MM-dd',
-- 业务字段
department STRING COMMENT '部门',
metric_name STRING COMMENT '指标名称',
metric_value DECIMAL(18,2) COMMENT '指标值',
etl_time TIMESTAMP COMMENT '加工时间'
)
PRIMARY KEY (id, stat_date)
PARTITION BY
HASH(id) PARTITIONS 4,
RANGE(stat_date) (
PARTITION '2026-01-01' <= VALUES < '2026-02-01',
PARTITION '2026-02-01' <= VALUES < '2026-03-01',
PARTITION '2026-03-01' <= VALUES < '2026-04-01',
PARTITION '2026-04-01' <= VALUES < '2026-05-01',
PARTITION '2026-05-01' <= VALUES < '2026-06-01'
)
STORED AS KUDU
TBLPROPERTIES (
'kudu.num_tablet_replicas' = '3',
'kudu.compression' = 'LZ4'
);
-- ============================================================================
-- 场景3多列主键
-- ============================================================================
CREATE TABLE IF NOT EXISTS db_name.kudu_composite_pk (
user_id BIGINT NOT NULL COMMENT '用户ID',
order_date STRING NOT NULL COMMENT '订单日期',
order_seq INT NOT NULL COMMENT '当日订单序号',
user_name STRING COMMENT '用户姓名',
product_name STRING COMMENT '商品名称',
quantity INT COMMENT '数量',
total_amount DECIMAL(18,2) COMMENT '总金额',
status STRING COMMENT '状态',
create_time TIMESTAMP COMMENT '创建时间'
)
PRIMARY KEY (user_id, order_date, order_seq)
PARTITION BY
HASH(user_id) PARTITIONS 8,
RANGE(order_date) (
PARTITION '2026-01-01' <= VALUES < '2026-02-01',
PARTITION '2026-02-01' <= VALUES < '2026-03-01',
PARTITION '2026-03-01' <= VALUES < '2026-04-01'
)
STORED AS KUDU
TBLPROPERTIES (
'kudu.num_tablet_replicas' = '3'
);
-- ============================================================================
-- 场景4纯 Range 分区
-- ============================================================================
-- 适用:按时间顺序写入,范围查询多
CREATE TABLE IF NOT EXISTS db_name.kudu_range_only (
id BIGINT NOT NULL COMMENT '主键ID',
stat_date STRING NOT NULL COMMENT '统计日期',
metric_value DECIMAL(18,2) COMMENT '指标值',
dimension STRING COMMENT '维度',
etl_time TIMESTAMP COMMENT '加工时间'
)
PRIMARY KEY (id, stat_date)
PARTITION BY RANGE(stat_date) (
PARTITION '2026-01-01' <= VALUES < '2026-04-01',
PARTITION '2026-04-01' <= VALUES < '2026-07-01',
PARTITION '2026-07-01' <= VALUES < '2026-10-01',
PARTITION '2026-10-01' <= VALUES < '2027-01-01'
)
STORED AS KUDU
TBLPROPERTIES (
'kudu.num_tablet_replicas' = '3'
);
-- ============================================================================
-- 场景5外部表映射已有 Kudu 表
-- ============================================================================
CREATE EXTERNAL TABLE IF NOT EXISTS db_name.kudu_external
STORED AS KUDU
TBLPROPERTIES (
'kudu.master_addresses' = 'kudu-master-1:7051,kudu-master-2:7051,kudu-master-3:7051',
'kudu.table_name' = 'impala.db_name.existing_table'
);
-- ============================================================================
-- 场景6带压缩和副本配置
-- ============================================================================
CREATE TABLE IF NOT EXISTS db_name.kudu_with_props (
id BIGINT NOT NULL COMMENT '主键ID',
data_date STRING NOT NULL COMMENT '数据日期',
content STRING COMMENT '内容',
value DOUBLE COMMENT '数值'
)
PRIMARY KEY (id, data_date)
PARTITION BY
HASH(id) PARTITIONS 8,
RANGE(data_date) (
PARTITION '2026-01-01' <= VALUES < '2026-02-01',
PARTITION '2026-02-01' <= VALUES < '2026-03-01'
)
STORED AS KUDU
TBLPROPERTIES (
'kudu.num_tablet_replicas' = '3',
'kudu.compression' = 'LZ4', -- 压缩算法
'kudu.encryption' = 'false' -- 加密
);
-- ============================================================================
-- 字段类型速查Kudu 支持的类型)
-- ============================================================================
/*
| 类型 | 说明 | 适用场景 |
|---------------|----------------|------------------------|
| BOOLEAN | 布尔 | 状态标志 |
| TINYINT | 1字节整数 | 小范围枚举 |
| SMALLINT | 2字节整数 | 小范围数值 |
| INT | 4字节整数 | 数量、等级 |
| BIGINT | 8字节整数 | ID、计数 |
| FLOAT | 4字节浮点 | 近似计算 |
| DOUBLE | 8字节浮点 | 科学计算 |
| DECIMAL(p,s) | 定点数 | 金额、精确数值 |
| STRING | 变长字符串 | 名称、描述 |
| VARCHAR(n) | 变长字符串 | 限定长度字符串 |
| CHAR(n) | 定长字符串 | 固定长度编码 |
| TIMESTAMP | 时间戳 | 时间字段(微秒精度) |
| DATE | 日期 | 日期字段 |
| BINARY | 二进制 | 二进制数据 |
注意Kudu 不支持 ARRAY, MAP, STRUCT 等复杂类型
*/
-- ============================================================================
-- 建表规范说明
-- ============================================================================
/*
1. 主键约束Kudu 特有)
- 每张 Kudu 表必须有 PRIMARY KEY
- 主键列不能为 NULL必须 NOT NULL
- 主键值不可 UPDATE只能删除后重新插入
- 主键列必须包含在分区列中
2. 分区策略
- Hash 分区:均匀分布,适合写入和点查
- Range 分区:按范围查询,适合时间序列
- Hash + Range 组合:兼顾两者优势(推荐)
- 分区数 = tablet 数量,影响并行度
3. 分区设计建议
- Hash 分区数:建议 4 的倍数,参考数据量
- Range 分区:按时间维度,定期添加新分区
- 单个 tablet 建议 1GB~10GB
4. 副本数
- 生产环境建议 3 副本(默认)
- Raft 协议保证一致性
5. 压缩
- 推荐 LZ4速度和压缩比平衡
- 可选SNAPPY, ZLIB, LZ4
6. 与 Hive/Spark 表的区别
- Kudu 表支持 UPDATE 和 DELETE
- Kudu 表不支持 INSERT OVERWRITE
- Kudu 表不支持复杂类型ARRAY, MAP, STRUCT
- Kudu 表主键有约束Hive/Spark 无约束
*/

View File

@@ -0,0 +1,146 @@
-- =====================================================================
-- @Name: KUDU-D-SQL-{表名}-ETL
-- @Version: 2.0
-- @Desc: Kudu (via Impala) ETL 数据处理模板(临时表链式处理)
-- @TargetDatabase: Apache Kudu (via Impala)
-- @说明: 统一规范:禁止 CTE每步物化为临时表先 DROP 再 CREATE
-- 最后一步用 UPSERT INTO 写入 Kudu 目标表
-- =====================================================================
-- ============================================================================
-- Step01: 基础清洗与过滤
-- ============================================================================
-- 说明:从源表读取数据,进行基础过滤和清洗
-- 输入:{源表名}
-- 输出:${db_tmp_env}.tmp_xxx_01
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_01;
CREATE TABLE ${db_tmp_env}.tmp_xxx_01 AS
SELECT
order_id,
user_id,
dept_id,
product_id,
quantity,
amount,
status,
stat_date
FROM db_name.source_table
WHERE stat_date = '${day_id}'
AND status IN ('completed', 'shipped') -- 业务过滤
AND amount > 0 -- 数据质量过滤
AND user_id IS NOT NULL; -- NULL过滤
-- ============================================================================
-- Step02: 多表关联与维度补全
-- ============================================================================
-- 说明:关联维度表,补全业务属性字段
-- 输入:${db_tmp_env}.tmp_xxx_01, dim_department, dim_product
-- 输出:${db_tmp_env}.tmp_xxx_02
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_02;
CREATE TABLE ${db_tmp_env}.tmp_xxx_02 AS
SELECT
a.order_id,
a.user_id,
a.amount,
a.quantity,
b.dept_name, -- 维度补全:部门名称
c.category_name, -- 维度补全:类别名称
a.stat_date
FROM ${db_tmp_env}.tmp_xxx_01 a
LEFT JOIN db_name.dim_department b
ON a.dept_id = b.dept_id
LEFT JOIN db_name.dim_product c
ON a.product_id = c.product_id;
-- ============================================================================
-- Step03: 聚合计算与指标生成
-- ============================================================================
-- 说明:按业务维度聚合,计算统计指标
-- 输入:${db_tmp_env}.tmp_xxx_02
-- 输出:${db_tmp_env}.tmp_xxx_03
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_03;
CREATE TABLE ${db_tmp_env}.tmp_xxx_03 AS
SELECT
stat_date,
dept_name,
category_name,
COUNT(*) AS record_count, -- 记录数
COUNT(DISTINCT user_id) AS unique_users, -- 去重用户数
SUM(amount) AS total_amount, -- 总金额
SUM(quantity) AS total_quantity, -- 总数量
AVG(amount) AS avg_amount, -- 平均金额
MAX(amount) AS max_amount -- 最大金额
FROM ${db_tmp_env}.tmp_xxx_02
GROUP BY stat_date, dept_name, category_name;
-- ============================================================================
-- Step04: 最终输出写入 Kudu 目标表
-- ============================================================================
-- 说明:使用 UPSERT 写入 Kudu 目标表
-- 输入:${db_tmp_env}.tmp_xxx_03
-- 输出Kudu 目标表
-- 方式1UPSERT推荐主键存在则更新不存在则插入
UPSERT INTO ${db_eda_env}.target_table
SELECT
-- 主键字段Kudu 表必须有主键)
dept_name,
category_name,
stat_date,
-- 指标字段
record_count,
unique_users,
total_amount,
total_quantity,
avg_amount,
max_amount,
-- 技术字段
NOW() AS etl_time -- 数据加工时间
FROM ${db_tmp_env}.tmp_xxx_03;
-- 方式2需要全量刷新时先删后插
-- DELETE FROM ${db_eda_env}.target_table WHERE stat_date = '${day_id}';
-- INSERT INTO ${db_eda_env}.target_table
-- SELECT ... FROM ${db_tmp_env}.tmp_xxx_03;
-- ============================================================================
-- 关键规则说明
-- ============================================================================
/*
1. 禁止使用 CTE (WITH 子句)
- 每个步骤必须物化为临时表
- 原因:便于调试、断点续跑、统一编码规范
2. 先 DROP 再 CREATE
- 每个临时表创建前必须先 DROP TABLE IF EXISTS
- 原因:防止表已存在导致失败
3. Kudu 写入方式
- 推荐 UPSERT INTOKudu 核心优势)
- 主键存在 → 更新(整行替换)
- 主键不存在 → 插入新行
- 需要全量刷新 → 先 DELETE 再 INSERT
4. Kudu 表约束
- 不支持 INSERT OVERWRITE用 UPSERT 或 DELETE + INSERT 替代)
- 必须有 PRIMARY KEY
- 主键列不能为 NULL
- 支持 UPDATE 和 DELETE
5. 过滤条件前置
- 所有过滤在最早阶段应用
- 减少中间数据量
6. 临时表命名规范
- 格式tmp_{业务简称}_{步骤序号}
7. Kudu 特有注意事项
- CONCAT 只接受 2 个参数,多参数用 CONCAT_WS
- 不支持 collect_list/collect_set用 GROUP_CONCAT 替代
- 近似去重用 NDV() 函数
*/

View File

@@ -0,0 +1,160 @@
-- =====================================================================
-- @Name: KUDU-D-SQL-{表名}-INSERT
-- @Version: 1.0
-- @Desc: Kudu (via Impala) 数据插入模板
-- @TargetDatabase: Apache Kudu (via Impala)
-- @说明: Kudu 表不支持 INSERT OVERWRITE支持 INSERT INTO 和 UPSERT
-- =====================================================================
-- ============================================================================
-- 场景1INSERT INTO追加写入
-- ============================================================================
-- 适用:向 Kudu 表追加新数据
INSERT INTO db_name.kudu_table
SELECT
id,
stat_date,
name,
department,
amount,
current_timestamp() AS etl_time
FROM db_name.source_table
WHERE stat_date = '${day_id}';
-- ============================================================================
-- 场景2UPSERT INTO更新插入Kudu 特有)
-- ============================================================================
-- 适用:如果主键存在则更新,不存在则插入
-- 这是 Kudu 的核心优势,其他 Hive/Spark 表不支持
-- 基础 UPSERT
UPSERT INTO db_name.kudu_table
SELECT
id,
stat_date,
name,
department,
amount,
current_timestamp() AS etl_time
FROM db_name.staging_table
WHERE stat_date = '${day_id}';
-- 聚合后 UPSERT增量更新指标表
UPSERT INTO db_name.kudu_metrics
SELECT
department,
'${day_id}' AS stat_date,
COUNT(*) AS order_count,
SUM(amount) AS total_amount,
current_timestamp() AS etl_time
FROM db_name.incremental_orders
WHERE stat_date = '${day_id}'
GROUP BY department;
-- ============================================================================
-- 场景3UPDATEKudu 表特有)
-- ============================================================================
-- 适用:修改已有数据
-- 注意:主键列不能被 UPDATE
-- 单条更新
UPDATE db_name.kudu_table
SET status = 'processed',
updated_at = current_timestamp()
WHERE id = 12345;
-- 批量条件更新
UPDATE db_name.kudu_table
SET status = 'expired',
updated_at = current_timestamp()
WHERE stat_date < '2026-01-01'
AND status = 'active';
-- 关联更新(用子查询)
UPDATE db_name.kudu_table t
SET t.department = d.new_dept_name
FROM db_name.dept_mapping d
WHERE t.department = d.old_dept_name;
-- ============================================================================
-- 场景4DELETEKudu 表特有)
-- ============================================================================
-- 适用:删除数据
-- 注意Kudu 的 DELETE 比 Hive/Spark 方便得多
-- 条件删除
DELETE FROM db_name.kudu_table
WHERE stat_date < '2026-01-01';
-- 按主键删除
DELETE FROM db_name.kudu_table
WHERE id IN (1001, 1002, 1003);
-- 关联删除(用子查询)
DELETE FROM db_name.kudu_table
WHERE user_id IN (
SELECT user_id FROM db_name.blacklist
);
-- ============================================================================
-- 场景5从查询结果写入
-- ============================================================================
-- 简单 ETL清洗后写入
INSERT INTO db_name.kudu_target
SELECT
id,
'${day_id}' AS stat_date,
name,
COALESCE(department, '未知') AS department,
amount,
current_timestamp() AS etl_time
FROM db_name.raw_data
WHERE stat_date = '${day_id}'
AND id IS NOT NULL
AND amount > 0;
-- ============================================================================
-- 场景6批量 VALUES 写入
-- ============================================================================
INSERT INTO db_name.kudu_table (id, stat_date, name, amount)
VALUES
(1, '2026-05-01', '测试1', 100.00),
(2, '2026-05-01', '测试2', 200.00),
(3, '2026-05-01', '测试3', 300.00);
-- ============================================================================
-- 关键规则说明
-- ============================================================================
/*
1. Kudu 表与 Hive/Spark 表的核心区别
- 支持 INSERT INTO
- 支持 INSERT OVERWRITE不支持
- 支持 UPSERTKudu 独有,核心能力)
- 支持 UPDATEKudu 独有)
- 支持 DELETEKudu 独有)
2. UPSERT 是 Kudu 的核心优势
- 主键存在 → 更新(整行替换)
- 主键不存在 → 插入新行
- 适用于:增量更新、数据修正、指标回填
3. INSERT INTO 注意事项
- 如果主键冲突会报错(不会自动去重)
- 需要确保写入数据的主键不重复,或使用 UPSERT
4. UPDATE 限制
- 主键列不能被 UPDATE
- WHERE 条件建议包含主键或分区列(性能)
5. DELETE 建议
- 删除大量数据时按分区范围删除
- 定期清理历史数据
6. 性能建议
- 批量写入优于逐条写入
- UPSERT 比 DELETE + INSERT 更高效
- 利用主键做点查,避免全表扫描
*/

View File

@@ -0,0 +1,179 @@
-- =====================================================================
-- @Name: KUDU-D-SQL-{表名}-QUERY
-- @Version: 1.0
-- @Desc: Kudu (via Impala) 查询模板
-- @TargetDatabase: Apache Kudu (via Impala)
-- =====================================================================
-- ============================================================================
-- 1. 单表查询
-- ============================================================================
SELECT
id,
name,
amount,
created_at
FROM db_name.kudu_table
WHERE stat_date = '${day_id}'
AND status = 'active'
ORDER BY created_at DESC
LIMIT 1000;
-- ============================================================================
-- 2. JOIN 查询
-- ============================================================================
-- 两表 JOINKudu 表 JOIN 非 Kudu 表也支持)
SELECT
k.id,
k.name,
k.amount,
d.dept_name
FROM db_name.kudu_table k
JOIN db_name.dim_department d ON k.dept_id = d.dept_id
WHERE k.stat_date = '${day_id}';
-- 多表 JOIN
SELECT
k.id,
k.user_name,
p.product_name,
k.quantity,
k.total_amount
FROM db_name.kudu_orders k
JOIN db_name.dim_users u ON k.user_id = u.user_id
JOIN db_name.dim_products p ON k.product_id = p.product_id
WHERE k.stat_date BETWEEN '${start_date}' AND '${end_date}';
-- ============================================================================
-- 3. 聚合查询
-- ============================================================================
SELECT
department,
COUNT(*) AS record_count,
SUM(amount) AS total_amount,
AVG(amount) AS avg_amount,
MAX(amount) AS max_amount,
MIN(amount) AS min_amount
FROM db_name.kudu_table
WHERE stat_date = '${day_id}'
GROUP BY department
HAVING COUNT(*) >= 5
ORDER BY total_amount DESC;
-- 多字段分组 + 去重计数
SELECT
stat_date,
region,
COUNT(*) AS order_count,
COUNT(DISTINCT user_id) AS unique_users,
SUM(amount) AS total_amount
FROM db_name.kudu_table
WHERE stat_date BETWEEN '${start_date}' AND '${end_date}'
GROUP BY stat_date, region;
-- ============================================================================
-- 4. 窗口函数
-- ============================================================================
-- ROW_NUMBER分组取Top N
SELECT *
FROM (
SELECT
department,
user_name,
amount,
ROW_NUMBER() OVER (PARTITION BY department ORDER BY amount DESC) AS rn
FROM db_name.kudu_table
WHERE stat_date = '${day_id}'
) t
WHERE rn <= 3;
-- 累计聚合
SELECT
stat_date,
daily_amount,
SUM(daily_amount) OVER (ORDER BY stat_date) AS cumulative_amount
FROM (
SELECT stat_date, SUM(amount) AS daily_amount
FROM db_name.kudu_table
GROUP BY stat_date
) t;
-- LAG/LEAD环比
SELECT
stat_date,
daily_amount,
LAG(daily_amount, 1) OVER (ORDER BY stat_date) AS prev_amount,
daily_amount - LAG(daily_amount, 1) OVER (ORDER BY stat_date) AS daily_change
FROM (
SELECT stat_date, SUM(amount) AS daily_amount
FROM db_name.kudu_table
GROUP BY stat_date
) t;
-- ============================================================================
-- 5. 子查询
-- ============================================================================
-- IN 子查询
SELECT *
FROM db_name.kudu_table
WHERE user_id IN (
SELECT user_id FROM db_name.vip_users WHERE vip_level >= 3
)
AND stat_date = '${day_id}';
-- EXISTS 子查询
SELECT *
FROM db_name.kudu_products p
WHERE EXISTS (
SELECT 1 FROM db_name.kudu_inventory i
WHERE i.product_id = p.product_id
AND i.quantity > 0
);
-- ============================================================================
-- 6. 条件聚合CASE WHEN + 聚合)
-- ============================================================================
SELECT
stat_date,
COUNT(*) AS total_count,
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed_count,
SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) AS cancelled_count,
SUM(CASE WHEN amount > 1000 THEN amount ELSE 0 END) AS high_value_amount
FROM db_name.kudu_table
WHERE stat_date = '${day_id}'
GROUP BY stat_date;
-- ============================================================================
-- 7. LIMIT / OFFSET分页
-- ============================================================================
SELECT id, name, amount
FROM db_name.kudu_table
WHERE stat_date = '${day_id}'
ORDER BY id
LIMIT 20 OFFSET 0;
-- ============================================================================
-- 8. Kudu 特有:通过主键高效点查
-- ============================================================================
-- Kudu 主键查询可跳过扫描,直接定位 tablet
-- 单主键点查
SELECT * FROM db_name.kudu_table
WHERE id = 12345;
-- 复合主键点查
SELECT * FROM db_name.kudu_composite_pk
WHERE user_id = 1001
AND order_date = '2026-05-01'
AND order_seq = 1;
-- 主键 IN 查询
SELECT * FROM db_name.kudu_table
WHERE id IN (1001, 1002, 1003, 1004, 1005);