Add one-skill
This commit is contained in:
@@ -0,0 +1,211 @@
|
||||
-- =====================================================================
|
||||
-- @Name: KUDU-D-SQL-{表名}-CREATE
|
||||
-- @Version: 1.0
|
||||
-- @Desc: Kudu (via Impala) 建表模板
|
||||
-- @TargetDatabase: Apache Kudu (via Impala)
|
||||
-- @说明: Kudu 通过 Impala 访问,使用 Impala DDL 操作 Kudu 表
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景1:基础表创建(Hash 分区)
|
||||
-- ============================================================================
|
||||
-- 适用:按主键 Hash 分布数据,写入和点查性能好
|
||||
|
||||
CREATE TABLE IF NOT EXISTS db_name.kudu_basic (
|
||||
-- 主键列(Kudu 表必须有主键)
|
||||
id BIGINT NOT NULL COMMENT '主键ID',
|
||||
|
||||
-- 业务字段
|
||||
name STRING COMMENT '名称',
|
||||
category STRING COMMENT '类别',
|
||||
amount DECIMAL(18,2) COMMENT '金额',
|
||||
status STRING COMMENT '状态',
|
||||
created_at TIMESTAMP COMMENT '创建时间',
|
||||
updated_at TIMESTAMP COMMENT '更新时间'
|
||||
)
|
||||
PRIMARY KEY (id)
|
||||
PARTITION BY HASH(id) PARTITIONS 8
|
||||
STORED AS KUDU
|
||||
TBLPROPERTIES (
|
||||
'kudu.num_tablet_replicas' = '3'
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景2:Hash + Range 组合分区
|
||||
-- ============================================================================
|
||||
-- 适用:按时间范围 + Hash 组合,兼顾范围查询和写入性能
|
||||
|
||||
CREATE TABLE IF NOT EXISTS db_name.kudu_range_hash (
|
||||
-- 主键列(必须包含分区列)
|
||||
id BIGINT NOT NULL COMMENT '主键ID',
|
||||
stat_date STRING NOT NULL COMMENT '统计日期 yyyy-MM-dd',
|
||||
|
||||
-- 业务字段
|
||||
department STRING COMMENT '部门',
|
||||
metric_name STRING COMMENT '指标名称',
|
||||
metric_value DECIMAL(18,2) COMMENT '指标值',
|
||||
etl_time TIMESTAMP COMMENT '加工时间'
|
||||
)
|
||||
PRIMARY KEY (id, stat_date)
|
||||
PARTITION BY
|
||||
HASH(id) PARTITIONS 4,
|
||||
RANGE(stat_date) (
|
||||
PARTITION '2026-01-01' <= VALUES < '2026-02-01',
|
||||
PARTITION '2026-02-01' <= VALUES < '2026-03-01',
|
||||
PARTITION '2026-03-01' <= VALUES < '2026-04-01',
|
||||
PARTITION '2026-04-01' <= VALUES < '2026-05-01',
|
||||
PARTITION '2026-05-01' <= VALUES < '2026-06-01'
|
||||
)
|
||||
STORED AS KUDU
|
||||
TBLPROPERTIES (
|
||||
'kudu.num_tablet_replicas' = '3',
|
||||
'kudu.compression' = 'LZ4'
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景3:多列主键
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS db_name.kudu_composite_pk (
|
||||
user_id BIGINT NOT NULL COMMENT '用户ID',
|
||||
order_date STRING NOT NULL COMMENT '订单日期',
|
||||
order_seq INT NOT NULL COMMENT '当日订单序号',
|
||||
|
||||
user_name STRING COMMENT '用户姓名',
|
||||
product_name STRING COMMENT '商品名称',
|
||||
quantity INT COMMENT '数量',
|
||||
total_amount DECIMAL(18,2) COMMENT '总金额',
|
||||
status STRING COMMENT '状态',
|
||||
create_time TIMESTAMP COMMENT '创建时间'
|
||||
)
|
||||
PRIMARY KEY (user_id, order_date, order_seq)
|
||||
PARTITION BY
|
||||
HASH(user_id) PARTITIONS 8,
|
||||
RANGE(order_date) (
|
||||
PARTITION '2026-01-01' <= VALUES < '2026-02-01',
|
||||
PARTITION '2026-02-01' <= VALUES < '2026-03-01',
|
||||
PARTITION '2026-03-01' <= VALUES < '2026-04-01'
|
||||
)
|
||||
STORED AS KUDU
|
||||
TBLPROPERTIES (
|
||||
'kudu.num_tablet_replicas' = '3'
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景4:纯 Range 分区
|
||||
-- ============================================================================
|
||||
-- 适用:按时间顺序写入,范围查询多
|
||||
|
||||
CREATE TABLE IF NOT EXISTS db_name.kudu_range_only (
|
||||
id BIGINT NOT NULL COMMENT '主键ID',
|
||||
stat_date STRING NOT NULL COMMENT '统计日期',
|
||||
metric_value DECIMAL(18,2) COMMENT '指标值',
|
||||
dimension STRING COMMENT '维度',
|
||||
etl_time TIMESTAMP COMMENT '加工时间'
|
||||
)
|
||||
PRIMARY KEY (id, stat_date)
|
||||
PARTITION BY RANGE(stat_date) (
|
||||
PARTITION '2026-01-01' <= VALUES < '2026-04-01',
|
||||
PARTITION '2026-04-01' <= VALUES < '2026-07-01',
|
||||
PARTITION '2026-07-01' <= VALUES < '2026-10-01',
|
||||
PARTITION '2026-10-01' <= VALUES < '2027-01-01'
|
||||
)
|
||||
STORED AS KUDU
|
||||
TBLPROPERTIES (
|
||||
'kudu.num_tablet_replicas' = '3'
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景5:外部表映射已有 Kudu 表
|
||||
-- ============================================================================
|
||||
|
||||
CREATE EXTERNAL TABLE IF NOT EXISTS db_name.kudu_external
|
||||
STORED AS KUDU
|
||||
TBLPROPERTIES (
|
||||
'kudu.master_addresses' = 'kudu-master-1:7051,kudu-master-2:7051,kudu-master-3:7051',
|
||||
'kudu.table_name' = 'impala.db_name.existing_table'
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景6:带压缩和副本配置
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS db_name.kudu_with_props (
|
||||
id BIGINT NOT NULL COMMENT '主键ID',
|
||||
data_date STRING NOT NULL COMMENT '数据日期',
|
||||
content STRING COMMENT '内容',
|
||||
value DOUBLE COMMENT '数值'
|
||||
)
|
||||
PRIMARY KEY (id, data_date)
|
||||
PARTITION BY
|
||||
HASH(id) PARTITIONS 8,
|
||||
RANGE(data_date) (
|
||||
PARTITION '2026-01-01' <= VALUES < '2026-02-01',
|
||||
PARTITION '2026-02-01' <= VALUES < '2026-03-01'
|
||||
)
|
||||
STORED AS KUDU
|
||||
TBLPROPERTIES (
|
||||
'kudu.num_tablet_replicas' = '3',
|
||||
'kudu.compression' = 'LZ4', -- 压缩算法
|
||||
'kudu.encryption' = 'false' -- 加密
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 字段类型速查(Kudu 支持的类型)
|
||||
-- ============================================================================
|
||||
/*
|
||||
| 类型 | 说明 | 适用场景 |
|
||||
|---------------|----------------|------------------------|
|
||||
| BOOLEAN | 布尔 | 状态标志 |
|
||||
| TINYINT | 1字节整数 | 小范围枚举 |
|
||||
| SMALLINT | 2字节整数 | 小范围数值 |
|
||||
| INT | 4字节整数 | 数量、等级 |
|
||||
| BIGINT | 8字节整数 | ID、计数 |
|
||||
| FLOAT | 4字节浮点 | 近似计算 |
|
||||
| DOUBLE | 8字节浮点 | 科学计算 |
|
||||
| DECIMAL(p,s) | 定点数 | 金额、精确数值 |
|
||||
| STRING | 变长字符串 | 名称、描述 |
|
||||
| VARCHAR(n) | 变长字符串 | 限定长度字符串 |
|
||||
| CHAR(n) | 定长字符串 | 固定长度编码 |
|
||||
| TIMESTAMP | 时间戳 | 时间字段(微秒精度) |
|
||||
| DATE | 日期 | 日期字段 |
|
||||
| BINARY | 二进制 | 二进制数据 |
|
||||
|
||||
注意:Kudu 不支持 ARRAY, MAP, STRUCT 等复杂类型
|
||||
*/
|
||||
|
||||
-- ============================================================================
|
||||
-- 建表规范说明
|
||||
-- ============================================================================
|
||||
/*
|
||||
1. 主键约束(Kudu 特有)
|
||||
- 每张 Kudu 表必须有 PRIMARY KEY
|
||||
- 主键列不能为 NULL(必须 NOT NULL)
|
||||
- 主键值不可 UPDATE(只能删除后重新插入)
|
||||
- 主键列必须包含在分区列中
|
||||
|
||||
2. 分区策略
|
||||
- Hash 分区:均匀分布,适合写入和点查
|
||||
- Range 分区:按范围查询,适合时间序列
|
||||
- Hash + Range 组合:兼顾两者优势(推荐)
|
||||
- 分区数 = tablet 数量,影响并行度
|
||||
|
||||
3. 分区设计建议
|
||||
- Hash 分区数:建议 4 的倍数,参考数据量
|
||||
- Range 分区:按时间维度,定期添加新分区
|
||||
- 单个 tablet 建议 1GB~10GB
|
||||
|
||||
4. 副本数
|
||||
- 生产环境建议 3 副本(默认)
|
||||
- Raft 协议保证一致性
|
||||
|
||||
5. 压缩
|
||||
- 推荐 LZ4(速度和压缩比平衡)
|
||||
- 可选:SNAPPY, ZLIB, LZ4
|
||||
|
||||
6. 与 Hive/Spark 表的区别
|
||||
- Kudu 表支持 UPDATE 和 DELETE
|
||||
- Kudu 表不支持 INSERT OVERWRITE
|
||||
- Kudu 表不支持复杂类型(ARRAY, MAP, STRUCT)
|
||||
- Kudu 表主键有约束,Hive/Spark 无约束
|
||||
*/
|
||||
@@ -0,0 +1,146 @@
|
||||
-- =====================================================================
|
||||
-- @Name: KUDU-D-SQL-{表名}-ETL
|
||||
-- @Version: 2.0
|
||||
-- @Desc: Kudu (via Impala) ETL 数据处理模板(临时表链式处理)
|
||||
-- @TargetDatabase: Apache Kudu (via Impala)
|
||||
-- @说明: 统一规范:禁止 CTE,每步物化为临时表,先 DROP 再 CREATE
|
||||
-- 最后一步用 UPSERT INTO 写入 Kudu 目标表
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- Step01: 基础清洗与过滤
|
||||
-- ============================================================================
|
||||
-- 说明:从源表读取数据,进行基础过滤和清洗
|
||||
-- 输入:{源表名}
|
||||
-- 输出:${db_tmp_env}.tmp_xxx_01
|
||||
|
||||
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_01;
|
||||
CREATE TABLE ${db_tmp_env}.tmp_xxx_01 AS
|
||||
SELECT
|
||||
order_id,
|
||||
user_id,
|
||||
dept_id,
|
||||
product_id,
|
||||
quantity,
|
||||
amount,
|
||||
status,
|
||||
stat_date
|
||||
FROM db_name.source_table
|
||||
WHERE stat_date = '${day_id}'
|
||||
AND status IN ('completed', 'shipped') -- 业务过滤
|
||||
AND amount > 0 -- 数据质量过滤
|
||||
AND user_id IS NOT NULL; -- NULL过滤
|
||||
|
||||
-- ============================================================================
|
||||
-- Step02: 多表关联与维度补全
|
||||
-- ============================================================================
|
||||
-- 说明:关联维度表,补全业务属性字段
|
||||
-- 输入:${db_tmp_env}.tmp_xxx_01, dim_department, dim_product
|
||||
-- 输出:${db_tmp_env}.tmp_xxx_02
|
||||
|
||||
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_02;
|
||||
CREATE TABLE ${db_tmp_env}.tmp_xxx_02 AS
|
||||
SELECT
|
||||
a.order_id,
|
||||
a.user_id,
|
||||
a.amount,
|
||||
a.quantity,
|
||||
b.dept_name, -- 维度补全:部门名称
|
||||
c.category_name, -- 维度补全:类别名称
|
||||
a.stat_date
|
||||
FROM ${db_tmp_env}.tmp_xxx_01 a
|
||||
LEFT JOIN db_name.dim_department b
|
||||
ON a.dept_id = b.dept_id
|
||||
LEFT JOIN db_name.dim_product c
|
||||
ON a.product_id = c.product_id;
|
||||
|
||||
-- ============================================================================
|
||||
-- Step03: 聚合计算与指标生成
|
||||
-- ============================================================================
|
||||
-- 说明:按业务维度聚合,计算统计指标
|
||||
-- 输入:${db_tmp_env}.tmp_xxx_02
|
||||
-- 输出:${db_tmp_env}.tmp_xxx_03
|
||||
|
||||
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_03;
|
||||
CREATE TABLE ${db_tmp_env}.tmp_xxx_03 AS
|
||||
SELECT
|
||||
stat_date,
|
||||
dept_name,
|
||||
category_name,
|
||||
COUNT(*) AS record_count, -- 记录数
|
||||
COUNT(DISTINCT user_id) AS unique_users, -- 去重用户数
|
||||
SUM(amount) AS total_amount, -- 总金额
|
||||
SUM(quantity) AS total_quantity, -- 总数量
|
||||
AVG(amount) AS avg_amount, -- 平均金额
|
||||
MAX(amount) AS max_amount -- 最大金额
|
||||
FROM ${db_tmp_env}.tmp_xxx_02
|
||||
GROUP BY stat_date, dept_name, category_name;
|
||||
|
||||
-- ============================================================================
|
||||
-- Step04: 最终输出写入 Kudu 目标表
|
||||
-- ============================================================================
|
||||
-- 说明:使用 UPSERT 写入 Kudu 目标表
|
||||
-- 输入:${db_tmp_env}.tmp_xxx_03
|
||||
-- 输出:Kudu 目标表
|
||||
|
||||
-- 方式1:UPSERT(推荐,主键存在则更新,不存在则插入)
|
||||
UPSERT INTO ${db_eda_env}.target_table
|
||||
SELECT
|
||||
-- 主键字段(Kudu 表必须有主键)
|
||||
dept_name,
|
||||
category_name,
|
||||
stat_date,
|
||||
|
||||
-- 指标字段
|
||||
record_count,
|
||||
unique_users,
|
||||
total_amount,
|
||||
total_quantity,
|
||||
avg_amount,
|
||||
max_amount,
|
||||
|
||||
-- 技术字段
|
||||
NOW() AS etl_time -- 数据加工时间
|
||||
FROM ${db_tmp_env}.tmp_xxx_03;
|
||||
|
||||
-- 方式2:需要全量刷新时(先删后插)
|
||||
-- DELETE FROM ${db_eda_env}.target_table WHERE stat_date = '${day_id}';
|
||||
-- INSERT INTO ${db_eda_env}.target_table
|
||||
-- SELECT ... FROM ${db_tmp_env}.tmp_xxx_03;
|
||||
|
||||
-- ============================================================================
|
||||
-- 关键规则说明
|
||||
-- ============================================================================
|
||||
/*
|
||||
1. 禁止使用 CTE (WITH 子句)
|
||||
- 每个步骤必须物化为临时表
|
||||
- 原因:便于调试、断点续跑、统一编码规范
|
||||
|
||||
2. 先 DROP 再 CREATE
|
||||
- 每个临时表创建前必须先 DROP TABLE IF EXISTS
|
||||
- 原因:防止表已存在导致失败
|
||||
|
||||
3. Kudu 写入方式
|
||||
- 推荐 UPSERT INTO(Kudu 核心优势)
|
||||
- 主键存在 → 更新(整行替换)
|
||||
- 主键不存在 → 插入新行
|
||||
- 需要全量刷新 → 先 DELETE 再 INSERT
|
||||
|
||||
4. Kudu 表约束
|
||||
- 不支持 INSERT OVERWRITE(用 UPSERT 或 DELETE + INSERT 替代)
|
||||
- 必须有 PRIMARY KEY
|
||||
- 主键列不能为 NULL
|
||||
- 支持 UPDATE 和 DELETE
|
||||
|
||||
5. 过滤条件前置
|
||||
- 所有过滤在最早阶段应用
|
||||
- 减少中间数据量
|
||||
|
||||
6. 临时表命名规范
|
||||
- 格式:tmp_{业务简称}_{步骤序号}
|
||||
|
||||
7. Kudu 特有注意事项
|
||||
- CONCAT 只接受 2 个参数,多参数用 CONCAT_WS
|
||||
- 不支持 collect_list/collect_set,用 GROUP_CONCAT 替代
|
||||
- 近似去重用 NDV() 函数
|
||||
*/
|
||||
@@ -0,0 +1,160 @@
|
||||
-- =====================================================================
|
||||
-- @Name: KUDU-D-SQL-{表名}-INSERT
|
||||
-- @Version: 1.0
|
||||
-- @Desc: Kudu (via Impala) 数据插入模板
|
||||
-- @TargetDatabase: Apache Kudu (via Impala)
|
||||
-- @说明: Kudu 表不支持 INSERT OVERWRITE,支持 INSERT INTO 和 UPSERT
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景1:INSERT INTO(追加写入)
|
||||
-- ============================================================================
|
||||
-- 适用:向 Kudu 表追加新数据
|
||||
|
||||
INSERT INTO db_name.kudu_table
|
||||
SELECT
|
||||
id,
|
||||
stat_date,
|
||||
name,
|
||||
department,
|
||||
amount,
|
||||
current_timestamp() AS etl_time
|
||||
FROM db_name.source_table
|
||||
WHERE stat_date = '${day_id}';
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景2:UPSERT INTO(更新插入,Kudu 特有)
|
||||
-- ============================================================================
|
||||
-- 适用:如果主键存在则更新,不存在则插入
|
||||
-- 这是 Kudu 的核心优势,其他 Hive/Spark 表不支持
|
||||
|
||||
-- 基础 UPSERT
|
||||
UPSERT INTO db_name.kudu_table
|
||||
SELECT
|
||||
id,
|
||||
stat_date,
|
||||
name,
|
||||
department,
|
||||
amount,
|
||||
current_timestamp() AS etl_time
|
||||
FROM db_name.staging_table
|
||||
WHERE stat_date = '${day_id}';
|
||||
|
||||
-- 聚合后 UPSERT(增量更新指标表)
|
||||
UPSERT INTO db_name.kudu_metrics
|
||||
SELECT
|
||||
department,
|
||||
'${day_id}' AS stat_date,
|
||||
COUNT(*) AS order_count,
|
||||
SUM(amount) AS total_amount,
|
||||
current_timestamp() AS etl_time
|
||||
FROM db_name.incremental_orders
|
||||
WHERE stat_date = '${day_id}'
|
||||
GROUP BY department;
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景3:UPDATE(Kudu 表特有)
|
||||
-- ============================================================================
|
||||
-- 适用:修改已有数据
|
||||
-- 注意:主键列不能被 UPDATE
|
||||
|
||||
-- 单条更新
|
||||
UPDATE db_name.kudu_table
|
||||
SET status = 'processed',
|
||||
updated_at = current_timestamp()
|
||||
WHERE id = 12345;
|
||||
|
||||
-- 批量条件更新
|
||||
UPDATE db_name.kudu_table
|
||||
SET status = 'expired',
|
||||
updated_at = current_timestamp()
|
||||
WHERE stat_date < '2026-01-01'
|
||||
AND status = 'active';
|
||||
|
||||
-- 关联更新(用子查询)
|
||||
UPDATE db_name.kudu_table t
|
||||
SET t.department = d.new_dept_name
|
||||
FROM db_name.dept_mapping d
|
||||
WHERE t.department = d.old_dept_name;
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景4:DELETE(Kudu 表特有)
|
||||
-- ============================================================================
|
||||
-- 适用:删除数据
|
||||
-- 注意:Kudu 的 DELETE 比 Hive/Spark 方便得多
|
||||
|
||||
-- 条件删除
|
||||
DELETE FROM db_name.kudu_table
|
||||
WHERE stat_date < '2026-01-01';
|
||||
|
||||
-- 按主键删除
|
||||
DELETE FROM db_name.kudu_table
|
||||
WHERE id IN (1001, 1002, 1003);
|
||||
|
||||
-- 关联删除(用子查询)
|
||||
DELETE FROM db_name.kudu_table
|
||||
WHERE user_id IN (
|
||||
SELECT user_id FROM db_name.blacklist
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景5:从查询结果写入
|
||||
-- ============================================================================
|
||||
|
||||
-- 简单 ETL:清洗后写入
|
||||
INSERT INTO db_name.kudu_target
|
||||
SELECT
|
||||
id,
|
||||
'${day_id}' AS stat_date,
|
||||
name,
|
||||
COALESCE(department, '未知') AS department,
|
||||
amount,
|
||||
current_timestamp() AS etl_time
|
||||
FROM db_name.raw_data
|
||||
WHERE stat_date = '${day_id}'
|
||||
AND id IS NOT NULL
|
||||
AND amount > 0;
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景6:批量 VALUES 写入
|
||||
-- ============================================================================
|
||||
|
||||
INSERT INTO db_name.kudu_table (id, stat_date, name, amount)
|
||||
VALUES
|
||||
(1, '2026-05-01', '测试1', 100.00),
|
||||
(2, '2026-05-01', '测试2', 200.00),
|
||||
(3, '2026-05-01', '测试3', 300.00);
|
||||
|
||||
-- ============================================================================
|
||||
-- 关键规则说明
|
||||
-- ============================================================================
|
||||
/*
|
||||
1. Kudu 表与 Hive/Spark 表的核心区别
|
||||
- 支持 INSERT INTO:✅
|
||||
- 支持 INSERT OVERWRITE:❌(不支持!)
|
||||
- 支持 UPSERT:✅(Kudu 独有,核心能力)
|
||||
- 支持 UPDATE:✅(Kudu 独有)
|
||||
- 支持 DELETE:✅(Kudu 独有)
|
||||
|
||||
2. UPSERT 是 Kudu 的核心优势
|
||||
- 主键存在 → 更新(整行替换)
|
||||
- 主键不存在 → 插入新行
|
||||
- 适用于:增量更新、数据修正、指标回填
|
||||
|
||||
3. INSERT INTO 注意事项
|
||||
- 如果主键冲突会报错(不会自动去重)
|
||||
- 需要确保写入数据的主键不重复,或使用 UPSERT
|
||||
|
||||
4. UPDATE 限制
|
||||
- 主键列不能被 UPDATE
|
||||
- WHERE 条件建议包含主键或分区列(性能)
|
||||
|
||||
5. DELETE 建议
|
||||
- 删除大量数据时按分区范围删除
|
||||
- 定期清理历史数据
|
||||
|
||||
6. 性能建议
|
||||
- 批量写入优于逐条写入
|
||||
- UPSERT 比 DELETE + INSERT 更高效
|
||||
- 利用主键做点查,避免全表扫描
|
||||
*/
|
||||
@@ -0,0 +1,179 @@
|
||||
-- =====================================================================
|
||||
-- @Name: KUDU-D-SQL-{表名}-QUERY
|
||||
-- @Version: 1.0
|
||||
-- @Desc: Kudu (via Impala) 查询模板
|
||||
-- @TargetDatabase: Apache Kudu (via Impala)
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- 1. 单表查询
|
||||
-- ============================================================================
|
||||
|
||||
SELECT
|
||||
id,
|
||||
name,
|
||||
amount,
|
||||
created_at
|
||||
FROM db_name.kudu_table
|
||||
WHERE stat_date = '${day_id}'
|
||||
AND status = 'active'
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1000;
|
||||
|
||||
-- ============================================================================
|
||||
-- 2. JOIN 查询
|
||||
-- ============================================================================
|
||||
|
||||
-- 两表 JOIN(Kudu 表 JOIN 非 Kudu 表也支持)
|
||||
SELECT
|
||||
k.id,
|
||||
k.name,
|
||||
k.amount,
|
||||
d.dept_name
|
||||
FROM db_name.kudu_table k
|
||||
JOIN db_name.dim_department d ON k.dept_id = d.dept_id
|
||||
WHERE k.stat_date = '${day_id}';
|
||||
|
||||
-- 多表 JOIN
|
||||
SELECT
|
||||
k.id,
|
||||
k.user_name,
|
||||
p.product_name,
|
||||
k.quantity,
|
||||
k.total_amount
|
||||
FROM db_name.kudu_orders k
|
||||
JOIN db_name.dim_users u ON k.user_id = u.user_id
|
||||
JOIN db_name.dim_products p ON k.product_id = p.product_id
|
||||
WHERE k.stat_date BETWEEN '${start_date}' AND '${end_date}';
|
||||
|
||||
-- ============================================================================
|
||||
-- 3. 聚合查询
|
||||
-- ============================================================================
|
||||
|
||||
SELECT
|
||||
department,
|
||||
COUNT(*) AS record_count,
|
||||
SUM(amount) AS total_amount,
|
||||
AVG(amount) AS avg_amount,
|
||||
MAX(amount) AS max_amount,
|
||||
MIN(amount) AS min_amount
|
||||
FROM db_name.kudu_table
|
||||
WHERE stat_date = '${day_id}'
|
||||
GROUP BY department
|
||||
HAVING COUNT(*) >= 5
|
||||
ORDER BY total_amount DESC;
|
||||
|
||||
-- 多字段分组 + 去重计数
|
||||
SELECT
|
||||
stat_date,
|
||||
region,
|
||||
COUNT(*) AS order_count,
|
||||
COUNT(DISTINCT user_id) AS unique_users,
|
||||
SUM(amount) AS total_amount
|
||||
FROM db_name.kudu_table
|
||||
WHERE stat_date BETWEEN '${start_date}' AND '${end_date}'
|
||||
GROUP BY stat_date, region;
|
||||
|
||||
-- ============================================================================
|
||||
-- 4. 窗口函数
|
||||
-- ============================================================================
|
||||
|
||||
-- ROW_NUMBER(分组取Top N)
|
||||
SELECT *
|
||||
FROM (
|
||||
SELECT
|
||||
department,
|
||||
user_name,
|
||||
amount,
|
||||
ROW_NUMBER() OVER (PARTITION BY department ORDER BY amount DESC) AS rn
|
||||
FROM db_name.kudu_table
|
||||
WHERE stat_date = '${day_id}'
|
||||
) t
|
||||
WHERE rn <= 3;
|
||||
|
||||
-- 累计聚合
|
||||
SELECT
|
||||
stat_date,
|
||||
daily_amount,
|
||||
SUM(daily_amount) OVER (ORDER BY stat_date) AS cumulative_amount
|
||||
FROM (
|
||||
SELECT stat_date, SUM(amount) AS daily_amount
|
||||
FROM db_name.kudu_table
|
||||
GROUP BY stat_date
|
||||
) t;
|
||||
|
||||
-- LAG/LEAD(环比)
|
||||
SELECT
|
||||
stat_date,
|
||||
daily_amount,
|
||||
LAG(daily_amount, 1) OVER (ORDER BY stat_date) AS prev_amount,
|
||||
daily_amount - LAG(daily_amount, 1) OVER (ORDER BY stat_date) AS daily_change
|
||||
FROM (
|
||||
SELECT stat_date, SUM(amount) AS daily_amount
|
||||
FROM db_name.kudu_table
|
||||
GROUP BY stat_date
|
||||
) t;
|
||||
|
||||
-- ============================================================================
|
||||
-- 5. 子查询
|
||||
-- ============================================================================
|
||||
|
||||
-- IN 子查询
|
||||
SELECT *
|
||||
FROM db_name.kudu_table
|
||||
WHERE user_id IN (
|
||||
SELECT user_id FROM db_name.vip_users WHERE vip_level >= 3
|
||||
)
|
||||
AND stat_date = '${day_id}';
|
||||
|
||||
-- EXISTS 子查询
|
||||
SELECT *
|
||||
FROM db_name.kudu_products p
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM db_name.kudu_inventory i
|
||||
WHERE i.product_id = p.product_id
|
||||
AND i.quantity > 0
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 6. 条件聚合(CASE WHEN + 聚合)
|
||||
-- ============================================================================
|
||||
|
||||
SELECT
|
||||
stat_date,
|
||||
COUNT(*) AS total_count,
|
||||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed_count,
|
||||
SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) AS cancelled_count,
|
||||
SUM(CASE WHEN amount > 1000 THEN amount ELSE 0 END) AS high_value_amount
|
||||
FROM db_name.kudu_table
|
||||
WHERE stat_date = '${day_id}'
|
||||
GROUP BY stat_date;
|
||||
|
||||
-- ============================================================================
|
||||
-- 7. LIMIT / OFFSET(分页)
|
||||
-- ============================================================================
|
||||
|
||||
SELECT id, name, amount
|
||||
FROM db_name.kudu_table
|
||||
WHERE stat_date = '${day_id}'
|
||||
ORDER BY id
|
||||
LIMIT 20 OFFSET 0;
|
||||
|
||||
-- ============================================================================
|
||||
-- 8. Kudu 特有:通过主键高效点查
|
||||
-- ============================================================================
|
||||
-- Kudu 主键查询可跳过扫描,直接定位 tablet
|
||||
|
||||
-- 单主键点查
|
||||
SELECT * FROM db_name.kudu_table
|
||||
WHERE id = 12345;
|
||||
|
||||
-- 复合主键点查
|
||||
SELECT * FROM db_name.kudu_composite_pk
|
||||
WHERE user_id = 1001
|
||||
AND order_date = '2026-05-01'
|
||||
AND order_seq = 1;
|
||||
|
||||
-- 主键 IN 查询
|
||||
SELECT * FROM db_name.kudu_table
|
||||
WHERE id IN (1001, 1002, 1003, 1004, 1005);
|
||||
Reference in New Issue
Block a user