Add one-skill
This commit is contained in:
@@ -0,0 +1,209 @@
|
||||
-- =====================================================================
|
||||
-- @Name: DORIS-D-SQL-{表名}-CREATE
|
||||
-- @Version: 1.0
|
||||
-- @Desc: Apache Doris 建表模板(OLAP 多模型)
|
||||
-- @TargetDatabase: Apache Doris
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景1:Duplicate Key 模型(明细表)
|
||||
-- ============================================================================
|
||||
-- 适用:保留原始明细数据,不做预聚合,数据无冗余
|
||||
-- 特点:数据按 Key 排序存储,支持所有列的查询和聚合
|
||||
|
||||
CREATE TABLE IF NOT EXISTS db_name.detail_table (
|
||||
-- Key 列(排序字段)
|
||||
order_id BIGINT COMMENT '订单ID',
|
||||
order_date DATE COMMENT '订单日期',
|
||||
user_id BIGINT COMMENT '用户ID',
|
||||
|
||||
-- Value 列
|
||||
user_name VARCHAR(50) COMMENT '用户姓名',
|
||||
product_id BIGINT COMMENT '商品ID',
|
||||
product_name VARCHAR(200) COMMENT '商品名称',
|
||||
quantity INT COMMENT '购买数量',
|
||||
unit_price DECIMAL(18,2) COMMENT '单价',
|
||||
total_amount DECIMAL(18,2) COMMENT '总金额',
|
||||
status VARCHAR(20) COMMENT '订单状态',
|
||||
create_time DATETIME COMMENT '创建时间'
|
||||
)
|
||||
DUPLICATE KEY(order_id, order_date, user_id)
|
||||
COMMENT '订单明细表'
|
||||
PARTITION BY RANGE(order_date) (
|
||||
PARTITION p202601 VALUES LESS THAN ('2026-02-01'),
|
||||
PARTITION p202602 VALUES LESS THAN ('2026-03-01'),
|
||||
PARTITION p202603 VALUES LESS THAN ('2026-04-01')
|
||||
)
|
||||
DISTRIBUTED BY HASH(order_id) BUCKETS 8
|
||||
PROPERTIES (
|
||||
'replication_num' = '3',
|
||||
'storage_format' = 'V2'
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景2:Aggregate Key 模型(聚合表)
|
||||
-- ============================================================================
|
||||
-- 适用:预聚合场景,相同 Key 的数据自动合并
|
||||
-- 特点:Value 列必须指定聚合函数(SUM, REPLACE, MAX, MIN, HLL_UNION, BITMAP_UNION)
|
||||
|
||||
CREATE TABLE IF NOT EXISTS db_name.agg_table (
|
||||
-- Key 列(聚合维度)
|
||||
stat_date DATE COMMENT '统计日期',
|
||||
department VARCHAR(100) COMMENT '部门名称',
|
||||
region VARCHAR(100) COMMENT '地区',
|
||||
|
||||
-- Value 列(带聚合函数)
|
||||
order_count BIGINT SUM COMMENT '订单总数',
|
||||
total_amount DECIMAL(18,2) SUM COMMENT '总金额',
|
||||
unique_users BIGINT REPLACE COMMENT '去重用户数(预计算值)',
|
||||
max_amount DECIMAL(18,2) MAX COMMENT '最大金额',
|
||||
last_update DATETIME REPLACE COMMENT '最后更新时间'
|
||||
)
|
||||
AGGREGATE KEY(stat_date, department, region)
|
||||
COMMENT '部门销售聚合表'
|
||||
PARTITION BY RANGE(stat_date) (
|
||||
PARTITION p202601 VALUES LESS THAN ('2026-02-01'),
|
||||
PARTITION p202602 VALUES LESS THAN ('2026-03-01')
|
||||
)
|
||||
DISTRIBUTED BY HASH(department) BUCKETS 8
|
||||
PROPERTIES (
|
||||
'replication_num' = '3',
|
||||
'storage_format' = 'V2'
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景3:Unique Key 模型(唯一主键表)
|
||||
-- ============================================================================
|
||||
-- 适用:需要按主键更新/去重的场景
|
||||
-- 特点:相同主键的数据保留最新一条(整行替换)
|
||||
|
||||
CREATE TABLE IF NOT EXISTS db_name.unique_table (
|
||||
-- Key 列(主键,必须唯一)
|
||||
user_id BIGINT COMMENT '用户ID',
|
||||
|
||||
-- Value 列
|
||||
user_name VARCHAR(50) COMMENT '用户姓名',
|
||||
phone VARCHAR(20) COMMENT '手机号',
|
||||
email VARCHAR(100) COMMENT '邮箱',
|
||||
vip_level INT COMMENT 'VIP等级',
|
||||
register_date DATE COMMENT '注册日期',
|
||||
last_login DATETIME COMMENT '最后登录时间',
|
||||
status VARCHAR(10) COMMENT '状态'
|
||||
)
|
||||
UNIQUE KEY(user_id)
|
||||
COMMENT '用户信息表(按主键更新)'
|
||||
DISTRIBUTED BY HASH(user_id) BUCKETS 16
|
||||
PROPERTIES (
|
||||
'replication_num' = '3',
|
||||
'enable_unique_key_merge_based_on_replica' = 'true'
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景4:带动态分区属性
|
||||
-- ============================================================================
|
||||
-- 适用:按日自动创建和管理分区
|
||||
|
||||
CREATE TABLE IF NOT EXISTS db_name.auto_partition_table (
|
||||
stat_date DATE COMMENT '统计日期',
|
||||
department VARCHAR(100) COMMENT '部门',
|
||||
metric_value DECIMAL(18,2) SUM COMMENT '指标值',
|
||||
record_count BIGINT SUM COMMENT '记录数'
|
||||
)
|
||||
AGGREGATE KEY(stat_date, department)
|
||||
COMMENT '自动分区示例表'
|
||||
PARTITION BY RANGE(stat_date) ()
|
||||
DISTRIBUTED BY HASH(department) BUCKETS 8
|
||||
PROPERTIES (
|
||||
'replication_num' = '3',
|
||||
'dynamic_partition.enable' = 'true',
|
||||
'dynamic_partition.time_unit' = 'DAY',
|
||||
'dynamic_partition.start' = '-30', -- 保留30天历史
|
||||
'dynamic_partition.end' = '3', -- 预创建3天
|
||||
'dynamic_partition.prefix' = 'p',
|
||||
'dynamic_partition.buckets' = '8'
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景5:多分区 + 多分桶
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS db_name.multi_partition_table (
|
||||
stat_date DATE COMMENT '统计日期',
|
||||
region VARCHAR(50) COMMENT '地区',
|
||||
city VARCHAR(50) COMMENT '城市',
|
||||
user_id BIGINT COMMENT '用户ID',
|
||||
amount DECIMAL(18,2) SUM COMMENT '金额'
|
||||
)
|
||||
AGGREGATE KEY(stat_date, region, city, user_id)
|
||||
COMMENT '多维度分区示例'
|
||||
PARTITION BY RANGE(stat_date) (
|
||||
PARTITION p202601 VALUES LESS THAN ('2026-02-01'),
|
||||
PARTITION p202602 VALUES LESS THAN ('2026-03-01')
|
||||
)
|
||||
DISTRIBUTED BY HASH(user_id) BUCKETS 32
|
||||
PROPERTIES (
|
||||
'replication_num' = '3',
|
||||
'in_memory' = 'false',
|
||||
'storage_format' = 'V2',
|
||||
'compression' = 'LZ4'
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 字段类型速查
|
||||
-- ============================================================================
|
||||
/*
|
||||
| 类型 | 说明 | 适用场景 |
|
||||
|---------------|----------------|------------------------|
|
||||
| BOOLEAN | 布尔 | 状态标志 |
|
||||
| TINYINT | 1字节整数 | 小范围枚举 |
|
||||
| SMALLINT | 2字节整数 | 小范围数值 |
|
||||
| INT | 4字节整数 | 数量、等级 |
|
||||
| BIGINT | 8字节整数 | ID、计数、大数值 |
|
||||
| LARGEINT | 16字节整数 | 超大数值 |
|
||||
| FLOAT | 4字节浮点 | 近似计算 |
|
||||
| DOUBLE | 8字节浮点 | 科学计算 |
|
||||
| DECIMAL(p,s) | 定点数 | 金额、精确数值 |
|
||||
| DATE | 日期 | 日期字段(无时间) |
|
||||
| DATETIME | 日期时间 | 时间戳(精确到秒) |
|
||||
| CHAR(n) | 定长字符串 | 固定长度编码 |
|
||||
| VARCHAR(n) | 变长字符串 | 名称、描述 |
|
||||
| STRING | 变长字符串 | 大文本(无长度限制) |
|
||||
| BITMAP | 位图 | 精确去重(仅聚合模型) |
|
||||
| HLL | HyperLogLog | 近似去重(仅聚合模型) |
|
||||
| JSON | JSON | JSON数据存储 |
|
||||
*/
|
||||
|
||||
-- ============================================================================
|
||||
-- 建表规范说明
|
||||
-- ============================================================================
|
||||
/*
|
||||
1. 模型选择
|
||||
- Duplicate Key:保留原始明细,不做预聚合
|
||||
- Aggregate Key:预聚合,相同 Key 的 Value 自动合并
|
||||
- Unique Key:按主键去重,保留最新数据
|
||||
|
||||
2. 分区设计
|
||||
- 按时间字段 RANGE 分区(最常用)
|
||||
- 支持动态分区自动管理
|
||||
- 单分区数据量建议 1GB~10GB
|
||||
|
||||
3. 分桶设计
|
||||
- 使用高基数列做 HASH 分桶
|
||||
- 分桶数 = BE节点数 × CPU核数(参考值)
|
||||
- 单桶数据量建议 100MB~1GB
|
||||
|
||||
4. 副本数
|
||||
- 生产环境建议 3 副本
|
||||
- 测试环境可设 1 副本
|
||||
|
||||
5. Key 列选择
|
||||
- Duplicate Key:高频过滤/排序字段
|
||||
- Aggregate Key:聚合维度字段
|
||||
- Unique Key:业务主键
|
||||
|
||||
6. 注意事项
|
||||
- Key 列必须在 Value 列之前
|
||||
- 分区列必须是 Key 列
|
||||
- 分桶列必须是 Key 列
|
||||
- BITMAP/HLL 仅用于 Aggregate 模型的 Value 列
|
||||
*/
|
||||
@@ -0,0 +1,128 @@
|
||||
-- =====================================================================
|
||||
-- @Name: DORIS-D-SQL-{表名}-ETL
|
||||
-- @Version: 2.0
|
||||
-- @Desc: Apache Doris ETL 数据处理模板(临时表链式处理)
|
||||
-- @TargetDatabase: Apache Doris
|
||||
-- @说明: 统一规范:禁止 CTE,每步物化为临时表,先 DROP 再 CREATE
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- Step01: 基础清洗与过滤
|
||||
-- ============================================================================
|
||||
-- 说明:从源表读取数据,进行基础过滤和清洗
|
||||
-- 输入:{源表名}
|
||||
-- 输出:${db_tmp_env}.tmp_xxx_01
|
||||
|
||||
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_01;
|
||||
CREATE TABLE ${db_tmp_env}.tmp_xxx_01 AS
|
||||
SELECT
|
||||
order_id,
|
||||
user_id,
|
||||
dept_id,
|
||||
total_amount,
|
||||
status,
|
||||
order_date
|
||||
FROM db_name.source_table
|
||||
WHERE order_date = '${day_id}'
|
||||
AND status IN ('completed', 'shipped') -- 业务过滤
|
||||
AND total_amount > 0 -- 数据质量过滤
|
||||
AND user_id IS NOT NULL; -- NULL过滤
|
||||
|
||||
-- ============================================================================
|
||||
-- Step02: 多表关联与维度补全
|
||||
-- ============================================================================
|
||||
-- 说明:关联维度表,补全业务属性字段
|
||||
-- 输入:${db_tmp_env}.tmp_xxx_01, dim_department, dim_category
|
||||
-- 输出:${db_tmp_env}.tmp_xxx_02
|
||||
|
||||
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_02;
|
||||
CREATE TABLE ${db_tmp_env}.tmp_xxx_02 AS
|
||||
SELECT
|
||||
a.order_id,
|
||||
a.user_id,
|
||||
a.total_amount,
|
||||
a.status,
|
||||
b.dept_name, -- 维度补全:部门名称
|
||||
c.category_name, -- 维度补全:类别名称
|
||||
a.order_date
|
||||
FROM ${db_tmp_env}.tmp_xxx_01 a
|
||||
LEFT JOIN db_name.dim_department b
|
||||
ON a.dept_id = b.dept_id
|
||||
LEFT JOIN db_name.dim_category c
|
||||
ON a.category_id = c.category_id;
|
||||
|
||||
-- ============================================================================
|
||||
-- Step03: 聚合计算与指标生成
|
||||
-- ============================================================================
|
||||
-- 说明:按业务维度聚合,计算统计指标
|
||||
-- 输入:${db_tmp_env}.tmp_xxx_02
|
||||
-- 输出:${db_tmp_env}.tmp_xxx_03
|
||||
|
||||
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_03;
|
||||
CREATE TABLE ${db_tmp_env}.tmp_xxx_03 AS
|
||||
SELECT
|
||||
order_date,
|
||||
dept_name,
|
||||
category_name,
|
||||
COUNT(*) AS record_count, -- 记录数
|
||||
COUNT(DISTINCT user_id) AS unique_users, -- 去重用户数
|
||||
SUM(total_amount) AS total_amount, -- 总金额
|
||||
AVG(total_amount) AS avg_amount, -- 平均金额
|
||||
MAX(total_amount) AS max_amount -- 最大金额
|
||||
FROM ${db_tmp_env}.tmp_xxx_02
|
||||
GROUP BY order_date, dept_name, category_name;
|
||||
|
||||
-- ============================================================================
|
||||
-- Step04: 最终输出写入目标表
|
||||
-- ============================================================================
|
||||
-- 说明:补全目标表标准字段,写入结果表
|
||||
-- 输入:${db_tmp_env}.tmp_xxx_03
|
||||
-- 输出:目标表
|
||||
|
||||
INSERT INTO ${db_eda_env}.target_table
|
||||
SELECT
|
||||
-- 业务字段
|
||||
dept_name,
|
||||
category_name,
|
||||
record_count,
|
||||
unique_users,
|
||||
total_amount,
|
||||
avg_amount,
|
||||
max_amount,
|
||||
|
||||
-- 技术字段
|
||||
NOW() AS etl_time, -- 数据加工时间
|
||||
'${day_id}' AS stat_date -- 统计日期
|
||||
FROM ${db_tmp_env}.tmp_xxx_03;
|
||||
|
||||
-- ============================================================================
|
||||
-- 关键规则说明
|
||||
-- ============================================================================
|
||||
/*
|
||||
1. 禁止使用 CTE (WITH 子句)
|
||||
- 每个步骤必须物化为临时表
|
||||
- 原因:便于调试、断点续跑、统一编码规范
|
||||
|
||||
2. 先 DROP 再 CREATE
|
||||
- 每个临时表创建前必须先 DROP TABLE IF EXISTS
|
||||
- 原因:防止表已存在导致失败
|
||||
|
||||
3. Doris 写入方式
|
||||
- 默认使用 INSERT INTO
|
||||
- Aggregate Key 表:自动合并相同 Key 的数据
|
||||
- Unique Key 表:自动按主键去重,保留最新数据
|
||||
- Doris 2.0+ 也支持 INSERT OVERWRITE
|
||||
|
||||
4. 过滤条件前置
|
||||
- 所有过滤在最早阶段应用
|
||||
- 减少中间数据量
|
||||
|
||||
5. 临时表命名规范
|
||||
- 格式:tmp_{业务简称}_{步骤序号}
|
||||
- 示例:tmp_order_stats_01, tmp_order_stats_02
|
||||
|
||||
6. Doris 特有注意事项
|
||||
- 不支持 LEFT SEMI JOIN / LEFT ANTI JOIN
|
||||
- 日期函数用 MySQL 风格:DATE_FORMAT, DATE_ADD(INTERVAL)
|
||||
- 不支持 collect_list/collect_set,用 GROUP_CONCAT 替代
|
||||
*/
|
||||
@@ -0,0 +1,147 @@
|
||||
-- =====================================================================
|
||||
-- @Name: DORIS-D-SQL-{表名}-INSERT
|
||||
-- @Version: 1.0
|
||||
-- @Desc: Apache Doris 数据插入模板
|
||||
-- @TargetDatabase: Apache Doris
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景1:INSERT INTO(追加写入)
|
||||
-- ============================================================================
|
||||
-- 适用:向 Doris 表追加数据,不会删除已有数据
|
||||
|
||||
INSERT INTO db_name.target_table
|
||||
SELECT
|
||||
stat_date,
|
||||
department,
|
||||
region,
|
||||
order_count,
|
||||
total_amount
|
||||
FROM db_name.source_table
|
||||
WHERE stat_date = '${day_id}';
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景2:INSERT OVERWRITE(覆盖写入)
|
||||
-- ============================================================================
|
||||
-- 适用:覆盖目标表(或指定分区)的全部数据
|
||||
-- 注意:Doris 2.0+ 支持,且仅适用于 Partition 表
|
||||
|
||||
-- 覆盖整表
|
||||
INSERT OVERWRITE db_name.target_table
|
||||
SELECT
|
||||
stat_date,
|
||||
department,
|
||||
region,
|
||||
order_count,
|
||||
total_amount
|
||||
FROM db_name.source_table;
|
||||
|
||||
-- 覆盖指定分区(推荐)
|
||||
INSERT OVERWRITE db_name.target_table
|
||||
PARTITION(p202605)
|
||||
SELECT
|
||||
department,
|
||||
region,
|
||||
order_count,
|
||||
total_amount
|
||||
FROM db_name.source_table
|
||||
WHERE stat_date >= '2026-05-01'
|
||||
AND stat_date < '2026-06-01';
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景3:从查询结果写入(ETL 场景)
|
||||
-- ============================================================================
|
||||
|
||||
-- 简单转换后写入
|
||||
INSERT INTO db_name.target_table
|
||||
SELECT
|
||||
order_date,
|
||||
department,
|
||||
COUNT(*) AS order_count,
|
||||
COUNT(DISTINCT user_id) AS unique_users,
|
||||
SUM(total_amount) AS total_amount,
|
||||
AVG(total_amount) AS avg_amount
|
||||
FROM db_name.source_orders o
|
||||
LEFT JOIN db_name.dim_department d ON o.dept_id = d.dept_id
|
||||
WHERE o.order_date = '${day_id}'
|
||||
GROUP BY order_date, department;
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景4:批量 VALUES 写入
|
||||
-- ============================================================================
|
||||
|
||||
INSERT INTO db_name.target_table (stat_date, department, amount)
|
||||
VALUES
|
||||
('2026-05-01', '市场部', 10000.00),
|
||||
('2026-05-01', '技术部', 25000.00),
|
||||
('2026-05-01', '运营部', 18000.00);
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景5:Stream Load(数据导入)
|
||||
-- ============================================================================
|
||||
-- 适用:大批量数据导入(百万级以上)
|
||||
-- 注意:Stream Load 通过 HTTP 协议提交,不是 SQL 语法
|
||||
|
||||
/*
|
||||
-- curl 命令示例
|
||||
curl --location-trusted -u user:password \
|
||||
-H "label:load_order_20260501" \
|
||||
-H "column_separator:," \
|
||||
-H "columns:order_id,order_date,user_id,total_amount" \
|
||||
-T data.csv \
|
||||
http://fe_host:8030/api/db_name/orders/_stream_load
|
||||
*/
|
||||
|
||||
-- ============================================================================
|
||||
-- 场景6:Broker Load(外部数据源导入)
|
||||
-- ============================================================================
|
||||
|
||||
/*
|
||||
LOAD LABEL db_name.load_label_20260501
|
||||
(
|
||||
DATA INFILE('hdfs://namenode:8020/path/to/data/*')
|
||||
INTO TABLE target_table
|
||||
COLUMNS TERMINATED BY ','
|
||||
(stat_date, department, region, amount)
|
||||
SET (amount = amount * 1.0)
|
||||
)
|
||||
WITH BROKER 'broker_name'
|
||||
(
|
||||
'username' = 'hdfs_user',
|
||||
'password' = 'hdfs_password'
|
||||
)
|
||||
PROPERTIES
|
||||
(
|
||||
'timeout' = '3600',
|
||||
'max_filter_ratio' = '0.01'
|
||||
);
|
||||
*/
|
||||
|
||||
-- ============================================================================
|
||||
-- 关键规则说明
|
||||
-- ============================================================================
|
||||
/*
|
||||
1. INSERT INTO vs INSERT OVERWRITE
|
||||
- INSERT INTO:追加数据,不删除已有数据
|
||||
- INSERT OVERWRITE:覆盖数据(Doris 2.0+ 支持)
|
||||
- 日常增量推荐 INSERT INTO,全量刷新推荐 INSERT OVERWRITE
|
||||
|
||||
2. Doris 不使用临时表链式处理
|
||||
- 与 Spark 不同,Doris 通常用单条 SQL 或 CTE 完成 ETL
|
||||
- 直接 INSERT INTO ... SELECT ... 即可
|
||||
|
||||
3. 字段顺序
|
||||
- SELECT 字段顺序必须与目标表列定义顺序一致
|
||||
- 或显式指定列名:INSERT INTO table (col1, col2) SELECT ...
|
||||
|
||||
4. 数据导入方式选择
|
||||
- 少量数据:INSERT INTO ... SELECT ... 或 INSERT INTO ... VALUES ...
|
||||
- 大批量导入:Stream Load(HTTP PUT,最高性能)
|
||||
- HDFS 导入:Broker Load
|
||||
- 外部数据源:Routine Load(Kafka 等)
|
||||
|
||||
5. 性能建议
|
||||
- 批量写入优于逐条写入
|
||||
- Stream Load 是最高性能的导入方式
|
||||
- 建议攒批后一次性写入,避免频繁小批量导入
|
||||
*/
|
||||
@@ -0,0 +1,189 @@
|
||||
-- =====================================================================
|
||||
-- @Name: DORIS-D-SQL-{表名}-QUERY
|
||||
-- @Version: 1.0
|
||||
-- @Desc: Apache Doris 查询模板
|
||||
-- @TargetDatabase: Apache Doris
|
||||
-- =====================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- 1. 单表查询
|
||||
-- ============================================================================
|
||||
|
||||
SELECT
|
||||
order_id,
|
||||
user_name,
|
||||
total_amount,
|
||||
create_time
|
||||
FROM db_name.orders
|
||||
WHERE order_date = '${day_id}'
|
||||
AND status = 'completed'
|
||||
ORDER BY total_amount DESC
|
||||
LIMIT 100;
|
||||
|
||||
-- ============================================================================
|
||||
-- 2. JOIN 查询
|
||||
-- ============================================================================
|
||||
|
||||
-- 两表 JOIN
|
||||
SELECT
|
||||
o.order_id,
|
||||
o.total_amount,
|
||||
u.user_name,
|
||||
u.vip_level
|
||||
FROM db_name.orders o
|
||||
JOIN db_name.users u ON o.user_id = u.user_id
|
||||
WHERE o.order_date = '${day_id}'
|
||||
AND o.status = 'completed';
|
||||
|
||||
-- 多表 JOIN
|
||||
SELECT
|
||||
o.order_id,
|
||||
u.user_name,
|
||||
p.product_name,
|
||||
oi.quantity,
|
||||
oi.unit_price
|
||||
FROM db_name.orders o
|
||||
JOIN db_name.users u ON o.user_id = u.user_id
|
||||
JOIN db_name.order_items oi ON o.order_id = oi.order_id
|
||||
JOIN db_name.products p ON oi.product_id = p.product_id
|
||||
WHERE o.order_date BETWEEN '${start_date}' AND '${end_date}';
|
||||
|
||||
-- ============================================================================
|
||||
-- 3. 聚合查询
|
||||
-- ============================================================================
|
||||
|
||||
-- GROUP BY + HAVING
|
||||
SELECT
|
||||
department,
|
||||
COUNT(*) AS order_count,
|
||||
SUM(total_amount) AS total_amount,
|
||||
AVG(total_amount) AS avg_amount
|
||||
FROM db_name.orders
|
||||
WHERE order_date = '${day_id}'
|
||||
GROUP BY department
|
||||
HAVING COUNT(*) >= 5
|
||||
ORDER BY total_amount DESC;
|
||||
|
||||
-- 多字段分组 + 去重计数
|
||||
SELECT
|
||||
order_date,
|
||||
region,
|
||||
COUNT(*) AS order_count,
|
||||
COUNT(DISTINCT user_id) AS unique_users,
|
||||
SUM(total_amount) AS total_amount
|
||||
FROM db_name.orders
|
||||
WHERE order_date BETWEEN '${start_date}' AND '${end_date}'
|
||||
GROUP BY order_date, region;
|
||||
|
||||
-- ============================================================================
|
||||
-- 4. 窗口函数
|
||||
-- ============================================================================
|
||||
|
||||
-- ROW_NUMBER(分组取Top N)
|
||||
SELECT *
|
||||
FROM (
|
||||
SELECT
|
||||
department,
|
||||
user_name,
|
||||
total_amount,
|
||||
ROW_NUMBER() OVER (PARTITION BY department ORDER BY total_amount DESC) AS rn
|
||||
FROM db_name.orders
|
||||
WHERE order_date = '${day_id}'
|
||||
) t
|
||||
WHERE rn <= 3;
|
||||
|
||||
-- 累计聚合
|
||||
SELECT
|
||||
order_date,
|
||||
daily_amount,
|
||||
SUM(daily_amount) OVER (ORDER BY order_date) AS cumulative_amount,
|
||||
AVG(daily_amount) OVER (
|
||||
ORDER BY order_date
|
||||
ROWS BETWEEN 6 PRECEDING AND CURRENT ROW
|
||||
) AS moving_avg_7d
|
||||
FROM (
|
||||
SELECT order_date, SUM(total_amount) AS daily_amount
|
||||
FROM db_name.orders
|
||||
GROUP BY order_date
|
||||
) t;
|
||||
|
||||
-- LAG/LEAD(环比计算)
|
||||
SELECT
|
||||
order_date,
|
||||
daily_amount,
|
||||
LAG(daily_amount, 1) OVER (ORDER BY order_date) AS prev_amount,
|
||||
daily_amount - LAG(daily_amount, 1) OVER (ORDER BY order_date) AS daily_change,
|
||||
ROUND(
|
||||
(daily_amount - LAG(daily_amount, 1) OVER (ORDER BY order_date))
|
||||
/ LAG(daily_amount, 1) OVER (ORDER BY order_date) * 100, 2
|
||||
) AS growth_rate_pct
|
||||
FROM (
|
||||
SELECT order_date, SUM(total_amount) AS daily_amount
|
||||
FROM db_name.orders
|
||||
GROUP BY order_date
|
||||
) t;
|
||||
|
||||
-- ============================================================================
|
||||
-- 5. 分页查询
|
||||
-- ============================================================================
|
||||
|
||||
-- LIMIT OFFSET 分页(Doris 原生支持)
|
||||
SELECT
|
||||
order_id, user_name, total_amount
|
||||
FROM db_name.orders
|
||||
WHERE order_date = '${day_id}'
|
||||
ORDER BY order_id
|
||||
LIMIT 20 OFFSET 0; -- 第1页,每页20条
|
||||
|
||||
-- ============================================================================
|
||||
-- 6. 子查询
|
||||
-- ============================================================================
|
||||
|
||||
-- IN 子查询
|
||||
SELECT *
|
||||
FROM db_name.orders
|
||||
WHERE user_id IN (
|
||||
SELECT user_id FROM db_name.users WHERE vip_level >= 3
|
||||
)
|
||||
AND order_date = '${day_id}';
|
||||
|
||||
-- EXISTS 子查询
|
||||
SELECT *
|
||||
FROM db_name.products p
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM db_name.inventory i
|
||||
WHERE i.product_id = p.product_id
|
||||
AND i.quantity > 0
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 7. 条件聚合(CASE WHEN + 聚合)
|
||||
-- ============================================================================
|
||||
|
||||
SELECT
|
||||
order_date,
|
||||
COUNT(*) AS total_orders,
|
||||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed_count,
|
||||
SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) AS cancelled_count,
|
||||
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending_count,
|
||||
SUM(CASE WHEN total_amount > 1000 THEN total_amount ELSE 0 END) AS high_value_amount
|
||||
FROM db_name.orders
|
||||
WHERE order_date = '${day_id}'
|
||||
GROUP BY order_date;
|
||||
|
||||
-- ============================================================================
|
||||
-- 8. Bitmap 精确去重(Doris 特有)
|
||||
-- ============================================================================
|
||||
|
||||
-- 使用 bitmap 做精确去重(需要在 Aggregate Key 模型中定义 BITMAP 类型列)
|
||||
-- 注意:bitmap 函数只能用于包含 BITMAP 类型列的表
|
||||
|
||||
-- 精确去重计数(预计算场景,在 Aggregate Key 表中使用 BITMAP_UNION)
|
||||
-- 建表时 Value 列定义:user_id_bitmap BITMAP BITMAP_UNION
|
||||
-- 查询时:
|
||||
-- SELECT date, bitmap_union_count(user_id_bitmap) AS uv FROM table GROUP BY date;
|
||||
|
||||
-- HLL 近似去重
|
||||
-- 建表时 Value 列定义:user_id_hll HLL HLL_UNION
|
||||
-- 查询时:
|
||||
-- SELECT date, hll_union_agg(user_id_hll) AS approx_uv FROM table GROUP BY date;
|
||||
Reference in New Issue
Block a user