Add one-skill

This commit is contained in:
Xin Wang
2026-05-13 11:03:00 +08:00
parent a4c8b29176
commit f9e36ef92d
34 changed files with 7656 additions and 0 deletions

View File

@@ -0,0 +1,209 @@
-- =====================================================================
-- @Name: DORIS-D-SQL-{表名}-CREATE
-- @Version: 1.0
-- @Desc: Apache Doris 建表模板OLAP 多模型)
-- @TargetDatabase: Apache Doris
-- =====================================================================
-- ============================================================================
-- 场景1Duplicate Key 模型(明细表)
-- ============================================================================
-- 适用:保留原始明细数据,不做预聚合,数据无冗余
-- 特点:数据按 Key 排序存储,支持所有列的查询和聚合
CREATE TABLE IF NOT EXISTS db_name.detail_table (
-- Key 列(排序字段)
order_id BIGINT COMMENT '订单ID',
order_date DATE COMMENT '订单日期',
user_id BIGINT COMMENT '用户ID',
-- Value 列
user_name VARCHAR(50) COMMENT '用户姓名',
product_id BIGINT COMMENT '商品ID',
product_name VARCHAR(200) COMMENT '商品名称',
quantity INT COMMENT '购买数量',
unit_price DECIMAL(18,2) COMMENT '单价',
total_amount DECIMAL(18,2) COMMENT '总金额',
status VARCHAR(20) COMMENT '订单状态',
create_time DATETIME COMMENT '创建时间'
)
DUPLICATE KEY(order_id, order_date, user_id)
COMMENT '订单明细表'
PARTITION BY RANGE(order_date) (
PARTITION p202601 VALUES LESS THAN ('2026-02-01'),
PARTITION p202602 VALUES LESS THAN ('2026-03-01'),
PARTITION p202603 VALUES LESS THAN ('2026-04-01')
)
DISTRIBUTED BY HASH(order_id) BUCKETS 8
PROPERTIES (
'replication_num' = '3',
'storage_format' = 'V2'
);
-- ============================================================================
-- 场景2Aggregate Key 模型(聚合表)
-- ============================================================================
-- 适用:预聚合场景,相同 Key 的数据自动合并
-- 特点Value 列必须指定聚合函数SUM, REPLACE, MAX, MIN, HLL_UNION, BITMAP_UNION
CREATE TABLE IF NOT EXISTS db_name.agg_table (
-- Key 列(聚合维度)
stat_date DATE COMMENT '统计日期',
department VARCHAR(100) COMMENT '部门名称',
region VARCHAR(100) COMMENT '地区',
-- Value 列(带聚合函数)
order_count BIGINT SUM COMMENT '订单总数',
total_amount DECIMAL(18,2) SUM COMMENT '总金额',
unique_users BIGINT REPLACE COMMENT '去重用户数(预计算值)',
max_amount DECIMAL(18,2) MAX COMMENT '最大金额',
last_update DATETIME REPLACE COMMENT '最后更新时间'
)
AGGREGATE KEY(stat_date, department, region)
COMMENT '部门销售聚合表'
PARTITION BY RANGE(stat_date) (
PARTITION p202601 VALUES LESS THAN ('2026-02-01'),
PARTITION p202602 VALUES LESS THAN ('2026-03-01')
)
DISTRIBUTED BY HASH(department) BUCKETS 8
PROPERTIES (
'replication_num' = '3',
'storage_format' = 'V2'
);
-- ============================================================================
-- 场景3Unique Key 模型(唯一主键表)
-- ============================================================================
-- 适用:需要按主键更新/去重的场景
-- 特点:相同主键的数据保留最新一条(整行替换)
CREATE TABLE IF NOT EXISTS db_name.unique_table (
-- Key 列(主键,必须唯一)
user_id BIGINT COMMENT '用户ID',
-- Value 列
user_name VARCHAR(50) COMMENT '用户姓名',
phone VARCHAR(20) COMMENT '手机号',
email VARCHAR(100) COMMENT '邮箱',
vip_level INT COMMENT 'VIP等级',
register_date DATE COMMENT '注册日期',
last_login DATETIME COMMENT '最后登录时间',
status VARCHAR(10) COMMENT '状态'
)
UNIQUE KEY(user_id)
COMMENT '用户信息表(按主键更新)'
DISTRIBUTED BY HASH(user_id) BUCKETS 16
PROPERTIES (
'replication_num' = '3',
'enable_unique_key_merge_based_on_replica' = 'true'
);
-- ============================================================================
-- 场景4带动态分区属性
-- ============================================================================
-- 适用:按日自动创建和管理分区
CREATE TABLE IF NOT EXISTS db_name.auto_partition_table (
stat_date DATE COMMENT '统计日期',
department VARCHAR(100) COMMENT '部门',
metric_value DECIMAL(18,2) SUM COMMENT '指标值',
record_count BIGINT SUM COMMENT '记录数'
)
AGGREGATE KEY(stat_date, department)
COMMENT '自动分区示例表'
PARTITION BY RANGE(stat_date) ()
DISTRIBUTED BY HASH(department) BUCKETS 8
PROPERTIES (
'replication_num' = '3',
'dynamic_partition.enable' = 'true',
'dynamic_partition.time_unit' = 'DAY',
'dynamic_partition.start' = '-30', -- 保留30天历史
'dynamic_partition.end' = '3', -- 预创建3天
'dynamic_partition.prefix' = 'p',
'dynamic_partition.buckets' = '8'
);
-- ============================================================================
-- 场景5多分区 + 多分桶
-- ============================================================================
CREATE TABLE IF NOT EXISTS db_name.multi_partition_table (
stat_date DATE COMMENT '统计日期',
region VARCHAR(50) COMMENT '地区',
city VARCHAR(50) COMMENT '城市',
user_id BIGINT COMMENT '用户ID',
amount DECIMAL(18,2) SUM COMMENT '金额'
)
AGGREGATE KEY(stat_date, region, city, user_id)
COMMENT '多维度分区示例'
PARTITION BY RANGE(stat_date) (
PARTITION p202601 VALUES LESS THAN ('2026-02-01'),
PARTITION p202602 VALUES LESS THAN ('2026-03-01')
)
DISTRIBUTED BY HASH(user_id) BUCKETS 32
PROPERTIES (
'replication_num' = '3',
'in_memory' = 'false',
'storage_format' = 'V2',
'compression' = 'LZ4'
);
-- ============================================================================
-- 字段类型速查
-- ============================================================================
/*
| 类型 | 说明 | 适用场景 |
|---------------|----------------|------------------------|
| BOOLEAN | 布尔 | 状态标志 |
| TINYINT | 1字节整数 | 小范围枚举 |
| SMALLINT | 2字节整数 | 小范围数值 |
| INT | 4字节整数 | 数量、等级 |
| BIGINT | 8字节整数 | ID、计数、大数值 |
| LARGEINT | 16字节整数 | 超大数值 |
| FLOAT | 4字节浮点 | 近似计算 |
| DOUBLE | 8字节浮点 | 科学计算 |
| DECIMAL(p,s) | 定点数 | 金额、精确数值 |
| DATE | 日期 | 日期字段(无时间) |
| DATETIME | 日期时间 | 时间戳(精确到秒) |
| CHAR(n) | 定长字符串 | 固定长度编码 |
| VARCHAR(n) | 变长字符串 | 名称、描述 |
| STRING | 变长字符串 | 大文本(无长度限制) |
| BITMAP | 位图 | 精确去重(仅聚合模型) |
| HLL | HyperLogLog | 近似去重(仅聚合模型) |
| JSON | JSON | JSON数据存储 |
*/
-- ============================================================================
-- 建表规范说明
-- ============================================================================
/*
1. 模型选择
- Duplicate Key保留原始明细不做预聚合
- Aggregate Key预聚合相同 Key 的 Value 自动合并
- Unique Key按主键去重保留最新数据
2. 分区设计
- 按时间字段 RANGE 分区(最常用)
- 支持动态分区自动管理
- 单分区数据量建议 1GB~10GB
3. 分桶设计
- 使用高基数列做 HASH 分桶
- 分桶数 = BE节点数 × CPU核数参考值
- 单桶数据量建议 100MB~1GB
4. 副本数
- 生产环境建议 3 副本
- 测试环境可设 1 副本
5. Key 列选择
- Duplicate Key高频过滤/排序字段
- Aggregate Key聚合维度字段
- Unique Key业务主键
6. 注意事项
- Key 列必须在 Value 列之前
- 分区列必须是 Key 列
- 分桶列必须是 Key 列
- BITMAP/HLL 仅用于 Aggregate 模型的 Value 列
*/

View File

@@ -0,0 +1,128 @@
-- =====================================================================
-- @Name: DORIS-D-SQL-{表名}-ETL
-- @Version: 2.0
-- @Desc: Apache Doris ETL 数据处理模板(临时表链式处理)
-- @TargetDatabase: Apache Doris
-- @说明: 统一规范:禁止 CTE每步物化为临时表先 DROP 再 CREATE
-- =====================================================================
-- ============================================================================
-- Step01: 基础清洗与过滤
-- ============================================================================
-- 说明:从源表读取数据,进行基础过滤和清洗
-- 输入:{源表名}
-- 输出:${db_tmp_env}.tmp_xxx_01
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_01;
CREATE TABLE ${db_tmp_env}.tmp_xxx_01 AS
SELECT
order_id,
user_id,
dept_id,
total_amount,
status,
order_date
FROM db_name.source_table
WHERE order_date = '${day_id}'
AND status IN ('completed', 'shipped') -- 业务过滤
AND total_amount > 0 -- 数据质量过滤
AND user_id IS NOT NULL; -- NULL过滤
-- ============================================================================
-- Step02: 多表关联与维度补全
-- ============================================================================
-- 说明:关联维度表,补全业务属性字段
-- 输入:${db_tmp_env}.tmp_xxx_01, dim_department, dim_category
-- 输出:${db_tmp_env}.tmp_xxx_02
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_02;
CREATE TABLE ${db_tmp_env}.tmp_xxx_02 AS
SELECT
a.order_id,
a.user_id,
a.total_amount,
a.status,
b.dept_name, -- 维度补全:部门名称
c.category_name, -- 维度补全:类别名称
a.order_date
FROM ${db_tmp_env}.tmp_xxx_01 a
LEFT JOIN db_name.dim_department b
ON a.dept_id = b.dept_id
LEFT JOIN db_name.dim_category c
ON a.category_id = c.category_id;
-- ============================================================================
-- Step03: 聚合计算与指标生成
-- ============================================================================
-- 说明:按业务维度聚合,计算统计指标
-- 输入:${db_tmp_env}.tmp_xxx_02
-- 输出:${db_tmp_env}.tmp_xxx_03
DROP TABLE IF EXISTS ${db_tmp_env}.tmp_xxx_03;
CREATE TABLE ${db_tmp_env}.tmp_xxx_03 AS
SELECT
order_date,
dept_name,
category_name,
COUNT(*) AS record_count, -- 记录数
COUNT(DISTINCT user_id) AS unique_users, -- 去重用户数
SUM(total_amount) AS total_amount, -- 总金额
AVG(total_amount) AS avg_amount, -- 平均金额
MAX(total_amount) AS max_amount -- 最大金额
FROM ${db_tmp_env}.tmp_xxx_02
GROUP BY order_date, dept_name, category_name;
-- ============================================================================
-- Step04: 最终输出写入目标表
-- ============================================================================
-- 说明:补全目标表标准字段,写入结果表
-- 输入:${db_tmp_env}.tmp_xxx_03
-- 输出:目标表
INSERT INTO ${db_eda_env}.target_table
SELECT
-- 业务字段
dept_name,
category_name,
record_count,
unique_users,
total_amount,
avg_amount,
max_amount,
-- 技术字段
NOW() AS etl_time, -- 数据加工时间
'${day_id}' AS stat_date -- 统计日期
FROM ${db_tmp_env}.tmp_xxx_03;
-- ============================================================================
-- 关键规则说明
-- ============================================================================
/*
1. 禁止使用 CTE (WITH 子句)
- 每个步骤必须物化为临时表
- 原因:便于调试、断点续跑、统一编码规范
2. 先 DROP 再 CREATE
- 每个临时表创建前必须先 DROP TABLE IF EXISTS
- 原因:防止表已存在导致失败
3. Doris 写入方式
- 默认使用 INSERT INTO
- Aggregate Key 表:自动合并相同 Key 的数据
- Unique Key 表:自动按主键去重,保留最新数据
- Doris 2.0+ 也支持 INSERT OVERWRITE
4. 过滤条件前置
- 所有过滤在最早阶段应用
- 减少中间数据量
5. 临时表命名规范
- 格式tmp_{业务简称}_{步骤序号}
- 示例tmp_order_stats_01, tmp_order_stats_02
6. Doris 特有注意事项
- 不支持 LEFT SEMI JOIN / LEFT ANTI JOIN
- 日期函数用 MySQL 风格DATE_FORMAT, DATE_ADD(INTERVAL)
- 不支持 collect_list/collect_set用 GROUP_CONCAT 替代
*/

View File

@@ -0,0 +1,147 @@
-- =====================================================================
-- @Name: DORIS-D-SQL-{表名}-INSERT
-- @Version: 1.0
-- @Desc: Apache Doris 数据插入模板
-- @TargetDatabase: Apache Doris
-- =====================================================================
-- ============================================================================
-- 场景1INSERT INTO追加写入
-- ============================================================================
-- 适用:向 Doris 表追加数据,不会删除已有数据
INSERT INTO db_name.target_table
SELECT
stat_date,
department,
region,
order_count,
total_amount
FROM db_name.source_table
WHERE stat_date = '${day_id}';
-- ============================================================================
-- 场景2INSERT OVERWRITE覆盖写入
-- ============================================================================
-- 适用:覆盖目标表(或指定分区)的全部数据
-- 注意Doris 2.0+ 支持,且仅适用于 Partition 表
-- 覆盖整表
INSERT OVERWRITE db_name.target_table
SELECT
stat_date,
department,
region,
order_count,
total_amount
FROM db_name.source_table;
-- 覆盖指定分区(推荐)
INSERT OVERWRITE db_name.target_table
PARTITION(p202605)
SELECT
department,
region,
order_count,
total_amount
FROM db_name.source_table
WHERE stat_date >= '2026-05-01'
AND stat_date < '2026-06-01';
-- ============================================================================
-- 场景3从查询结果写入ETL 场景)
-- ============================================================================
-- 简单转换后写入
INSERT INTO db_name.target_table
SELECT
order_date,
department,
COUNT(*) AS order_count,
COUNT(DISTINCT user_id) AS unique_users,
SUM(total_amount) AS total_amount,
AVG(total_amount) AS avg_amount
FROM db_name.source_orders o
LEFT JOIN db_name.dim_department d ON o.dept_id = d.dept_id
WHERE o.order_date = '${day_id}'
GROUP BY order_date, department;
-- ============================================================================
-- 场景4批量 VALUES 写入
-- ============================================================================
INSERT INTO db_name.target_table (stat_date, department, amount)
VALUES
('2026-05-01', '市场部', 10000.00),
('2026-05-01', '技术部', 25000.00),
('2026-05-01', '运营部', 18000.00);
-- ============================================================================
-- 场景5Stream Load数据导入
-- ============================================================================
-- 适用:大批量数据导入(百万级以上)
-- 注意Stream Load 通过 HTTP 协议提交,不是 SQL 语法
/*
-- curl 命令示例
curl --location-trusted -u user:password \
-H "label:load_order_20260501" \
-H "column_separator:," \
-H "columns:order_id,order_date,user_id,total_amount" \
-T data.csv \
http://fe_host:8030/api/db_name/orders/_stream_load
*/
-- ============================================================================
-- 场景6Broker Load外部数据源导入
-- ============================================================================
/*
LOAD LABEL db_name.load_label_20260501
(
DATA INFILE('hdfs://namenode:8020/path/to/data/*')
INTO TABLE target_table
COLUMNS TERMINATED BY ','
(stat_date, department, region, amount)
SET (amount = amount * 1.0)
)
WITH BROKER 'broker_name'
(
'username' = 'hdfs_user',
'password' = 'hdfs_password'
)
PROPERTIES
(
'timeout' = '3600',
'max_filter_ratio' = '0.01'
);
*/
-- ============================================================================
-- 关键规则说明
-- ============================================================================
/*
1. INSERT INTO vs INSERT OVERWRITE
- INSERT INTO追加数据不删除已有数据
- INSERT OVERWRITE覆盖数据Doris 2.0+ 支持)
- 日常增量推荐 INSERT INTO全量刷新推荐 INSERT OVERWRITE
2. Doris 不使用临时表链式处理
- 与 Spark 不同Doris 通常用单条 SQL 或 CTE 完成 ETL
- 直接 INSERT INTO ... SELECT ... 即可
3. 字段顺序
- SELECT 字段顺序必须与目标表列定义顺序一致
- 或显式指定列名INSERT INTO table (col1, col2) SELECT ...
4. 数据导入方式选择
- 少量数据INSERT INTO ... SELECT ... 或 INSERT INTO ... VALUES ...
- 大批量导入Stream LoadHTTP PUT最高性能
- HDFS 导入Broker Load
- 外部数据源Routine LoadKafka 等)
5. 性能建议
- 批量写入优于逐条写入
- Stream Load 是最高性能的导入方式
- 建议攒批后一次性写入,避免频繁小批量导入
*/

View File

@@ -0,0 +1,189 @@
-- =====================================================================
-- @Name: DORIS-D-SQL-{表名}-QUERY
-- @Version: 1.0
-- @Desc: Apache Doris 查询模板
-- @TargetDatabase: Apache Doris
-- =====================================================================
-- ============================================================================
-- 1. 单表查询
-- ============================================================================
SELECT
order_id,
user_name,
total_amount,
create_time
FROM db_name.orders
WHERE order_date = '${day_id}'
AND status = 'completed'
ORDER BY total_amount DESC
LIMIT 100;
-- ============================================================================
-- 2. JOIN 查询
-- ============================================================================
-- 两表 JOIN
SELECT
o.order_id,
o.total_amount,
u.user_name,
u.vip_level
FROM db_name.orders o
JOIN db_name.users u ON o.user_id = u.user_id
WHERE o.order_date = '${day_id}'
AND o.status = 'completed';
-- 多表 JOIN
SELECT
o.order_id,
u.user_name,
p.product_name,
oi.quantity,
oi.unit_price
FROM db_name.orders o
JOIN db_name.users u ON o.user_id = u.user_id
JOIN db_name.order_items oi ON o.order_id = oi.order_id
JOIN db_name.products p ON oi.product_id = p.product_id
WHERE o.order_date BETWEEN '${start_date}' AND '${end_date}';
-- ============================================================================
-- 3. 聚合查询
-- ============================================================================
-- GROUP BY + HAVING
SELECT
department,
COUNT(*) AS order_count,
SUM(total_amount) AS total_amount,
AVG(total_amount) AS avg_amount
FROM db_name.orders
WHERE order_date = '${day_id}'
GROUP BY department
HAVING COUNT(*) >= 5
ORDER BY total_amount DESC;
-- 多字段分组 + 去重计数
SELECT
order_date,
region,
COUNT(*) AS order_count,
COUNT(DISTINCT user_id) AS unique_users,
SUM(total_amount) AS total_amount
FROM db_name.orders
WHERE order_date BETWEEN '${start_date}' AND '${end_date}'
GROUP BY order_date, region;
-- ============================================================================
-- 4. 窗口函数
-- ============================================================================
-- ROW_NUMBER分组取Top N
SELECT *
FROM (
SELECT
department,
user_name,
total_amount,
ROW_NUMBER() OVER (PARTITION BY department ORDER BY total_amount DESC) AS rn
FROM db_name.orders
WHERE order_date = '${day_id}'
) t
WHERE rn <= 3;
-- 累计聚合
SELECT
order_date,
daily_amount,
SUM(daily_amount) OVER (ORDER BY order_date) AS cumulative_amount,
AVG(daily_amount) OVER (
ORDER BY order_date
ROWS BETWEEN 6 PRECEDING AND CURRENT ROW
) AS moving_avg_7d
FROM (
SELECT order_date, SUM(total_amount) AS daily_amount
FROM db_name.orders
GROUP BY order_date
) t;
-- LAG/LEAD环比计算
SELECT
order_date,
daily_amount,
LAG(daily_amount, 1) OVER (ORDER BY order_date) AS prev_amount,
daily_amount - LAG(daily_amount, 1) OVER (ORDER BY order_date) AS daily_change,
ROUND(
(daily_amount - LAG(daily_amount, 1) OVER (ORDER BY order_date))
/ LAG(daily_amount, 1) OVER (ORDER BY order_date) * 100, 2
) AS growth_rate_pct
FROM (
SELECT order_date, SUM(total_amount) AS daily_amount
FROM db_name.orders
GROUP BY order_date
) t;
-- ============================================================================
-- 5. 分页查询
-- ============================================================================
-- LIMIT OFFSET 分页Doris 原生支持)
SELECT
order_id, user_name, total_amount
FROM db_name.orders
WHERE order_date = '${day_id}'
ORDER BY order_id
LIMIT 20 OFFSET 0; -- 第1页每页20条
-- ============================================================================
-- 6. 子查询
-- ============================================================================
-- IN 子查询
SELECT *
FROM db_name.orders
WHERE user_id IN (
SELECT user_id FROM db_name.users WHERE vip_level >= 3
)
AND order_date = '${day_id}';
-- EXISTS 子查询
SELECT *
FROM db_name.products p
WHERE EXISTS (
SELECT 1 FROM db_name.inventory i
WHERE i.product_id = p.product_id
AND i.quantity > 0
);
-- ============================================================================
-- 7. 条件聚合CASE WHEN + 聚合)
-- ============================================================================
SELECT
order_date,
COUNT(*) AS total_orders,
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed_count,
SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) AS cancelled_count,
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending_count,
SUM(CASE WHEN total_amount > 1000 THEN total_amount ELSE 0 END) AS high_value_amount
FROM db_name.orders
WHERE order_date = '${day_id}'
GROUP BY order_date;
-- ============================================================================
-- 8. Bitmap 精确去重Doris 特有)
-- ============================================================================
-- 使用 bitmap 做精确去重(需要在 Aggregate Key 模型中定义 BITMAP 类型列)
-- 注意bitmap 函数只能用于包含 BITMAP 类型列的表
-- 精确去重计数(预计算场景,在 Aggregate Key 表中使用 BITMAP_UNION
-- 建表时 Value 列定义user_id_bitmap BITMAP BITMAP_UNION
-- 查询时:
-- SELECT date, bitmap_union_count(user_id_bitmap) AS uv FROM table GROUP BY date;
-- HLL 近似去重
-- 建表时 Value 列定义user_id_hll HLL HLL_UNION
-- 查询时:
-- SELECT date, hll_union_agg(user_id_hll) AS approx_uv FROM table GROUP BY date;