Implement audio visualizers and refactor AssistantPage
- Introduce three new audio visualizer components: AuraVisualizer, SpectrumVisualizer, and WaveVisualizer, enhancing the audio interaction experience. - Replace the deprecated VoiceVisualizer with the new visualizers, ensuring a cohesive visual language across components. - Update the AssistantPage to support dynamic visualization style switching, improving user engagement during audio interactions. - Refactor DebugVoicePanel to accommodate the new visualizer props and enhance the overall debugging interface.
This commit is contained in:
Xin Wang
262
frontend/src/components/ui/aura-visualizer.tsx
Normal file
262
frontend/src/components/ui/aura-visualizer.tsx
Normal file
@@ -0,0 +1,262 @@
|
||||
"use client";
|
||||
|
||||
import * as React from "react";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { useAudioAnalyser } from "@/hooks/use-audio-analyser";
|
||||
import { readPalette, type RGB } from "@/lib/visualizer-palette";
|
||||
|
||||
export type AuraVisualizerProps = {
|
||||
/** 是否激活：true 时采集麦克风并随音量律动，false 时显示静态呼吸态 */
|
||||
active?: boolean;
|
||||
/** 外部分析器；提供后组件不再自行申请麦克风 */
|
||||
analyser?: AnalyserNode | null;
|
||||
/** 外部音频流；提供后用它构建分析器，而不调用 getUserMedia */
|
||||
stream?: MediaStream | null;
|
||||
/** 画布直径（px） */
|
||||
size?: number;
|
||||
/** 申请麦克风失败时回调 */
|
||||
onError?: (error: unknown) => void;
|
||||
className?: string;
|
||||
};
|
||||
|
||||
const VERT = `
|
||||
attribute vec2 a_pos;
|
||||
void main() {
|
||||
gl_Position = vec4(a_pos, 0.0, 1.0);
|
||||
}
|
||||
`;
|
||||
|
||||
// 光环本体：极坐标下一个被 fbm 噪声轻微扰动边缘的柔光环 + 中心柔光，
|
||||
// 由 u_volume（音量）驱动缩放 / 亮度。配色随半径在三色间平滑过渡。
|
||||
const FRAG = `
|
||||
precision highp float;
|
||||
|
||||
uniform vec2 u_resolution;
|
||||
uniform float u_time;
|
||||
uniform float u_volume; // 0~1，已平滑
|
||||
uniform float u_active; // 0 静态 / 1 激活
|
||||
uniform float u_theme; // 0 暗色 / 1 亮色
|
||||
uniform vec3 u_c0; // sky
|
||||
uniform vec3 u_c1; // lavender
|
||||
uniform vec3 u_c2; // rose
|
||||
|
||||
float hash(vec2 p) {
|
||||
p = fract(p * vec2(123.34, 456.21));
|
||||
p += dot(p, p + 45.32);
|
||||
return fract(p.x * p.y);
|
||||
}
|
||||
|
||||
float noise(vec2 p) {
|
||||
vec2 i = floor(p);
|
||||
vec2 f = fract(p);
|
||||
vec2 u = f * f * (3.0 - 2.0 * f);
|
||||
return mix(
|
||||
mix(hash(i + vec2(0.0, 0.0)), hash(i + vec2(1.0, 0.0)), u.x),
|
||||
mix(hash(i + vec2(0.0, 1.0)), hash(i + vec2(1.0, 1.0)), u.x),
|
||||
u.y
|
||||
);
|
||||
}
|
||||
|
||||
float fbm(vec2 p) {
|
||||
float v = 0.0;
|
||||
float a = 0.5;
|
||||
mat2 m = mat2(1.6, 1.2, -1.2, 1.6);
|
||||
for (int i = 0; i < 5; i++) {
|
||||
v += a * noise(p);
|
||||
p = m * p;
|
||||
a *= 0.5;
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
// 静态抖动，消除平滑渐变上的色带
|
||||
float dither(vec2 p) {
|
||||
return (hash(p) - 0.5) / 255.0;
|
||||
}
|
||||
|
||||
void main() {
|
||||
vec2 uv = (gl_FragCoord.xy - 0.5 * u_resolution) / min(u_resolution.x, u_resolution.y);
|
||||
float r = length(uv);
|
||||
float ang = atan(uv.y, uv.x);
|
||||
|
||||
float vol = clamp(u_volume, 0.0, 1.0) * u_active;
|
||||
float breathe = 0.5 + 0.5 * sin(u_time * 0.8);
|
||||
|
||||
// 单层缓慢扰动的有机边缘
|
||||
float n = fbm(vec2(cos(ang), sin(ang)) * 1.6 + u_time * 0.12);
|
||||
float baseR = 0.32 + 0.03 * breathe + 0.09 * vol;
|
||||
float edge = baseR + (n - 0.5) * (0.03 + 0.05 * vol);
|
||||
float dr = r - edge;
|
||||
|
||||
float ring = exp(-dr * dr * 120.0); // 柔和发光环
|
||||
float halo = exp(-r * r * 5.0); // 中心柔光
|
||||
// 亮色模式削弱中心光晕，避免在白底上铺成灰雾
|
||||
float intensity = ring * (0.9 + 0.6 * vol)
|
||||
+ halo * (mix(0.28, 0.12, u_theme) + 0.4 * vol);
|
||||
|
||||
// 平滑的径向配色，不随时间闪烁
|
||||
vec3 col = mix(u_c0, u_c1, smoothstep(0.0, 0.55, r));
|
||||
col = mix(col, u_c2, smoothstep(0.4, 0.95, r));
|
||||
|
||||
// 亮色 token 偏浅：提升饱和并适度加深，使颜色在浅背景上不发灰
|
||||
float luma = dot(col, vec3(0.299, 0.587, 0.114));
|
||||
col = clamp(mix(vec3(luma), col, mix(1.2, 1.65, u_theme)), 0.0, 1.0);
|
||||
col *= mix(1.0, 0.72, u_theme);
|
||||
|
||||
float bright = 0.85 + vol + 0.12 * breathe;
|
||||
vec3 hdr = col * intensity * bright;
|
||||
|
||||
// 暗色：辉光优雅泛白；亮色：保持饱和色，不向白过曝
|
||||
vec3 darkMap = vec3(1.0) - exp(-hdr * 1.5);
|
||||
vec3 lightMap = col * clamp(intensity * bright, 0.0, 1.0);
|
||||
vec3 mapped = mix(darkMap, lightMap, u_theme);
|
||||
mapped += dither(gl_FragCoord.xy);
|
||||
|
||||
float alpha = clamp(intensity * mix(1.1, 1.4, u_theme), 0.0, 1.0);
|
||||
gl_FragColor = vec4(mapped, alpha);
|
||||
}
|
||||
`;
|
||||
|
||||
function compile(gl: WebGLRenderingContext, type: number, src: string) {
|
||||
const sh = gl.createShader(type);
|
||||
if (!sh) return null;
|
||||
gl.shaderSource(sh, src);
|
||||
gl.compileShader(sh);
|
||||
if (!gl.getShaderParameter(sh, gl.COMPILE_STATUS)) {
|
||||
gl.deleteShader(sh);
|
||||
return null;
|
||||
}
|
||||
return sh;
|
||||
}
|
||||
|
||||
const norm = ({ r, g, b }: RGB): [number, number, number] => [
|
||||
r / 255,
|
||||
g / 255,
|
||||
b / 255,
|
||||
];
|
||||
|
||||
export function AuraVisualizer({
|
||||
active = false,
|
||||
analyser = null,
|
||||
stream = null,
|
||||
size = 220,
|
||||
onError,
|
||||
className,
|
||||
}: AuraVisualizerProps) {
|
||||
const canvasRef = React.useRef<HTMLCanvasElement>(null);
|
||||
const volumeRef = React.useRef(0);
|
||||
const activeRef = React.useRef(active);
|
||||
const analyserRef = useAudioAnalyser({ active, analyser, stream, onError });
|
||||
|
||||
React.useEffect(() => {
|
||||
activeRef.current = active;
|
||||
}, [active]);
|
||||
|
||||
React.useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
const gl = canvas.getContext("webgl", {
|
||||
alpha: true,
|
||||
premultipliedAlpha: false,
|
||||
antialias: true,
|
||||
});
|
||||
if (!gl) return;
|
||||
|
||||
const vs = compile(gl, gl.VERTEX_SHADER, VERT);
|
||||
const fs = compile(gl, gl.FRAGMENT_SHADER, FRAG);
|
||||
const prog = gl.createProgram();
|
||||
if (!vs || !fs || !prog) return;
|
||||
gl.attachShader(prog, vs);
|
||||
gl.attachShader(prog, fs);
|
||||
gl.linkProgram(prog);
|
||||
if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) return;
|
||||
gl.useProgram(prog);
|
||||
|
||||
const buf = gl.createBuffer();
|
||||
gl.bindBuffer(gl.ARRAY_BUFFER, buf);
|
||||
gl.bufferData(
|
||||
gl.ARRAY_BUFFER,
|
||||
new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]),
|
||||
gl.STATIC_DRAW,
|
||||
);
|
||||
const aPos = gl.getAttribLocation(prog, "a_pos");
|
||||
gl.enableVertexAttribArray(aPos);
|
||||
gl.vertexAttribPointer(aPos, 2, gl.FLOAT, false, 0, 0);
|
||||
|
||||
const uRes = gl.getUniformLocation(prog, "u_resolution");
|
||||
const uTime = gl.getUniformLocation(prog, "u_time");
|
||||
const uVol = gl.getUniformLocation(prog, "u_volume");
|
||||
const uActive = gl.getUniformLocation(prog, "u_active");
|
||||
const uTheme = gl.getUniformLocation(prog, "u_theme");
|
||||
const uC0 = gl.getUniformLocation(prog, "u_c0");
|
||||
const uC1 = gl.getUniformLocation(prog, "u_c1");
|
||||
const uC2 = gl.getUniformLocation(prog, "u_c2");
|
||||
|
||||
const dpr = Math.min(window.devicePixelRatio || 1, 2);
|
||||
const px = Math.round(size * dpr);
|
||||
canvas.width = px;
|
||||
canvas.height = px;
|
||||
gl.viewport(0, 0, px, px);
|
||||
gl.uniform2f(uRes, px, px);
|
||||
|
||||
gl.enable(gl.BLEND);
|
||||
gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
|
||||
|
||||
const freq = new Uint8Array(256);
|
||||
let raf = 0;
|
||||
const start = performance.now();
|
||||
|
||||
const draw = () => {
|
||||
const t = (performance.now() - start) / 1000;
|
||||
|
||||
// 由频谱算出单一音量标量（低中频，人声主能量），再平滑
|
||||
const node = analyserRef.current;
|
||||
let target = 0;
|
||||
if (node) {
|
||||
node.getByteFrequencyData(freq);
|
||||
const bins = Math.floor(freq.length * 0.6);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < bins; i++) sum += freq[i];
|
||||
target = sum / bins / 255;
|
||||
}
|
||||
volumeRef.current += (target - volumeRef.current) * 0.18;
|
||||
|
||||
const { sky, lav, rose } = readPalette(canvas);
|
||||
const light = document.documentElement.classList.contains("dark")
|
||||
? 0
|
||||
: 1;
|
||||
gl.uniform1f(uTime, t);
|
||||
gl.uniform1f(uVol, volumeRef.current);
|
||||
gl.uniform1f(uActive, activeRef.current ? 1 : 0);
|
||||
gl.uniform1f(uTheme, light);
|
||||
gl.uniform3fv(uC0, norm(sky));
|
||||
gl.uniform3fv(uC1, norm(lav));
|
||||
gl.uniform3fv(uC2, norm(rose));
|
||||
|
||||
gl.clearColor(0, 0, 0, 0);
|
||||
gl.clear(gl.COLOR_BUFFER_BIT);
|
||||
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
||||
|
||||
raf = requestAnimationFrame(draw);
|
||||
};
|
||||
|
||||
raf = requestAnimationFrame(draw);
|
||||
return () => {
|
||||
cancelAnimationFrame(raf);
|
||||
gl.deleteProgram(prog);
|
||||
gl.deleteShader(vs);
|
||||
gl.deleteShader(fs);
|
||||
gl.deleteBuffer(buf);
|
||||
};
|
||||
}, [size, analyserRef]);
|
||||
|
||||
return (
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
role="img"
|
||||
aria-label="麦克风音频可视化（光环）"
|
||||
style={{ width: size, height: size }}
|
||||
className={cn("select-none", className)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user