From 56bd0ff3f171dc080c8fb0972ea2088e9d70833a Mon Sep 17 00:00:00 2001 From: a-32x <168935571+a-32x@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:54:41 +0200 Subject: [PATCH 1/2] perf(spectrum): offload fft to rust and render bars via canvas Move 4096-point Hann FFT calculation from the JavaScript main thread to the background meter tick in Rust using realfft. Emit pre-binned 80 dB values over audio://spectrum instead of streaming 8192 raw PCM float samples over IPC. Replace dynamic SVG rect re-creation with HTML5 2D canvas in the spectrum component to eliminate DOM reconciliation overhead. Add teardown to tauri event listeners to prevent memory accumulation in WebView2. --- src-tauri/Cargo.lock | 1 + src-tauri/Cargo.toml | 1 + src-tauri/src/audio/pipeline/meter.rs | 54 ++--- src-tauri/src/audio/pipeline/mod.rs | 1 + src-tauri/src/audio/pipeline/spectrum.rs | 122 ++++++++++++ src/lib/components/waveform_scope.svelte | 9 +- .../modules/flow/ui/effect/spectrum.svelte | 184 +++++------------- src/lib/utils/tauri_event.ts | 55 ++++-- 8 files changed, 243 insertions(+), 184 deletions(-) create mode 100644 src-tauri/src/audio/pipeline/spectrum.rs diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index ec092546..a3e31270 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -6258,6 +6258,7 @@ dependencies = [ "pipewire", "png 0.17.16", "raw-window-handle", + "realfft", "reqwest 0.12.28", "resvg", "rtrb", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 5e2e5400..87b05f6e 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -75,6 +75,7 @@ thiserror = "1" cpal = "0.15" rtrb = "0.3" rubato = "0.16" +realfft = "3.4" audio_thread_priority = "0.36" ebur128 = "0.1" flac-codec = "1.3" diff --git a/src-tauri/src/audio/pipeline/meter.rs b/src-tauri/src/audio/pipeline/meter.rs index dc4ba185..4448d0e3 100644 --- a/src-tauri/src/audio/pipeline/meter.rs +++ b/src-tauri/src/audio/pipeline/meter.rs @@ -12,11 +12,13 @@ use crate::audio::health; use super::dag::{OutputMeta, SourceMeta, DSP_BLOCK_FRAMES}; use super::output::LIVE_SPEAKER_STREAMS; +use super::spectrum::SpectrumCalculator; const METER_EVENT: &str = "audio://meter"; const LUFS_EVENT: &str = "audio://lufs"; const GR_EVENT: &str = "audio://gr"; const SCOPE_EVENT: &str = "audio://scope"; +const SPECTRUM_EVENT: &str = "audio://spectrum"; const METER_TICK: Duration = Duration::from_millis(33); const XRUN_TICK: Duration = Duration::from_millis(1000); @@ -347,6 +349,7 @@ pub(super) fn spawn_meter_thread( let join = thread::Builder::new() .name("meter-tick".into()) .spawn(move || { + let mut spectrum_calc = SpectrumCalculator::new(); while !stop_thread.load(Ordering::SeqCst) { thread::sleep(METER_TICK); for m in &meters { @@ -387,15 +390,22 @@ pub(super) fn spawn_meter_thread( let _ = app.emit(GR_EVENT, json!({ "nodeId": g.node_id, "grLin": gr_lin })); } for s in &scopes { - // Scopes emit a delta since the last tick; spectrum emits the - // full contiguous window it needs for its FFT. - let (start_frame, interleaved, ch) = if s.is_spectrum() { - let (v, ch) = s.snapshot(); - (None, v, ch) - } else { - let (start, v, ch) = s.drain(); - (Some(start), v, ch) - }; + if s.is_spectrum() { + let (interleaved, ch) = s.snapshot(); + let bars = spectrum_calc.calculate_bars(&interleaved, ch, s.sample_rate); + let _ = app.emit( + SPECTRUM_EVENT, + json!({ + "nodeId": s.node_id, + "bars": bars, + "sampleRate": s.sample_rate, + }), + ); + continue; + } + + // Scopes emit a delta since the last tick (waveform / recording) + let (start, interleaved, ch) = s.drain(); let frames = interleaved.len() / ch; if frames == 0 { continue; @@ -407,23 +417,15 @@ pub(super) fn spawn_meter_thread( chans[c].push(interleaved[base + c]); } } - let payload = match start_frame { - Some(start) => json!({ - "nodeId": s.node_id, - "channels": ch, - "data": chans, - "sampleRate": s.sample_rate, - "startFrame": start, - "session": s.session, - "baseFrames": s.base_frames, - }), - None => json!({ - "nodeId": s.node_id, - "channels": ch, - "data": chans, - "sampleRate": s.sample_rate, - }), - }; + let payload = json!({ + "nodeId": s.node_id, + "channels": ch, + "data": chans, + "sampleRate": s.sample_rate, + "startFrame": start, + "session": s.session, + "baseFrames": s.base_frames, + }); let _ = app.emit(SCOPE_EVENT, payload); } } diff --git a/src-tauri/src/audio/pipeline/mod.rs b/src-tauri/src/audio/pipeline/mod.rs index 2b62e8e5..527af412 100644 --- a/src-tauri/src/audio/pipeline/mod.rs +++ b/src-tauri/src/audio/pipeline/mod.rs @@ -41,6 +41,7 @@ mod output; #[cfg(target_os = "linux")] pub(crate) use worker::RtThread; mod sig; +pub(crate) mod spectrum; mod worker; use dag::{build_output_graph, ring_capacity_frames, OutputGraph, OutputMeta, SourceMeta}; diff --git a/src-tauri/src/audio/pipeline/spectrum.rs b/src-tauri/src/audio/pipeline/spectrum.rs new file mode 100644 index 00000000..d11d16ce --- /dev/null +++ b/src-tauri/src/audio/pipeline/spectrum.rs @@ -0,0 +1,122 @@ +use realfft::num_complex::Complex; +use realfft::{RealFftPlanner, RealToComplex}; +use std::f32::consts::PI; +use std::sync::Arc; + +pub const FFT_SIZE: usize = 4096; +pub const BINS: usize = FFT_SIZE / 2; +pub const BARS: usize = 80; +pub const F_MIN: f32 = 20.0; +pub const DB_FLOOR: f32 = -96.0; +pub const MAX_CH: usize = 8; + +pub struct SpectrumCalculator { + r2c: Arc>, + hann: Vec, + in_buf: Vec, + spectrum_buf: Vec>, + last_sample_rate: u32, + band_bins: Vec<(usize, usize)>, +} + +impl SpectrumCalculator { + pub fn new() -> Self { + let mut planner = RealFftPlanner::::new(); + let r2c = planner.plan_fft_forward(FFT_SIZE); + let in_buf = r2c.make_input_vec(); + let spectrum_buf = r2c.make_output_vec(); + + let mut hann = vec![0.0_f32; FFT_SIZE]; + for (i, h) in hann.iter_mut().enumerate() { + *h = 0.5 * (1.0 - (2.0 * PI * i as f32 / (FFT_SIZE - 1) as f32).cos()); + } + + let mut s = Self { + r2c, + hann, + in_buf, + spectrum_buf, + last_sample_rate: 0, + band_bins: Vec::with_capacity(BARS), + }; + s.update_bands(48000); + s + } + + fn update_bands(&mut self, sample_rate: u32) { + if sample_rate == self.last_sample_rate && !self.band_bins.is_empty() { + return; + } + self.last_sample_rate = sample_rate; + self.band_bins.clear(); + let f_max = (sample_rate as f32) / 2.0; + let bins_f = BINS as f32; + + for b in 0..BARS { + let f0 = F_MIN * (f_max / F_MIN).powf(b as f32 / BARS as f32); + let f1 = F_MIN * (f_max / F_MIN).powf((b + 1) as f32 / BARS as f32); + let lo = (1usize).max(((f0 / f_max) * bins_f).floor() as usize); + let hi = (lo + 1).max(((f1 / f_max) * bins_f).ceil() as usize); + self.band_bins.push((lo.min(BINS), hi.min(BINS))); + } + } + + /// Takes interleaved audio with `frames` frames and `channels` channels. + /// Runs FFT over up to `MAX_CH` channels, takes max magnitude per bin, + /// and returns 80 log-spaced bar levels in decibels [DB_FLOOR..0 dB]. + pub fn calculate_bars( + &mut self, + interleaved: &[f32], + channels: usize, + sample_rate: u32, + ) -> Vec { + self.update_bands(sample_rate); + let ch = channels.max(1); + let cap_ch = ch.min(MAX_CH); + let frames = interleaved.len() / ch; + let n = FFT_SIZE.min(frames); + + let mut band_peaks = vec![0.0_f32; BARS]; + + for c in 0..cap_ch { + // Fill input with Hann-windowed samples for this channel + for i in 0..n { + self.in_buf[i] = interleaved[i * ch + c] * self.hann[i]; + } + for i in n..FFT_SIZE { + self.in_buf[i] = 0.0; + } + + if self + .r2c + .process(&mut self.in_buf, &mut self.spectrum_buf) + .is_ok() + { + for b in 0..BARS { + let (lo, hi) = self.band_bins[b]; + let mut peak = band_peaks[b]; + for k in lo..hi { + let c = self.spectrum_buf[k]; + let mag = (c.re * c.re + c.im * c.im).sqrt(); + if mag > peak { + peak = mag; + } + } + band_peaks[b] = peak; + } + } + } + + // Convert to normalized dB + let bins_norm = BINS as f32; + let mut bars_db = vec![DB_FLOOR; BARS]; + for b in 0..BARS { + let norm = band_peaks[b] / bins_norm; + if norm > 1e-7 { + bars_db[b] = (20.0 * norm.log10()).max(DB_FLOOR); + } + } + + bars_db + } +} diff --git a/src/lib/components/waveform_scope.svelte b/src/lib/components/waveform_scope.svelte index 9756b731..3d718a6e 100644 --- a/src/lib/components/waveform_scope.svelte +++ b/src/lib/components/waveform_scope.svelte @@ -1304,10 +1304,13 @@ } } + let unlistenScope: (() => void) | undefined; + let unlistenProgress: (() => void) | undefined; + onMount(() => { ctx = canvas.getContext('2d'); - tauriListen('audio://scope', (p) => onScope(p)); - tauriListen('audio://recorder_progress', (p) => onProgress(p)); + unlistenScope = tauriListen('audio://scope', (p) => onScope(p)); + unlistenProgress = tauriListen('audio://recorder_progress', (p) => onProgress(p)); ro = new ResizeObserver((entries) => { const rect = entries[0].contentRect; const w = rect.width; @@ -1329,6 +1332,8 @@ onDestroy(() => { disposed = true; + unlistenScope?.(); + unlistenProgress?.(); if (stopTimer) clearTimeout(stopTimer); ro?.disconnect(); ctx = null; diff --git a/src/lib/modules/flow/ui/effect/spectrum.svelte b/src/lib/modules/flow/ui/effect/spectrum.svelte index 10c752df..615e248a 100644 --- a/src/lib/modules/flow/ui/effect/spectrum.svelte +++ b/src/lib/modules/flow/ui/effect/spectrum.svelte @@ -23,24 +23,16 @@ flow.updateNodeData(id, { smoothing: v }); } - // The engine ships one contiguous 4096-frame window per spectrum node - // (SPECTRUM_FRAMES) so the low end resolves (~11.7 Hz/bin at 48 kHz) without - // the discontinuities that concatenating separate 1024 snapshots would cause. - const FFT_SIZE = 4096; - const BINS = FFT_SIZE / 2; - const MAX_CH = 8; // per-channel FFTs beyond this cost more than they reveal + const BARS = 80; const F_MIN = 20; - const BARS = 80; // log-spaced across the audible range const DB_FLOOR = -96; - // Monitors run at the max input rate, not always 48 kHz; the scope event - // carries the real rate so the frequency axis stays honest. let sampleRate = $state(48_000); let fMax = $derived(sampleRate / 2); const SCALE_W = 28; - const RIGHT_PAD = 12; // mirrors the gap the dB rail leaves on the left - const TOP_PAD = 9; // headroom so the 0 dB line and its label aren't clipped + const RIGHT_PAD = 12; + const TOP_PAD = 9; const AXIS_H = 12; const DB_TICKS = [0, -12, -24, -36, -48, -60, -72, -84, -96]; @@ -48,6 +40,8 @@ let W = $state(260); let Hpx = $state(140); let plotWrap: HTMLDivElement; + let canvas: HTMLCanvasElement; + let ctx: CanvasRenderingContext2D | null = null; $effect(() => { if (!plotWrap) return; @@ -67,21 +61,6 @@ let plotW = $derived(Math.max(1, W - SCALE_W - RIGHT_PAD)); let plotH = $derived(Math.max(1, Hpx - AXIS_H)); - // Bar band edges in bin space, log-spaced. Recomputed if the monitor rate - // changes (a graph rebuild), else stable. - let bandBins = $derived.by<[number, number][]>(() => { - const out: [number, number][] = []; - for (let b = 0; b < BARS; b++) { - const f0 = F_MIN * Math.pow(fMax / F_MIN, b / BARS); - const f1 = F_MIN * Math.pow(fMax / F_MIN, (b + 1) / BARS); - const lo = Math.max(1, Math.floor((f0 / fMax) * BINS)); - const hi = Math.max(lo + 1, Math.ceil((f1 / fMax) * BINS)); - out.push([lo, Math.min(hi, BINS)]); - } - return out; - }); - - // Log-spaced 1-2-5 frequency ticks over the audible range. let freqTicks = $derived.by<{ f: number; label: string; major: boolean }[]>(() => { const out: { f: number; label: string; major: boolean }[] = []; for (let d = 10; d <= fMax; d *= 10) { @@ -94,149 +73,74 @@ return out; }); - const hann = new Float32Array(FFT_SIZE); - for (let i = 0; i < FFT_SIZE; i++) { - hann[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (FFT_SIZE - 1))); - } - - const re = new Float32Array(FFT_SIZE); - const im = new Float32Array(FFT_SIZE); - const bandPeak = new Float32Array(BARS); const barDb = new Float32Array(BARS).fill(DB_FLOOR); let hasSignal = false; - // In-place iterative radix-2 FFT. - function fft(reBuf: Float32Array, imBuf: Float32Array) { - const n = reBuf.length; - for (let i = 1, j = 0; i < n; i++) { - let bit = n >> 1; - for (; j & bit; bit >>= 1) j ^= bit; - j ^= bit; - if (i < j) { - [reBuf[i], reBuf[j]] = [reBuf[j], reBuf[i]]; - [imBuf[i], imBuf[j]] = [imBuf[j], imBuf[i]]; - } - } - for (let len = 2; len <= n; len <<= 1) { - const ang = (-2 * Math.PI) / len; - const wr = Math.cos(ang); - const wi = Math.sin(ang); - const half = len >> 1; - for (let i = 0; i < n; i += len) { - let cr = 1; - let ci = 0; - for (let k = 0; k < half; k++) { - const ar = reBuf[i + k]; - const ai = imBuf[i + k]; - const br = reBuf[i + k + half] * cr - imBuf[i + k + half] * ci; - const bi = reBuf[i + k + half] * ci + imBuf[i + k + half] * cr; - reBuf[i + k] = ar + br; - imBuf[i + k] = ai + bi; - reBuf[i + k + half] = ar - br; - imBuf[i + k + half] = ai - bi; - const ncr = cr * wr - ci * wi; - ci = cr * wi + ci * wr; - cr = ncr; - } - } - } + function dbToY(db: number): number { + const t = (db - DB_FLOOR) / -DB_FLOOR; // 0 at floor, 1 at 0 dB + return plotH - Math.max(0, Math.min(1, t)) * (plotH - TOP_PAD); + } + + function freqX(f: number): number { + const t = Math.log(f / F_MIN) / Math.log(fMax / F_MIN); + return SCALE_W + plotW * Math.max(0, Math.min(1, t)); } - function analyze(chans: number[][], sr: number) { - const ch = chans.length; - if (ch === 0) return; + function updateBars(bars: number[], sr: number) { if (sr > 0) sampleRate = sr; - const cap = Math.min(ch, MAX_CH); - const n = Math.min(FFT_SIZE, chans[0].length); - const bands = bandBins; - bandPeak.fill(0); - // Per-channel FFT combined by max: a summed mono downmix would cancel - // out-of-phase content and hide real energy. - for (let c = 0; c < cap; c++) { - const src = chans[c]; - for (let i = 0; i < n; i++) { - re[i] = (src[i] ?? 0) * hann[i]; - im[i] = 0; - } - for (let i = n; i < FFT_SIZE; i++) { - re[i] = 0; - im[i] = 0; - } - fft(re, im); - for (let b = 0; b < BARS; b++) { - const [lo, hi] = bands[b]; - let peak = bandPeak[b]; - for (let k = lo; k < hi; k++) { - const mag = Math.hypot(re[k], im[k]); - if (mag > peak) peak = mag; - } - bandPeak[b] = peak; - } - } - // Higher smoothing → smaller step toward target, so bars glide instead of - // jittering. Rise faster than fall so transients still read. const rise = 1 - 0.85 * smoothing; const fall = 0.4 - 0.37 * smoothing; for (let b = 0; b < BARS; b++) { - const norm = bandPeak[b] / BINS; - const target = norm > 1e-7 ? Math.max(DB_FLOOR, 20 * Math.log10(norm)) : DB_FLOOR; + const target = bars[b] ?? DB_FLOOR; const cur = barDb[b]; barDb[b] = cur + (target - cur) * (target > cur ? rise : fall); } hasSignal = true; } - function dbToY(db: number): number { - const t = (db - DB_FLOOR) / -DB_FLOOR; // 0 at floor, 1 at 0 dB - return plotH - Math.max(0, Math.min(1, t)) * (plotH - TOP_PAD); - } + function renderCanvas() { + if (!canvas) return; + if (!ctx) ctx = canvas.getContext('2d'); + if (!ctx) return; - function freqX(f: number): number { - const t = Math.log(f / F_MIN) / Math.log(fMax / F_MIN); - return SCALE_W + plotW * Math.max(0, Math.min(1, t)); - } - - let bars = $state<{ x: number; w: number; y: number; h: number; hue: number }[]>([]); + ctx.clearRect(0, 0, W, Hpx); + if (!hasSignal) return; - function buildFrame() { const step = plotW / BARS; - const out = new Array(BARS); for (let b = 0; b < BARS; b++) { const y = dbToY(barDb[b]); - out[b] = { - x: SCALE_W + b * step, - w: Math.max(1, step - 1), - y, - h: plotH - y, - hue: 210 - (b / BARS) * 210 - }; + const h = plotH - y; + if (h > 0) { + ctx.fillStyle = `hsl(${210 - (b / BARS) * 210} 85% 55%)`; + ctx.fillRect(SCALE_W + b * step, y, Math.max(1, step - 1), h); + } } - bars = out; } let rafId: number | undefined; function frame() { - if (hasSignal) buildFrame(); + if (hasSignal) renderCanvas(); rafId = requestAnimationFrame(frame); } - interface ScopeTick { + interface SpectrumTick { nodeId: string; - channels: number; - data: number[][]; + bars: number[]; sampleRate: number; } - tauriListen('audio://scope', (p) => { - if (p.nodeId !== id) return; - analyze(p.data, p.sampleRate); - }); + let unlisten: (() => void) | undefined; onMount(() => { + unlisten = tauriListen('audio://spectrum', (p) => { + if (p.nodeId !== id) return; + updateBars(p.bars, p.sampleRate); + }); rafId = requestAnimationFrame(frame); }); onDestroy(() => { + unlisten?.(); if (rafId !== undefined) cancelAnimationFrame(rafId); }); @@ -285,10 +189,16 @@ {#if !isPreview} {/if} -
-
+ + + + +
{#if !isPreview} diff --git a/src/lib/utils/tauri_event.ts b/src/lib/utils/tauri_event.ts index e338b5a2..3c03db96 100644 --- a/src/lib/utils/tauri_event.ts +++ b/src/lib/utils/tauri_event.ts @@ -2,23 +2,44 @@ import { onDestroy } from 'svelte'; import { listen, type UnlistenFn } from '@tauri-apps/api/event'; /** - * Component-bound `listen`. Call during component initialisation; the listener - * is registered as soon as the bridge resolves and is always unregistered on - * destroy -- including the race where the component unmounts before the - * mount-time promise settles, which would otherwise leak the listener forever - * and keep waking the dead component on every event. + * Component-bound `listen`. Safely subscribes to Tauri events with reliable teardown. + * Unregisters automatically on destroy when called during component initialization, + * and also returns an explicit cleanup function for use in onMount or manual teardown. */ -export function tauriListen(event: string, handler: (payload: T) => void): void { - let unlisten: UnlistenFn | undefined; - const p = listen(event, (e) => { - if (!unlisten) return; +export function tauriListen(event: string, handler: (payload: T) => void): () => void { + let unlistenFn: UnlistenFn | null = null; + let disposed = false; + + listen(event, (e) => { + if (disposed) return; handler(e.payload); - }); - onDestroy(() => { - p.then( - (u) => u(), - (e) => console.warn(`tauriListen(${event}):`, e) - ); - }); - p.then((u) => (unlisten = u)); + }) + .then((fn) => { + if (disposed) { + fn(); + } else { + unlistenFn = fn; + } + }) + .catch((err) => { + console.warn(`tauriListen(${event}) setup failed:`, err); + }); + + const cleanup = () => { + if (disposed) return; + disposed = true; + if (unlistenFn) { + unlistenFn(); + unlistenFn = null; + } + }; + + try { + onDestroy(cleanup); + } catch { + // Lifecycle called outside component initialization (e.g. inside onMount); + // caller should retain and call the returned cleanup function. + } + + return cleanup; } From 87fdd89c3556addb7a921a368297b12b12ade1b8 Mon Sep 17 00:00:00 2001 From: a-32x <168935571+a-32x@users.noreply.github.com> Date: Mon, 26 Oct 2026 03:22:51 +0100 Subject: [PATCH 2/2] fix(spectrum): scale canvas for retina displays and feed storybook harness --- .../modules/flow/ui/effect/spectrum.svelte | 14 ++- src/stories/harness/signal.ts | 87 +++++++++++++++++++ 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/src/lib/modules/flow/ui/effect/spectrum.svelte b/src/lib/modules/flow/ui/effect/spectrum.svelte index 615e248a..92ee0492 100644 --- a/src/lib/modules/flow/ui/effect/spectrum.svelte +++ b/src/lib/modules/flow/ui/effect/spectrum.svelte @@ -103,6 +103,15 @@ if (!ctx) ctx = canvas.getContext('2d'); if (!ctx) return; + const dpr = window.devicePixelRatio || 1; + const bw = Math.max(1, Math.round(W * dpr)); + const bh = Math.max(1, Math.round(Hpx * dpr)); + if (canvas.width !== bw) canvas.width = bw; + if (canvas.height !== bh) canvas.height = bh; + if (canvas.style.width !== `${W}px`) canvas.style.width = `${W}px`; + if (canvas.style.height !== `${Hpx}px`) canvas.style.height = `${Hpx}px`; + + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); ctx.clearRect(0, 0, W, Hpx); if (!hasSignal) return; @@ -193,9 +202,8 @@ + class="pointer-events-none absolute inset-0 block h-full w-full bg-[#111]" + aria-hidden="true">