fef783aVoice-driven rhythm capture: voices trained by example, takes classified onto them, synth playbackJeremy Magland 1/**
2 * Per-hit features. Two roles are kept apart: `peakDb` is expression (how
3 * hard the hit was; it becomes the note's strength and is deliberately
4 * excluded from classification, so playing softly does not change which
5 * voice a hit lands on), and `vec` is identity (spectral shape, attack,
6 * decay), which is what the classifier compares. Every entry in `vec` is a
7 * log or a ratio, so distances behave across the huge dynamic ranges
8 * involved.
9 */
10import { powerSpectrum } from '../audio/fft.ts';
12export interface HitFeatures {
13 /** Peak short-term RMS over the hit, dBFS. */
14 peakDb: number;
15 /** Identity vector; order fixed, see VEC_NAMES. */
16 vec: number[];
17}
19export const VEC_NAMES = [
20 'log centroid', 'log rolloff85',
21 'band 0-200', 'band 200-800', 'band 800-2500', 'band 2500+',
22 'zcr kHz', 'log attack ms', 'log decay ms',
23];
25const SPEC_N = 4096;
26const ENV_WIN = 256;
27const ENV_HOP = 128;
28const EPS = 1e-12;
30/** Extract features for the hit starting at sample s0; s1 bounds it (the
31 * next onset or the end of the clip). */
32export function extractHit(x: Float32Array, sr: number, s0: number, s1: number): HitFeatures {
33 const end = Math.min(x.length, s1, s0 + Math.floor(0.5 * sr));
34 const seg = x.subarray(s0, Math.max(end, s0 + ENV_WIN));
36 // Short-term RMS envelope in dB.
37 const nEnv = Math.max(1, Math.floor((seg.length - ENV_WIN) / ENV_HOP) + 1);
38 const envDb = new Float32Array(nEnv);
39 let peakDb = -160, peakI = 0;
40 for (let i = 0; i < nEnv; i++) {
41 const off = i * ENV_HOP;
42 let e = 0;
43 for (let j = 0; j < ENV_WIN && off + j < seg.length; j++) e += seg[off + j] * seg[off + j];
44 envDb[i] = 10 * Math.log10(e / ENV_WIN + EPS);
45 if (envDb[i] > peakDb) { peakDb = envDb[i]; peakI = i; }
46 }
47 const attackMs = (peakI * ENV_HOP * 1000) / sr;
48 let decayI = nEnv - 1;
49 for (let i = peakI; i < nEnv; i++) if (envDb[i] < peakDb - 25) { decayI = i; break; }
50 const decayMs = ((decayI - peakI) * ENV_HOP * 1000) / sr;
52 // Spectral shape of the attack portion (first ~85 ms), one padded FFT.
53 const spec = powerSpectrum(seg.subarray(0, Math.min(seg.length, SPEC_N)), SPEC_N);
54 const hzPerBin = sr / SPEC_N;
55 const kMax = Math.min(spec.length, Math.floor(9000 / hzPerBin));
56 let total = 0, weighted = 0;
57 for (let k = 1; k < kMax; k++) { total += spec[k]; weighted += spec[k] * k * hzPerBin; }
58 const centroid = total > 0 ? weighted / total : 0;
59 let acc = 0, rolloff = 0;
60 for (let k = 1; k < kMax; k++) { acc += spec[k]; if (acc >= 0.85 * total) { rolloff = k * hzPerBin; break; } }
62 // Band energies as log ratios with the mean removed, so overall level
63 // cancels and only the balance between bands remains.
64 const edges = [0, 200, 800, 2500, 9000];
65 const bands: number[] = [];
66 for (let b = 0; b < 4; b++) {
67 const k0 = Math.max(1, Math.floor(edges[b] / hzPerBin));
68 const k1 = Math.min(spec.length, Math.floor(edges[b + 1] / hzPerBin));
69 let e = 0;
70 for (let k = k0; k < k1; k++) e += spec[k];
71 bands.push(Math.log10(e + EPS));
72 }
73 const bandMean = bands.reduce((a, v) => a + v, 0) / 4;
74 for (let b = 0; b < 4; b++) bands[b] -= bandMean;
76 // Zero-crossing rate over the first 50 ms, in crossings per ms (~kHz).
77 const zN = Math.min(seg.length, Math.floor(0.05 * sr));
78 let zc = 0;
79 for (let i = 1; i < zN; i++) if ((seg[i] >= 0) !== (seg[i - 1] >= 0)) zc++;
80 const zcr = zN > 1 ? zc / ((zN * 1000) / sr) : 0;
82 return {
83 peakDb,
84 vec: [
85 Math.log10(centroid + 1), Math.log10(rolloff + 1),
86 bands[0], bands[1], bands[2], bands[3],
87 zcr, Math.log10(attackMs + 1), Math.log10(decayMs + 1),
88 ],
89 };
90}