diff --git a/src/arch/canary/model.cpp b/src/arch/canary/model.cpp index cdaf3ac6..8f4d57cd 100644 --- a/src/arch/canary/model.cpp +++ b/src/arch/canary/model.cpp @@ -791,6 +791,14 @@ transcribe_status run(transcribe_session * session, return mst; } cc->t_mel_us = ggml_time_us() - t_mel_start; + // Dump the full frontend output (what NeMo's preprocessor returns) + // before the padding frame is dropped. + if (transcribe::debug::enabled()) { + const long long shape[2] = { mel_n_mels, mel_n_frames }; + transcribe::debug::dump_host_f32("enc.mel.in", cc->mel_buf.data(), static_cast(cc->mel_buf.size()), + shape, 2, "encoder.mel"); + } + mel_n_frames = cm->mel->drop_nemo_padding_frame(cc->mel_buf, mel_n_frames, static_cast(n_samples)); // Input-length gate: T_enc must stay within enc_pos_emb_max_len or the // runtime pos table aliases past the trained range. T_enc is a @@ -859,7 +867,6 @@ transcribe_status run(transcribe_session * session, } ggml_backend_tensor_set(eb.mel_in, cc->mel_buf.data(), 0, cc->mel_buf.size() * sizeof(float)); - transcribe::debug::dump_tensor("enc.mel.in", eb.mel_in, "encoder.mel"); // Sinusoidal relative positional embedding. if (eb.pos_emb_in != nullptr) { @@ -1669,7 +1676,7 @@ transcribe_status run_batch(transcribe_session * session, if (cm->mel->compute(pcm[b], static_cast(n_samples[b]), mel_bufs[b], nm, nf, /*n_threads=*/1) == TRANSCRIBE_OK && nf > 0) { - mel_nf[b] = nf; + mel_nf[b] = cm->mel->drop_nemo_padding_frame(mel_bufs[b], nf, static_cast(n_samples[b])); valid[b] = 1; } return true; diff --git a/src/arch/canary_qwen/model.cpp b/src/arch/canary_qwen/model.cpp index 5b00b406..5eb31f0d 100644 --- a/src/arch/canary_qwen/model.cpp +++ b/src/arch/canary_qwen/model.cpp @@ -794,6 +794,14 @@ transcribe_status run(transcribe_session * context, return mst; } cc->t_mel_us = ggml_time_us() - t_mel_start; + // Dump the full frontend output (what NeMo's preprocessor returns) + // before the padding frame is dropped. + if (transcribe::debug::enabled()) { + const long long shape[2] = { mel_n_mels, mel_n_frames }; + transcribe::debug::dump_host_f32("enc.mel.in", cc->mel_buf.data(), static_cast(cc->mel_buf.size()), + shape, 2, "frontend.mel.norm"); + } + mel_n_frames = cm->mel->drop_nemo_padding_frame(cc->mel_buf, mel_n_frames, static_cast(n_samples)); // Encoder. if (cc->compute_ctx != nullptr) { @@ -835,11 +843,6 @@ transcribe_status run(transcribe_session * context, } ggml_backend_tensor_set(eb.mel_in, cc->mel_buf.data(), 0, cc->mel_buf.size() * sizeof(float)); - if (transcribe::debug::enabled()) { - const long long shape[2] = { mel_n_mels, mel_n_frames }; - transcribe::debug::dump_host_f32("enc.mel.in", cc->mel_buf.data(), static_cast(cc->mel_buf.size()), - shape, 2, "frontend.mel.norm"); - } // Upload relative-position embedding. if (eb.pos_emb_in != nullptr) { @@ -1411,7 +1414,7 @@ transcribe_status run_batch(transcribe_session * session, int nm = 0, nf = 0; if (cm->mel->compute(pcm[b], static_cast(n_samples[b]), mel_bufs[b], nm, nf, /*n_threads=*/1) == TRANSCRIBE_OK) { - mel_nf[b] = nf; + mel_nf[b] = cm->mel->drop_nemo_padding_frame(mel_bufs[b], nf, static_cast(n_samples[b])); } return true; }); diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index a877781e..81be6f20 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -1060,6 +1060,17 @@ transcribe_status run_one_shot_inner(ParakeetSession * pc, return mst; } pc->t_mel_us = ggml_time_us() - t_mel_start; + // Dump the full frontend output (what NeMo's preprocessor returns) + // before the padding frame is dropped. + if (transcribe::debug::enabled()) { + const long long shape[2] = { mel_n_mels, mel_n_frames }; + transcribe::debug::dump_host_f32("enc.mel.in", pc->mel_buf.data(), static_cast(pc->mel_buf.size()), + shape, 2, "encoder.mel"); + } + // Kestrel-masked checkpoints mask the padding frame in-graph instead. + if (!pm->hparams.kestrel_length_masking) { + mel_n_frames = pm->mel->drop_nemo_padding_frame(pc->mel_buf, mel_n_frames, static_cast(n_samples)); + } // Multitalker masked mode (NeMo mask_features): binarize the target // speaker's per-diar-frame activity, expand 8x to feature frames from @@ -1170,8 +1181,6 @@ transcribe_status run_one_shot_inner(ParakeetSession * pc, // byte-identical to the ggml ne=[n_frames, num_mels, 1, 1] input. ggml_backend_tensor_set(eb.mel_in, pc->mel_buf.data(), 0, pc->mel_buf.size() * sizeof(float)); - transcribe::debug::dump_tensor("enc.mel.in", eb.mel_in, "encoder.mel"); - int T_valid = -1; if (kestrel_masks) { std::vector real_tenc; @@ -2114,7 +2123,9 @@ transcribe_status run_batch(transcribe_session * session, if (st != TRANSCRIBE_OK || this_frames <= 0) { return false; } - nf[i] = this_frames; + nf[i] = pm->hparams.kestrel_length_masking ? + this_frames : + pm->mel->drop_nemo_padding_frame(mels[i], this_frames, static_cast(n_samples[i])); n_mels_per[i] = this_mels; return true; }); diff --git a/src/transcribe-mel.cpp b/src/transcribe-mel.cpp index af4e23ca..811c2d7b 100644 --- a/src/transcribe-mel.cpp +++ b/src/transcribe-mel.cpp @@ -373,6 +373,20 @@ int MelFrontend::n_frames_for(size_t n_samples) const { return static_cast(n_samples / static_cast(cfg_.hop_length)) + 1; } +int MelFrontend::drop_nemo_padding_frame(std::vector & mel, int n_frames, size_t n_samples) const { + const int valid = static_cast(n_samples / static_cast(cfg_.hop_length)); + if (valid >= n_frames) { + return n_frames; + } + const size_t n_mels = static_cast(cfg_.num_mels); + for (size_t m = 1; m < n_mels; ++m) { + std::memmove(mel.data() + m * static_cast(valid), mel.data() + m * static_cast(n_frames), + static_cast(valid) * sizeof(float)); + } + mel.resize(n_mels * static_cast(valid)); + return valid; +} + transcribe_status MelFrontend::compute(const float * pcm, size_t n_samples, std::vector & out_mel, diff --git a/src/transcribe-mel.h b/src/transcribe-mel.h index 49abaef9..57541fbf 100644 --- a/src/transcribe-mel.h +++ b/src/transcribe-mel.h @@ -173,6 +173,20 @@ class MelFrontend { // Matches NeMo: floor(n_samples / hop_length) + 1. int n_frames_for(size_t n_samples) const; + // Drop the trailing center-pad frame NeMo does not count (#252). + // + // The centered STFT emits floor(n / hop) + 1 frames, but NeMo's + // get_seq_len reports floor(n / hop) and masks everything past it in + // pre_encode, attention and the conv module. Masking a conv input to + // zero past its valid length equals truncating it there, so NeMo's valid + // encoder frames are what the encoder computes on the first + // floor(n / hop) mel frames. Fed unmasked, the padding frame becomes an + // extra encoder frame whenever floor(n / hop) is a multiple of 8. + // + // Truncates compute()'s row-major [num_mels, n_frames] buffer in place + // and returns the new frame count. + int drop_nemo_padding_frame(std::vector & mel, int n_frames, size_t n_samples) const; + // Read-only accessors for unit tests. Not part of the runtime // API; the goal is to validate the precomputed buffers in // isolation without running the full pipeline. diff --git a/tests/mel_unit.cpp b/tests/mel_unit.cpp index 218785b7..5bf31135 100644 --- a/tests/mel_unit.cpp +++ b/tests/mel_unit.cpp @@ -213,6 +213,40 @@ void test_n_frames_for() { CHECK(mf.n_frames_for(0) == 1); } +// #252: the frontend emits floor(n / hop) + 1 frames, NeMo counts +// floor(n / hop). The trailing frame of every mel row must go and the +// row-major layout must survive the in-place compaction. +void test_drop_nemo_padding_frame() { + const transcribe::MelFrontend mf(parakeet_config()); + const int n_mels = mf.num_mels(); + // The #252 clips (80640, 168960 samples) and unaligned neighbours. + for (const size_t n : { size_t{ 80640 }, size_t{ 80639 }, size_t{ 168960 }, size_t{ 160 }, size_t{ 1601 } }) { + const int n_frames = mf.n_frames_for(n); + const int valid = n_frames - 1; + std::vector mel(static_cast(n_mels) * static_cast(n_frames)); + for (int m = 0; m < n_mels; ++m) { + for (int t = 0; t < n_frames; ++t) { + mel[static_cast(m) * static_cast(n_frames) + static_cast(t)] = + static_cast(m * 10000 + t); + } + } + CHECK(mf.drop_nemo_padding_frame(mel, n_frames, n) == valid); + CHECK(mel.size() == static_cast(n_mels) * static_cast(valid)); + bool layout_ok = true; + for (int m = 0; m < n_mels; ++m) { + for (int t = 0; t < valid; ++t) { + layout_ok = + layout_ok && mel[static_cast(m) * static_cast(valid) + static_cast(t)] == + static_cast(m * 10000 + t); + } + } + CHECK(layout_ok); + // Already at the valid length: untouched. + CHECK(mf.drop_nemo_padding_frame(mel, valid, n) == valid); + CHECK(mel.size() == static_cast(n_mels) * static_cast(valid)); + } +} + // SpeechBrain (ecapa_tdnn) config: periodic Hamming, constant pad, no // pre-emphasis, 10*log10 with an 80 dB top-dB floor, per-bin mean only. // The filterbank is a synthetic one-hot bank (mel m reads FFT bin 3*m), @@ -317,6 +351,7 @@ int main() { test_window(); test_mel_filterbank(); test_n_frames_for(); + test_drop_nemo_padding_frame(); test_hamming_window(); test_sentence_mean();