mel: drop the NeMo padding frame before the encoder (parakeet, canary, canary-qwen) (#252)

src/arch/canary/model.cpp

@@ -791,6 +791,14 @@ transcribe_status run(transcribe_session *          session,
         return mst;
     }
     cc->t_mel_us = ggml_time_us() - t_mel_start;
+    // Dump the full frontend output (what NeMo's preprocessor returns)
+    // before the padding frame is dropped.
+    if (transcribe::debug::enabled()) {
+        const long long shape[2] = { mel_n_mels, mel_n_frames };
+        transcribe::debug::dump_host_f32("enc.mel.in", cc->mel_buf.data(), static_cast<long long>(cc->mel_buf.size()),
+                                         shape, 2, "encoder.mel");
+    }
+    mel_n_frames = cm->mel->drop_nemo_padding_frame(cc->mel_buf, mel_n_frames, static_cast<size_t>(n_samples));
 
     // Input-length gate: T_enc must stay within enc_pos_emb_max_len or the
     // runtime pos table aliases past the trained range. T_enc is a
@@ -859,7 +867,6 @@ transcribe_status run(transcribe_session *          session,
     }
 
     ggml_backend_tensor_set(eb.mel_in, cc->mel_buf.data(), 0, cc->mel_buf.size() * sizeof(float));
-    transcribe::debug::dump_tensor("enc.mel.in", eb.mel_in, "encoder.mel");
 
     // Sinusoidal relative positional embedding.
     if (eb.pos_emb_in != nullptr) {
@@ -1669,7 +1676,7 @@ transcribe_status run_batch(transcribe_session *          session,
         if (cm->mel->compute(pcm[b], static_cast<size_t>(n_samples[b]), mel_bufs[b], nm, nf, /*n_threads=*/1) ==
                 TRANSCRIBE_OK &&
             nf > 0) {
-            mel_nf[b] = nf;
+            mel_nf[b] = cm->mel->drop_nemo_padding_frame(mel_bufs[b], nf, static_cast<size_t>(n_samples[b]));
             valid[b]  = 1;
         }
         return true;

src/arch/canary_qwen/model.cpp

@@ -794,6 +794,14 @@ transcribe_status run(transcribe_session *          context,
         return mst;
     }
     cc->t_mel_us = ggml_time_us() - t_mel_start;
+    // Dump the full frontend output (what NeMo's preprocessor returns)
+    // before the padding frame is dropped.
+    if (transcribe::debug::enabled()) {
+        const long long shape[2] = { mel_n_mels, mel_n_frames };
+        transcribe::debug::dump_host_f32("enc.mel.in", cc->mel_buf.data(), static_cast<long long>(cc->mel_buf.size()),
+                                         shape, 2, "frontend.mel.norm");
+    }
+    mel_n_frames = cm->mel->drop_nemo_padding_frame(cc->mel_buf, mel_n_frames, static_cast<size_t>(n_samples));
 
     // Encoder.
     if (cc->compute_ctx != nullptr) {
@@ -835,11 +843,6 @@ transcribe_status run(transcribe_session *          context,
     }
 
     ggml_backend_tensor_set(eb.mel_in, cc->mel_buf.data(), 0, cc->mel_buf.size() * sizeof(float));
-    if (transcribe::debug::enabled()) {
-        const long long shape[2] = { mel_n_mels, mel_n_frames };
-        transcribe::debug::dump_host_f32("enc.mel.in", cc->mel_buf.data(), static_cast<long long>(cc->mel_buf.size()),
-                                         shape, 2, "frontend.mel.norm");
-    }
 
     // Upload relative-position embedding.
     if (eb.pos_emb_in != nullptr) {
@@ -1411,7 +1414,7 @@ transcribe_status run_batch(transcribe_session *          session,
         int nm = 0, nf = 0;
         if (cm->mel->compute(pcm[b], static_cast<size_t>(n_samples[b]), mel_bufs[b], nm, nf, /*n_threads=*/1) ==
             TRANSCRIBE_OK) {
-            mel_nf[b] = nf;
+            mel_nf[b] = cm->mel->drop_nemo_padding_frame(mel_bufs[b], nf, static_cast<size_t>(n_samples[b]));
         }
         return true;
     });

src/arch/parakeet/model.cpp

@@ -1060,6 +1060,17 @@ transcribe_status run_one_shot_inner(ParakeetSession *             pc,
         return mst;
     }
     pc->t_mel_us = ggml_time_us() - t_mel_start;
+    // Dump the full frontend output (what NeMo's preprocessor returns)
+    // before the padding frame is dropped.
+    if (transcribe::debug::enabled()) {
+        const long long shape[2] = { mel_n_mels, mel_n_frames };
+        transcribe::debug::dump_host_f32("enc.mel.in", pc->mel_buf.data(), static_cast<long long>(pc->mel_buf.size()),
+                                         shape, 2, "encoder.mel");
+    }
+    // Kestrel-masked checkpoints mask the padding frame in-graph instead.
+    if (!pm->hparams.kestrel_length_masking) {
+        mel_n_frames = pm->mel->drop_nemo_padding_frame(pc->mel_buf, mel_n_frames, static_cast<size_t>(n_samples));
+    }
 
     // Multitalker masked mode (NeMo mask_features): binarize the target
     // speaker's per-diar-frame activity, expand 8x to feature frames from
@@ -1170,8 +1181,6 @@ transcribe_status run_one_shot_inner(ParakeetSession *             pc,
     // byte-identical to the ggml ne=[n_frames, num_mels, 1, 1] input.
     ggml_backend_tensor_set(eb.mel_in, pc->mel_buf.data(), 0, pc->mel_buf.size() * sizeof(float));
 
-    transcribe::debug::dump_tensor("enc.mel.in", eb.mel_in, "encoder.mel");
-
     int T_valid = -1;
     if (kestrel_masks) {
         std::vector<int> real_tenc;
@@ -2114,7 +2123,9 @@ transcribe_status run_batch(transcribe_session *          session,
         if (st != TRANSCRIBE_OK || this_frames <= 0) {
             return false;
         }
-        nf[i]         = this_frames;
+        nf[i]         = pm->hparams.kestrel_length_masking ?
+                            this_frames :
+                            pm->mel->drop_nemo_padding_frame(mels[i], this_frames, static_cast<size_t>(n_samples[i]));
         n_mels_per[i] = this_mels;
         return true;
     });

src/transcribe-mel.cpp

@@ -373,6 +373,20 @@ int MelFrontend::n_frames_for(size_t n_samples) const {
     return static_cast<int>(n_samples / static_cast<size_t>(cfg_.hop_length)) + 1;
 }
 
+int MelFrontend::drop_nemo_padding_frame(std::vector<float> & mel, int n_frames, size_t n_samples) const {
+    const int valid = static_cast<int>(n_samples / static_cast<size_t>(cfg_.hop_length));
+    if (valid >= n_frames) {
+        return n_frames;
+    }
+    const size_t n_mels = static_cast<size_t>(cfg_.num_mels);
+    for (size_t m = 1; m < n_mels; ++m) {
+        std::memmove(mel.data() + m * static_cast<size_t>(valid), mel.data() + m * static_cast<size_t>(n_frames),
+                     static_cast<size_t>(valid) * sizeof(float));
+    }
+    mel.resize(n_mels * static_cast<size_t>(valid));
+    return valid;
+}
+
 transcribe_status MelFrontend::compute(const float *        pcm,
                                        size_t               n_samples,
                                        std::vector<float> & out_mel,

src/transcribe-mel.h

@@ -173,6 +173,20 @@ class MelFrontend {
     // Matches NeMo: floor(n_samples / hop_length) + 1.
     int n_frames_for(size_t n_samples) const;
 
+    // Drop the trailing center-pad frame NeMo does not count (#252).
+    //
+    // The centered STFT emits floor(n / hop) + 1 frames, but NeMo's
+    // get_seq_len reports floor(n / hop) and masks everything past it in
+    // pre_encode, attention and the conv module. Masking a conv input to
+    // zero past its valid length equals truncating it there, so NeMo's valid
+    // encoder frames are what the encoder computes on the first
+    // floor(n / hop) mel frames. Fed unmasked, the padding frame becomes an
+    // extra encoder frame whenever floor(n / hop) is a multiple of 8.
+    //
+    // Truncates compute()'s row-major [num_mels, n_frames] buffer in place
+    // and returns the new frame count.
+    int drop_nemo_padding_frame(std::vector<float> & mel, int n_frames, size_t n_samples) const;
+
     // Read-only accessors for unit tests. Not part of the runtime
     // API; the goal is to validate the precomputed buffers in
     // isolation without running the full pipeline.

tests/mel_unit.cpp

@@ -213,6 +213,40 @@ void test_n_frames_for() {
     CHECK(mf.n_frames_for(0) == 1);
 }
 
+// #252: the frontend emits floor(n / hop) + 1 frames, NeMo counts
+// floor(n / hop). The trailing frame of every mel row must go and the
+// row-major layout must survive the in-place compaction.
+void test_drop_nemo_padding_frame() {
+    const transcribe::MelFrontend mf(parakeet_config());
+    const int                     n_mels = mf.num_mels();
+    // The #252 clips (80640, 168960 samples) and unaligned neighbours.
+    for (const size_t n : { size_t{ 80640 }, size_t{ 80639 }, size_t{ 168960 }, size_t{ 160 }, size_t{ 1601 } }) {
+        const int          n_frames = mf.n_frames_for(n);
+        const int          valid    = n_frames - 1;
+        std::vector<float> mel(static_cast<size_t>(n_mels) * static_cast<size_t>(n_frames));
+        for (int m = 0; m < n_mels; ++m) {
+            for (int t = 0; t < n_frames; ++t) {
+                mel[static_cast<size_t>(m) * static_cast<size_t>(n_frames) + static_cast<size_t>(t)] =
+                    static_cast<float>(m * 10000 + t);
+            }
+        }
+        CHECK(mf.drop_nemo_padding_frame(mel, n_frames, n) == valid);
+        CHECK(mel.size() == static_cast<size_t>(n_mels) * static_cast<size_t>(valid));
+        bool layout_ok = true;
+        for (int m = 0; m < n_mels; ++m) {
+            for (int t = 0; t < valid; ++t) {
+                layout_ok =
+                    layout_ok && mel[static_cast<size_t>(m) * static_cast<size_t>(valid) + static_cast<size_t>(t)] ==
+                                     static_cast<float>(m * 10000 + t);
+            }
+        }
+        CHECK(layout_ok);
+        // Already at the valid length: untouched.
+        CHECK(mf.drop_nemo_padding_frame(mel, valid, n) == valid);
+        CHECK(mel.size() == static_cast<size_t>(n_mels) * static_cast<size_t>(valid));
+    }
+}
+
 // SpeechBrain (ecapa_tdnn) config: periodic Hamming, constant pad, no
 // pre-emphasis, 10*log10 with an 80 dB top-dB floor, per-bin mean only.
 // The filterbank is a synthetic one-hot bank (mel m reads FFT bin 3*m),
@@ -317,6 +351,7 @@ int main() {
     test_window();
     test_mel_filterbank();
     test_n_frames_for();
+    test_drop_nemo_padding_frame();
     test_hamming_window();
     test_sentence_mean();