diff --git a/BENCHMARK_RESULTS.md b/BENCHMARK_RESULTS.md index 3f02d5e..1d7db65 100644 --- a/BENCHMARK_RESULTS.md +++ b/BENCHMARK_RESULTS.md @@ -97,6 +97,36 @@ Comprehensive benchmark results for eddy ASR on LibriSpeech test-clean and FLEUR --- +## Nemotron Streaming Multilingual 0.6B (FLEURS) + +**Model**: `nemotron-streaming-int8` (weight-only INT8 encoder, FP16 decoder/joint) +**Device**: Intel NPU · **Software**: OpenVINO 2025.0 · **Decoding**: greedy, forced language +**Preprocessor**: native C++ log-mel featurizer (replaces the dynamic-shape OV preprocessor) +**Scoring**: FluidAudio methodology — WER for spaced languages, character-level CER for CJK +(`ja`/`zh`), Whisper-style punctuation/symbol stripping + +| Language | Metric | eddy NPU (int8) | OpenVINO FP32 ref | FluidAudio CoreML ref | RTFx | Samples | +|----------|--------|----------------:|------------------:|----------------------:|-----:|--------:| +| English (US) | WER | 12.48 | 11.78 | 12.09 | 22.3× | 350 | +| Spanish (LatAm) | WER | 7.03 | 6.99 | 9.01 | 22.5× | 350 | +| French (France) | WER | 13.35 | 12.92 | 15.18 | 21.7× | 350 | +| Chinese (Mandarin) | CER | 20.18 | 21.05 | 24.54 | 23.7× | 945 | +| Japanese | CER | 15.46 | 15.12 | 16.86 | 23.3× | 650 | + +**Audio-weighted RTFx**: ~22–24× on Intel NPU (≈2× the CPU figure of ~11×). + +**Notes**: +- INT8-on-NPU accuracy matches the OpenVINO FP32 reference within noise and beats the + FluidAudio CoreML reference on es/fr/zh/ja. +- **NPU enablement**: the OpenVINO NPU plugin miscompiles `BitwiseNot` on a boolean + (integer complement → mask all-true → encoder collapses to ~0 → empty transcripts). + eddy rewrites `BitwiseNot → LogicalNot` in the encoder IR before compiling (no-op on + CPU/GPU); without it the NPU produces empty output for this model. +- Unlike Parakeet (overlapping-chunk + 2D dedup), Nemotron is cache-aware streaming RNNT + with per-chunk `prompt_id` language conditioning. + +--- + ## Performance Notes ### Best Performing Languages (WER < 10%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c55c96..85614bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,10 @@ target_sources(eddy src/models/parakeet-v2/parakeet_decoder.cpp src/models/parakeet-v2/parakeet_chunking.cpp src/models/parakeet-v2/tokenizer.cpp + + # Nemotron cache-aware streaming implementation + src/models/nemotron/nemotron_openvino.cpp + src/models/nemotron/nemotron_featurizer.cpp ) # Link dependencies diff --git a/claude.md b/claude.md index 1c5a245..1b05d8a 100644 --- a/claude.md +++ b/claude.md @@ -28,12 +28,17 @@ Match FluidAudio's Parakeet v2 (Swift/CoreML) implementation in C++/OpenVINO: ## Architecture -**Batch Chunking** (current focus): +Two first-class ASR paths are supported: + +**Batch (Parakeet)** — `eddy::parakeet`, stateless overlapping-chunk encoder: - 10s chunks with 3s overlap - 2D search deduplication at boundaries - LSTM state continuity across chunks -**NOT building streaming yet** - focus is on batch processing of complete audio files. +**Streaming (Nemotron)** — `eddy::nemotron`, cache-aware streaming FastConformer-RNNT: +- Carries `cache_channel`/`cache_time`/`cache_len` across chunks (`att_context=[56,0]`) +- Integer `prompt_id` per-chunk language conditioning (40+ languages) +- Plain RNNT greedy decode (no TDT duration bins) ## Testing diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 4d8c615..a7cd1fa 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -7,10 +7,16 @@ endif() add_executable(parakeet_cli parakeet_cli.cpp) target_link_libraries(parakeet_cli PRIVATE eddy) +add_executable(nemotron_cli nemotron_cli.cpp) +target_link_libraries(nemotron_cli PRIVATE eddy) + add_executable(hf_fetch_models hf_fetch_models.cpp) target_link_libraries(hf_fetch_models PRIVATE eddy) add_executable(benchmark_fleurs benchmark_fleurs.cpp) target_link_libraries(benchmark_fleurs PRIVATE eddy) -install(TARGETS parakeet_cli hf_fetch_models benchmark_fleurs DESTINATION bin) +add_executable(benchmark_nemotron_fleurs benchmark_nemotron_fleurs.cpp) +target_link_libraries(benchmark_nemotron_fleurs PRIVATE eddy) + +install(TARGETS parakeet_cli nemotron_cli hf_fetch_models benchmark_fleurs benchmark_nemotron_fleurs DESTINATION bin) diff --git a/examples/cpp/benchmark_fleurs.cpp b/examples/cpp/benchmark_fleurs.cpp index dc51b0c..fa68192 100644 --- a/examples/cpp/benchmark_fleurs.cpp +++ b/examples/cpp/benchmark_fleurs.cpp @@ -507,7 +507,7 @@ int main(int argc, char* argv[]) { // Load Parakeet v3 models auto cache_model_dir = eddy::get_model_assets_dir("parakeet-v3"); std::string fetch_err; - if (!eddy::parakeet::check_models_available(cache_model_dir, &fetch_err)) { + if (!eddy::model_utils::check_models_available(cache_model_dir, &fetch_err)) { if (!fetch_err.empty()) std::cout << "[INFO] " << fetch_err << "\n"; } diff --git a/examples/cpp/benchmark_nemotron_fleurs.cpp b/examples/cpp/benchmark_nemotron_fleurs.cpp new file mode 100644 index 0000000..97c6437 --- /dev/null +++ b/examples/cpp/benchmark_nemotron_fleurs.cpp @@ -0,0 +1,379 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 +// +// FLEURS Multilingual ASR Benchmark for the Nemotron streaming backend. +// +// Mirrors benchmark_fleurs.cpp (Parakeet) but drives eddy::nemotron:: +// OpenVINONemotron with per-language prompt conditioning. Loads the model once +// per language (language is fixed at handle construction), loops the FLEURS +// split, and reports WER / CER / RTFx. +// +// Usage: +// benchmark_nemotron_fleurs.exe --languages en_us,es_419,fr_fr \ +// --samples 0 --device NPU --model-dir --output results.json + +#include "eddy/backends/openvino_backend.hpp" +#include "eddy/core/app_dir.hpp" +#include "eddy/models/nemotron/nemotron.hpp" +#include "eddy/utils/audio_utils.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +// FLEURS code -> human name (subset; only used for display). +const std::map LANG_NAMES = { + {"en_us", "English (US)"}, {"es_419", "Spanish (LatAm)"}, {"fr_fr", "French (France)"}, + {"de_de", "German (Germany)"}, {"it_it", "Italian (Italy)"}, {"ru_ru", "Russian (Russia)"}, + {"nl_nl", "Dutch"}, {"pl_pl", "Polish"}, {"uk_ua", "Ukrainian"}, + {"sk_sk", "Slovak"}, {"cs_cz", "Czech"}, {"bg_bg", "Bulgarian"}, + {"hr_hr", "Croatian"}, {"ro_ro", "Romanian"}, {"fi_fi", "Finnish"}, + {"hu_hu", "Hungarian"}, {"sv_se", "Swedish"}, {"et_ee", "Estonian"}, + {"da_dk", "Danish"}, {"lt_lt", "Lithuanian"}, {"el_gr", "Greek"}, + {"mt_mt", "Maltese"}, {"lv_lv", "Latvian"}, {"sl_si", "Slovenian"}, + {"cmn_hans_cn", "Chinese (Mandarin)"}, {"ja_jp", "Japanese"}, +}; + +// Map a FLEURS code ("en_us") to a Nemotron prompt-dictionary tag ("en-US"). +// es_419 (Latin-American Spanish) is special-cased to es-US; everything else is +// the generic "xx_yy" -> "xx-YY". Unknown tags fall back inside the backend +// (2-letter, then "auto"). +std::string fleurs_to_nemotron_lang(const std::string& code) { + if (code == "es_419") return "es-US"; + if (code == "cmn_hans_cn") return "zh-CN"; // FLEURS Mandarin -> Nemotron zh-CN + if (code == "ja_jp") return "ja-JP"; + auto us = code.find('_'); + if (us == std::string::npos) return code; + std::string lang = code.substr(0, us); + std::string region = code.substr(us + 1); + for (char& c : region) c = static_cast(std::toupper(static_cast(c))); + return lang + "-" + region; +} + +// Split a UTF-8 string into codepoint substrings (1-4 bytes each). +std::vector utf8_chars(const std::string& s) { + std::vector cps; + size_t i = 0; + while (i < s.size()) { + unsigned char c = static_cast(s[i]); + size_t len = (c < 0x80) ? 1 : ((c >> 5) == 0x6) ? 2 : ((c >> 4) == 0xE) ? 3 + : ((c >> 3) == 0x1E) ? 4 : 1; + if (i + len > s.size()) len = 1; + cps.push_back(s.substr(i, len)); + i += len; + } + return cps; +} + +// Decode a 1-4 byte UTF-8 codepoint string to its Unicode scalar value. +uint32_t cp_scalar(const std::string& cp) { + unsigned char c0 = static_cast(cp[0]); + if (cp.size() == 1) return c0; + if (cp.size() == 2) return ((c0 & 0x1F) << 6) | (static_cast(cp[1]) & 0x3F); + if (cp.size() == 3) + return ((c0 & 0x0F) << 12) | ((static_cast(cp[1]) & 0x3F) << 6) | + (static_cast(cp[2]) & 0x3F); + return ((c0 & 0x07) << 18) | ((static_cast(cp[1]) & 0x3F) << 12) | + ((static_cast(cp[2]) & 0x3F) << 6) | (static_cast(cp[3]) & 0x3F); +} + +// Approximate the Whisper/FluidAudio normalizer's "replace every Mark/Symbol/ +// Punctuation (Unicode category M/S/P) with a space" step over the codepoint +// blocks that actually occur in FLEURS refs/hyps. Keeps letters (incl. CJK, +// kana, hangul, accented Latin, Greek, Cyrillic) and digits. +bool is_punct_or_symbol(uint32_t c) { + return (c >= 0x00A1 && c <= 0x00BF) || c == 0x00D7 || c == 0x00F7 || // Latin-1 punct/symbols, × ÷ + (c >= 0x2000 && c <= 0x206F) || // general punctuation – — ' ' " " … + (c >= 0x2070 && c <= 0x20CF) || // super/subscripts, currency symbols + (c >= 0x2100 && c <= 0x2BFF) || // letterlike/number forms, arrows, math, misc symbols + (c >= 0x3000 && c <= 0x303F) || // CJK symbols and punctuation 。、「」（） + (c >= 0xFF01 && c <= 0xFF0F) || // fullwidth ！＂＃…／ + (c >= 0xFF1A && c <= 0xFF20) || // fullwidth ：；＜＝＞？＠ + (c >= 0xFF3B && c <= 0xFF40) || // fullwidth ［＼］＾＿｀ + (c >= 0xFF5B && c <= 0xFF65); // fullwidth ｛｜｝、。etc +} + +// Normalize ~ FluidAudio's basicNormalize: lowercase ASCII, replace Unicode +// punctuation/symbols (M/S/P) with single spaces, keep letters/digits and +// diacritics/CJK, collapse whitespace. (Whisper's English number-word folding +// is intentionally omitted — "similar enough" per the multilingual path.) +std::string normalize_text(const std::string& text) { + std::string result; + result.reserve(text.size()); + bool last_space = false; + auto sep = [&]() { if (!last_space && !result.empty()) { result += ' '; last_space = true; } }; + for (const std::string& cp : utf8_chars(text)) { + if (cp.size() == 1) { + unsigned char c = static_cast(cp[0]); + if (std::isalnum(c)) { result += static_cast(std::tolower(c)); last_space = false; } + else sep(); + } else if (is_punct_or_symbol(cp_scalar(cp))) { + sep(); + } else { + result += cp; // letter / CJK / diacritic — keep + last_space = false; + } + } + if (!result.empty() && result.back() == ' ') result.pop_back(); + return result; +} + +// CJK / no-space scripts: word-level WER over whitespace tokens is meaningless, +// so FluidAudio routes these through character-level scoring (matches Whisper / +// ESPnet). FLEURS code prefixes. +bool is_cjk_lang(const std::string& code) { + auto p = [&](const char* s) { return code.rfind(s, 0) == 0; }; + return p("ja") || p("ko") || p("zh") || p("cmn") || p("yue") || p("th") || p("lo"); +} + +int levenshtein(const std::vector& a, const std::vector& b) { + const size_t m = a.size(), n = b.size(); + std::vector prev(n + 1), cur(n + 1); + for (size_t j = 0; j <= n; ++j) prev[j] = static_cast(j); + for (size_t i = 1; i <= m; ++i) { + cur[0] = static_cast(i); + for (size_t j = 1; j <= n; ++j) { + if (a[i - 1] == b[j - 1]) cur[j] = prev[j - 1]; + else cur[j] = 1 + std::min({prev[j], cur[j - 1], prev[j - 1]}); + } + std::swap(prev, cur); + } + return prev[n]; +} + +std::vector words(const std::string& s) { + std::vector w; + std::istringstream iss(s); + std::string t; + while (iss >> t) w.push_back(t); + return w; +} + +double wer(const std::string& ref, const std::string& hyp) { + auto r = words(normalize_text(ref)), h = words(normalize_text(hyp)); + if (r.empty()) return h.empty() ? 0.0 : 1.0; + return static_cast(levenshtein(r, h)) / r.size(); +} + +double cer(const std::string& ref, const std::string& hyp) { + // Character error rate over UTF-8 codepoints (spaces dropped). Codepoint- + // level, so CJK characters count as one token each. + std::vector r, h; + for (const auto& cp : utf8_chars(normalize_text(ref))) if (cp != " ") r.push_back(cp); + for (const auto& cp : utf8_chars(normalize_text(hyp))) if (cp != " ") h.push_back(cp); + if (r.empty()) return h.empty() ? 0.0 : 1.0; + return static_cast(levenshtein(r, h)) / r.size(); +} + +struct Sample { std::string id, audio_path, transcription; }; + +std::vector load_samples(const std::string& cache_dir, const std::string& lang, int max_samples) { + std::vector samples; + fs::path dir = fs::path(cache_dir) / lang; + if (!fs::exists(dir)) { + std::cerr << "Warning: language dir not found: " << dir << "\n"; + return samples; + } + std::map trans; + fs::path tf = dir / (lang + ".trans.txt"); + if (fs::exists(tf)) { + std::ifstream f(tf); + std::string line; + while (std::getline(f, line)) { + size_t sp = line.find(' '); + if (sp != std::string::npos) trans[line.substr(0, sp)] = line.substr(sp + 1); + } + } + std::vector wavs; + for (const auto& e : fs::directory_iterator(dir)) + if (e.path().extension() == ".wav") wavs.push_back(e.path()); + std::sort(wavs.begin(), wavs.end()); + if (max_samples > 0 && wavs.size() > static_cast(max_samples)) + wavs.resize(max_samples); + for (const auto& w : wavs) { + std::string id = w.stem().string(); + Sample s{id, w.string(), trans.count(id) ? trans[id] : ""}; + samples.push_back(s); + } + return samples; +} + +struct LangResult { + std::string lang, name; + double wer = 0, cer = 0, rtfx = 0, total_audio = 0, total_proc = 0; + int processed = 0, skipped = 0; +}; + +void print_usage(const char* p) { + std::cout << "Nemotron FLEURS Benchmark\n\nUsage: " << p << " [options]\n" + << " --languages Comma-separated FLEURS codes (default: en_us,es_419,fr_fr)\n" + << " --samples Max samples per language (0 = all; default 0)\n" + << " --device OpenVINO device CPU/NPU/GPU/AUTO (default CPU)\n" + << " --model-dir Dir with nemotron_*.xml/bin + metadata.json\n" + << " --output Output JSON (default nemotron_fleurs_results.json)\n" + << " --debug Per-file hypothesis/reference\n"; +} + +int main(int argc, char* argv[]) { + std::cout.setf(std::ios::unitbuf); + if (argc < 2) { print_usage(argv[0]); return 1; } + + std::string cache_dir = argv[1]; + std::vector langs = {"en_us", "es_419", "fr_fr"}; + int max_samples = 0; + std::string device = "CPU", model_dir, output = "nemotron_fleurs_results.json"; + bool debug = false; + + for (int i = 2; i < argc; ++i) { + std::string a = argv[i]; + auto next = [&](std::string& dst) { if (i + 1 < argc) dst = argv[++i]; }; + if (a == "--help" || a == "-h") { print_usage(argv[0]); return 0; } + else if (a == "--languages") { std::string v; next(v); langs.clear(); + std::istringstream ss(v); std::string t; while (std::getline(ss, t, ',')) langs.push_back(t); } + else if (a == "--samples") { std::string v; next(v); max_samples = std::stoi(v); } + else if (a == "--device") next(device); + else if (a == "--model-dir") next(model_dir); + else if (a == "--output") next(output); + else if (a == "--debug") debug = true; + } + + // Default model dir: %LOCALAPPDATA%/eddy/models/nemotron-streaming-int8/files + if (model_dir.empty()) { + const char* lad = std::getenv("LOCALAPPDATA"); + if (lad) model_dir = (fs::path(lad) / "eddy" / "models" / "nemotron-streaming-int8" / "files").string(); + } + + std::cout << "=== Nemotron FLEURS Benchmark ===\n"; + std::cout << "Cache: " << cache_dir << "\n"; + std::cout << "Model dir: " << model_dir << "\n"; + std::cout << "Device: " << device << "\n"; + std::cout << "Samples: " << (max_samples == 0 ? "all" : std::to_string(max_samples)) << " per language\n\n"; + + eddy::nemotron::ModelPaths paths{ + .preprocessor = (fs::path(model_dir) / "nemotron_preprocessor.xml").string(), + .encoder = (fs::path(model_dir) / "nemotron_encoder.xml").string(), + .decoder = (fs::path(model_dir) / "nemotron_decoder.xml").string(), + .joint = (fs::path(model_dir) / "nemotron_joint.xml").string(), + .vocab_json = (fs::path(model_dir) / "nemotron_vocab.json").string(), + .metadata_json = (fs::path(model_dir) / "metadata.json").string(), + }; + + eddy::OpenVINOOptions ov_opts; + ov_opts.device = device; + ov_opts.cache_dir = eddy::get_model_dir("nemotron-streaming-int8").string(); + auto backend = std::make_shared(ov_opts); + + std::vector results; + for (const auto& lang : langs) { + std::string tag = fleurs_to_nemotron_lang(lang); + std::cout << "Processing " << lang << " (prompt lang=" << tag << ")...\n"; + + auto samples = load_samples(cache_dir, lang, max_samples); + if (samples.empty()) { std::cerr << " no samples; skipping\n\n"; continue; } + std::cout << " Loaded " << samples.size() << " samples\n"; + + eddy::nemotron::Config cfg; + cfg.device = device; + cfg.language = tag; + + std::shared_ptr model; + try { + model = std::make_shared(backend, paths, cfg); + std::cout << " Compiling + warming up (" << device << ") ... "; + model->warmup(); + std::cout << "[OK]\n"; + } catch (const std::exception& e) { + std::cerr << " [ERROR] model init failed: " << e.what() << "\n\n"; + continue; + } + + LangResult lr; + lr.lang = lang; + lr.name = LANG_NAMES.count(lang) ? LANG_NAMES.at(lang) : lang; + const bool cjk = is_cjk_lang(lang); + double sum_wer = 0, sum_cer = 0; + + for (const auto& s : samples) { + try { + auto pcm = eddy::audio::read_wav(s.audio_path); + double audio_sec = pcm.size() / 16000.0; + auto t0 = std::chrono::high_resolution_clock::now(); + auto res = model->transcribe(pcm); + auto t1 = std::chrono::high_resolution_clock::now(); + double proc_sec = std::chrono::duration_cast(t1 - t0).count() / 1000.0; + + if (!s.transcription.empty()) { + // CJK: character-level rate reported in both WER and CER + // (FluidAudio convention — whitespace WER is meaningless). + double c = cer(s.transcription, res.text); + double w = cjk ? c : wer(s.transcription, res.text); + sum_wer += w; sum_cer += c; + if (debug) { + std::cout << " [" << s.id << "] WER=" << std::fixed << std::setprecision(1) << (w * 100) << "%\n" + << " hyp: " << res.text << "\n ref: " << s.transcription << "\n"; + } + } + lr.total_audio += audio_sec; + lr.total_proc += proc_sec; + lr.processed++; + } catch (const std::exception& e) { + std::cerr << " Warning: " << s.id << ": " << e.what() << "\n"; + lr.skipped++; + } + } + + if (lr.processed > 0) { + lr.wer = sum_wer / lr.processed; + lr.cer = sum_cer / lr.processed; + lr.rtfx = lr.total_proc > 0 ? lr.total_audio / lr.total_proc : 0; + } + results.push_back(lr); + std::cout << " " << lang << ": WER=" << std::fixed << std::setprecision(2) << (lr.wer * 100) + << "% CER=" << (lr.cer * 100) << "% RTFx=" << std::setprecision(2) << lr.rtfx + << "x (" << lr.processed << " processed, " << lr.skipped << " skipped)\n\n"; + } + + // JSON output + { + std::ofstream f(output); + f << "{\n \"benchmark\": \"FLEURS Nemotron streaming\",\n \"device\": \"" << device + << "\",\n \"model_dir\": \""; + for (char c : model_dir) { if (c == '\\') f << "\\\\"; else f << c; } + f << "\",\n \"results\": [\n"; + for (size_t i = 0; i < results.size(); ++i) { + const auto& r = results[i]; + f << " {\"language\": \"" << r.lang << "\", \"wer\": " << (r.wer * 100) + << ", \"cer\": " << (r.cer * 100) << ", \"rtfx\": " << r.rtfx + << ", \"processed\": " << r.processed << ", \"skipped\": " << r.skipped + << ", \"audioSec\": " << r.total_audio << ", \"procSec\": " << r.total_proc << "}"; + f << (i + 1 < results.size() ? ",\n" : "\n"); + } + f << " ]\n}\n"; + } + + std::cout << std::string(72, '=') << "\nSUMMARY (device=" << device << ")\n" << std::string(72, '=') << "\n"; + double tot_audio = 0, tot_proc = 0; + for (const auto& r : results) { + std::cout << std::left << std::setw(18) << r.name << " | WER=" << std::right << std::fixed + << std::setprecision(2) << std::setw(6) << (r.wer * 100) << "% CER=" << std::setw(6) + << (r.cer * 100) << "% RTFx=" << std::setw(5) << std::setprecision(2) << r.rtfx + << "x n=" << r.processed << "\n"; + tot_audio += r.total_audio; tot_proc += r.total_proc; + } + std::cout << std::string(72, '-') << "\nAudio-weighted RTFx: " << std::fixed << std::setprecision(2) + << (tot_proc > 0 ? tot_audio / tot_proc : 0) << "x (total audio " + << std::setprecision(0) << tot_audio << "s / proc " << tot_proc << "s)\n"; + std::cout << "Results saved to: " << output << "\n"; + return 0; +} diff --git a/examples/cpp/hf_fetch_models.cpp b/examples/cpp/hf_fetch_models.cpp index 108142c..d16b851 100644 --- a/examples/cpp/hf_fetch_models.cpp +++ b/examples/cpp/hf_fetch_models.cpp @@ -53,7 +53,9 @@ int main(int argc, char** argv) { auto it = MODEL_MAP.find(model_name); if (it == MODEL_MAP.end()) { std::cerr << "ERROR: Unknown model: " << model_name << "\n"; - std::cerr << "Available models: parakeet-v2\n"; + std::cerr << "Available models:"; + for (const auto& [k, _] : MODEL_MAP) std::cerr << " " << k; + std::cerr << "\n"; return 1; } @@ -85,7 +87,7 @@ int main(int argc, char** argv) { // Download models using library function std::string error_msg; - bool success = eddy::parakeet::download_models( + bool success = eddy::model_utils::download_models( config, fs::path(target_dir), &error_msg, diff --git a/examples/cpp/nemotron_cli.cpp b/examples/cpp/nemotron_cli.cpp new file mode 100644 index 0000000..4935964 --- /dev/null +++ b/examples/cpp/nemotron_cli.cpp @@ -0,0 +1,135 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 +// +// CLI for the NVIDIA Nemotron cache-aware streaming ASR backend: the +// 3.5-ASR-Streaming-Multilingual 0.6B model and the English speech-streaming +// 0.6B model (selected via --model). + +#include "eddy/backends/openvino_backend.hpp" +#include "eddy/core/app_dir.hpp" +#include "eddy/models/nemotron/nemotron.hpp" +#include "eddy/utils/audio_utils.hpp" + +#include +#include +#include +#include +#include + +void print_usage(const char* prog) { + std::cout << "Usage: " << prog << " [options]\n\n"; + std::cout << "Options:\n"; + std::cout << " --device OpenVINO device (default: CPU). CPU, AUTO, NPU\n"; + std::cout << " --lang Language: en-US, zh-CN, ... or auto (default: auto).\n"; + std::cout << " Ignored by the English speech-streaming model.\n"; + std::cout << " --model Model variant (selects the cache dir):\n"; + std::cout << " nemotron-streaming[-int8] multilingual (40+ langs)\n"; + std::cout << " nemotron-speech-streaming[-int8] English, no language prompt\n"; + std::cout << " Default: nemotron-streaming (FP16).\n"; + std::cout << " --model-dir Directory with nemotron_*.xml/bin + metadata.json\n"; + std::cout << " (overrides --model; default: cache for the --model variant)\n"; + std::cout << " --help Show this help\n"; +} + +int main(int argc, char* argv[]) { + std::cout.setf(std::ios::unitbuf); + if (argc < 2) { + print_usage(argv[0]); + return 1; + } + + std::string audio_file, device = "CPU", lang = "auto", model_dir_arg, + model_name = "nemotron-streaming"; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + // A flag that needs a value but is the last arg must error, not fall through + // to the positional branch (which would swallow the flag as the audio path). + auto take_value = [&](const char* flag, std::string& dst) -> bool { + if (i + 1 >= argc) { + std::cerr << "Error: " << flag << " requires an argument\n"; + return false; + } + dst = argv[++i]; + return true; + }; + if (a == "--help" || a == "-h") { + print_usage(argv[0]); + return 0; + } else if (a == "--device") { + if (!take_value("--device", device)) return 1; + } else if (a == "--lang") { + if (!take_value("--lang", lang)) return 1; + } else if (a == "--model") { + if (!take_value("--model", model_name)) return 1; + } else if (a == "--model-dir") { + if (!take_value("--model-dir", model_dir_arg)) return 1; + } else if (!a.empty() && a[0] == '-') { + std::cerr << "Error: unknown option " << a << "\n\n"; + print_usage(argv[0]); + return 1; + } else { + audio_file = a; + } + } + if (audio_file.empty()) { + std::cerr << "Error: no audio file specified\n\n"; + print_usage(argv[0]); + return 1; + } + + std::cout << "=== Nemotron ASR Streaming CLI ===\n\n"; + + try { + auto pcm = eddy::audio::read_wav(audio_file); + const float audio_seconds = static_cast(pcm.size()) / 16000.0f; + std::cout << "Audio: " << audio_file << " (" << std::fixed << std::setprecision(2) + << audio_seconds << "s)\n"; + + std::filesystem::path model_dir = + model_dir_arg.empty() ? eddy::get_model_assets_dir(model_name) + : std::filesystem::path(model_dir_arg); + std::cout << "Model: " << model_name << "\n"; + std::cout << "Models: " << model_dir.string() << "\n"; + + eddy::OpenVINOOptions ov_opts; + ov_opts.device = device; + ov_opts.cache_dir = eddy::get_model_dir(model_name).string(); + auto backend = std::make_shared(ov_opts); + + eddy::nemotron::ModelPaths paths{ + .preprocessor = (model_dir / "nemotron_preprocessor.xml").string(), + .encoder = (model_dir / "nemotron_encoder.xml").string(), + .decoder = (model_dir / "nemotron_decoder.xml").string(), + .joint = (model_dir / "nemotron_joint.xml").string(), + .vocab_json = (model_dir / "nemotron_vocab.json").string(), + .metadata_json = (model_dir / "metadata.json").string(), + }; + eddy::nemotron::Config cfg; + cfg.device = device; + cfg.language = lang; + + eddy::nemotron::OpenVINONemotron model(backend, paths, cfg); + std::cout << "Compiling + warming up (" << device << ") ... "; + model.warmup(); + std::cout << "[OK]\n\n"; + + std::cout << std::string(70, '=') << "\nTRANSCRIBING...\n" << std::string(70, '=') << "\n\n"; + const auto result = model.transcribe(pcm); + + const float rtfx = result.latency_ms > 0.0 + ? audio_seconds / static_cast(result.latency_ms / 1000.0) + : 0.0f; + + std::cout << "Result:\n" << std::string(70, '-') << "\n"; + std::cout << result.text << "\n" << std::string(70, '-') << "\n\n"; + std::cout << "prompt_id_used: " << result.prompt_id_used << "\n"; + std::cout << "detected_lang: " << (result.detected_language.empty() ? "(none)" : result.detected_language) << "\n"; + std::cout << "tokens: " << result.token_ids.size() << "\n"; + std::cout << "processing time: " << std::fixed << std::setprecision(0) << result.latency_ms << " ms\n"; + std::cout << "real-time factor:" << std::fixed << std::setprecision(1) << rtfx << "x\n"; + return 0; + } catch (const std::exception& e) { + std::cerr << "\n[ERROR] " << e.what() << "\n"; + return 1; + } +} diff --git a/examples/cpp/parakeet_cli.cpp b/examples/cpp/parakeet_cli.cpp index 70cd10e..021c5cb 100644 --- a/examples/cpp/parakeet_cli.cpp +++ b/examples/cpp/parakeet_cli.cpp @@ -109,7 +109,7 @@ int main(int argc, char* argv[]) { auto cache_model_dir = eddy::get_model_assets_dir(model_name); std::filesystem::path model_dir; std::string fetch_err; - if (!eddy::parakeet::check_models_available(cache_model_dir, &fetch_err)) { + if (!eddy::model_utils::check_models_available(cache_model_dir, &fetch_err)) { if (!fetch_err.empty()) std::cout << "[INFO] " << fetch_err << "\n"; } diff --git a/include/eddy/core/model_configs.hpp b/include/eddy/core/model_configs.hpp index 129cba6..09eedd7 100644 --- a/include/eddy/core/model_configs.hpp +++ b/include/eddy/core/model_configs.hpp @@ -14,6 +14,9 @@ struct ModelConfig { std::string repo_id; // HuggingFace repository ID (e.g., "org/model-name") std::vector required_files; // List of required model files (xml, bin, json) std::string cache_subdir; // Subdirectory name in cache (e.g., "parakeet-v2") + std::string repo_subdir; // Optional subfolder within the repo (e.g., "fp16"); + // files download from /resolve/main// + // but are stored flat in the cache. Empty => repo root. }; // Available model configurations (similar to FluidAudio's ModelNames.swift) @@ -41,10 +44,68 @@ namespace model_configs { .cache_subdir = "parakeet-v3" }; + // NVIDIA Nemotron-3.5-ASR-Streaming-Multilingual 0.6B (cache-aware + // streaming FastConformer-RNNT, prompt-conditioned multilingual). + // Distinct file set + metadata.json (cache shapes, prompt_dictionary, + // lang_tag_token_ids) consumed by the eddy::nemotron backend. The mel + // preprocessor is computed natively in C++ (eddy::nemotron::MelFeaturizer), + // so nemotron_preprocessor.xml/.bin are intentionally NOT required. + inline const std::vector NEMOTRON_FILES = { + "nemotron_encoder.xml", "nemotron_encoder.bin", + "nemotron_decoder.xml", "nemotron_decoder.bin", + "nemotron_joint.xml", "nemotron_joint.bin", + "nemotron_vocab.json", "metadata.json" + }; + + inline const ModelConfig NEMOTRON_STREAMING = { + .repo_id = "FluidInference/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b-ov", + .required_files = NEMOTRON_FILES, + .cache_subdir = "nemotron-streaming", + // FP16 IR (identical transcripts to FP32, ~half the size, NPU-friendly). + // FP32 also available under the "fp32" subfolder of the same repo. + .repo_subdir = "fp16" + }; + + // INT8 weight-only encoder (per-channel symmetric; Conformer relative-pos + // projections kept FP16) + FP16 decoder/joint/preprocessor. WER matches + // FP16/FP32 (en_us 10.99 vs 11.78); ~half the RAM of FP16 (2.1GB vs 3.9GB) + // and ~half the disk. No CPU speed gain (weights decompress to float on + // x86); the win is memory footprint — chiefly for Intel NPU / constrained + // deployments. Same repo, "int8" subfolder. + inline const ModelConfig NEMOTRON_STREAMING_INT8 = { + .repo_id = "FluidInference/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b-ov", + .required_files = NEMOTRON_FILES, + .cache_subdir = "nemotron-streaming-int8", + .repo_subdir = "int8" + }; + + // NVIDIA nemotron-speech-streaming-en-0.6b: the monolingual (English) sibling + // of the multilingual model. Same FastConformer cache-aware RNNT, but no + // prompt/language conditioning (the eddy backend auto-detects the absent + // encoder prompt_id input). Same flat file set as NEMOTRON_FILES. + // Shares the multilingual HF repo (no separate space) under "en/" subfolders. + inline const ModelConfig NEMOTRON_SPEECH = { + .repo_id = "FluidInference/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b-ov", + .required_files = NEMOTRON_FILES, + .cache_subdir = "nemotron-speech-streaming", + .repo_subdir = "en/fp16" + }; + + inline const ModelConfig NEMOTRON_SPEECH_INT8 = { + .repo_id = "FluidInference/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b-ov", + .required_files = NEMOTRON_FILES, + .cache_subdir = "nemotron-speech-streaming-int8", + .repo_subdir = "en/int8" + }; + // Model name lookup map inline const std::map MODEL_MAP = { {"parakeet-v2", PARAKEET_V2}, - {"parakeet-v3", PARAKEET_V3} + {"parakeet-v3", PARAKEET_V3}, + {"nemotron-streaming", NEMOTRON_STREAMING}, + {"nemotron-streaming-int8", NEMOTRON_STREAMING_INT8}, + {"nemotron-speech-streaming", NEMOTRON_SPEECH}, + {"nemotron-speech-streaming-int8", NEMOTRON_SPEECH_INT8} }; // Default model diff --git a/include/eddy/eddy_c.h b/include/eddy/eddy_c.h index dba5d6a..aec82da 100644 --- a/include/eddy/eddy_c.h +++ b/include/eddy/eddy_c.h @@ -226,6 +226,56 @@ EDDY_API EddyError eddy_parakeet_infer_buffer(EddyParakeetModel model, const flo EDDY_API char* eddy_parakeet_decode_tokens(EddyParakeetModel model, const int* token_ids, size_t count); EDDY_API void eddy_parakeet_free_result(EddyParakeetResult* result); +// ----------------------------- +// Nemotron streaming (OpenVINO) C API +// ----------------------------- + +// Opaque handle for a Nemotron streaming model. +typedef void* EddyNemotronModel; + +typedef struct { + const char* device; // "CPU", "NPU", or "AUTO" (default "CPU") + const char* model_dir; // Dir with nemotron_*.xml/bin + metadata.json. + // NULL or "cache" => Eddy cache for "nemotron-streaming". + const char* language; // "en-US", "zh-CN", ..., or "auto" (default "auto") +} EddyNemotronConfig; + +typedef struct { + char* text; // full transcript (lang-tag tokens stripped); free with eddy_nemotron_free_result + char* detected_language; // first tag emitted, or "" ; freed with the result + int* token_ids; // raw emitted token ids (pre-strip); must be freed with eddy_nemotron_free_result + size_t num_tokens; + int prompt_id_used; // integer prompt id selected for conditioning + double latency_ms; +} EddyNemotronResult; + +/** + * @brief Create a Nemotron streaming model. + * @param config Device / model_dir / language. Language conditions decoding and + * is fixed at creation; recreate the handle to change it. + * @param error_message Out param for error (can be NULL). Free with eddy_free_string. + * @return Model handle or NULL on failure. + */ +EDDY_API EddyNemotronModel eddy_nemotron_create(EddyNemotronConfig config, char** error_message); + +/** @brief Destroy a Nemotron model handle. */ +EDDY_API void eddy_nemotron_destroy(EddyNemotronModel model); + +/** + * @brief Transcribe a 16 kHz mono WAV file. + * @param result Out param; free with eddy_nemotron_free_result. + */ +EDDY_API EddyError eddy_nemotron_infer_file(EddyNemotronModel model, const char* wav_path, EddyNemotronResult* result, char** error_message); + +/** + * @brief Transcribe a raw float32 PCM buffer (16 kHz mono, normalized [-1, 1]). + * @param result Out param; free with eddy_nemotron_free_result. + */ +EDDY_API EddyError eddy_nemotron_infer_buffer(EddyNemotronModel model, const float* pcm, size_t length, int sample_rate, EddyNemotronResult* result, char** error_message); + +/** @brief Free an EddyNemotronResult (text, detected_language, token_ids). */ +EDDY_API void eddy_nemotron_free_result(EddyNemotronResult* result); + // Utility /** diff --git a/include/eddy/models/nemotron/nemotron.hpp b/include/eddy/models/nemotron/nemotron.hpp new file mode 100644 index 0000000..b5892c4 --- /dev/null +++ b/include/eddy/models/nemotron/nemotron.hpp @@ -0,0 +1,84 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 +// +// NVIDIA Nemotron-3.5-ASR-Streaming-Multilingual 0.6B backend. +// +// Unlike the Parakeet TDT path (stateless encoder + overlapping-chunk +// dedup + token/duration heads), Nemotron is a *cache-aware streaming* +// FastConformer-RNNT: the encoder carries cache tensors across chunks and +// takes an int `prompt_id` for language conditioning. Decoding is plain +// RNNT (no duration head). This warrants a separate module rather than +// overloading the Parakeet pipeline. + +#pragma once + +#include +#include +#include + +namespace eddy { +class OpenVINOBackend; +} + +namespace eddy::nemotron { + +/// Paths to the exported OpenVINO IR plus tokenizer/metadata. +/// File layout matches export_openvino.py / the HF model repo. +struct ModelPaths { + std::string preprocessor; // nemotron_preprocessor.xml (audio -> mel) + std::string encoder; // nemotron_encoder.xml (mel + caches + prompt_id -> encoded + caches) + std::string decoder; // nemotron_decoder.xml (token + lstm state -> dec_out + state) + std::string joint; // nemotron_joint.xml (enc_step + dec_step -> logits) + std::string vocab_json; // nemotron_vocab.json (id -> piece) + std::string metadata_json; // metadata.json (shapes, blank_idx, prompt_dictionary, ...) +}; + +struct Config { + // CPU is the default (safe, tested path; matches the eddy_c C API default and + // the CLI). Set "AUTO"/"NPU"/"GPU" to target other OpenVINO devices. + std::string device = "CPU"; // OpenVINO device for encoder/decoder/joint (preprocessor always CPU) + /// Language for prompt conditioning. Accepts dictionary keys ("en-US"), + /// 2-letter codes ("en" -> first "en-*"), or "auto" (model self-detects). + std::string language = "auto"; + size_t max_symbols_per_frame = 10; // RNNT inner-loop safety cap +}; + +struct TranscriptionResult { + std::string text; // lang-tag tokens stripped + std::string detected_language; // first tag emitted, if any (empty otherwise) + int prompt_id_used = 0; + std::vector token_ids; // raw emitted token ids (pre-strip) + double latency_ms = 0.0; +}; + +/// Streaming Nemotron ASR over OpenVINO. Construct, then transcribe whole +/// PCM buffers (internally chunked with cache-aware state continuity). +class OpenVINONemotron { +public: + OpenVINONemotron(std::shared_ptr backend, ModelPaths paths, Config config); + ~OpenVINONemotron(); + + OpenVINONemotron(const OpenVINONemotron&) = delete; + OpenVINONemotron& operator=(const OpenVINONemotron&) = delete; + + /// Compile models + load tokenizer/metadata (lazy; called by transcribe()). + void warmup(); + + /// Transcribe 16 kHz mono float32 PCM in [-1, 1]. + TranscriptionResult transcribe(const std::vector& pcm_16k_mono); + + /// Resolve a language string to its integer prompt id using the model's + /// prompt_dictionary (falls back to "auto"). + [[nodiscard]] int resolve_prompt_id(const std::string& language) const; + + struct Impl; + +private: + // const: only mutates *impl_ (reachable through the unique_ptr in a const + // method) and is std::call_once-guarded, so resolve_prompt_id() (const) can + // lazily compile without a const_cast. + void ensure_compiled() const; + std::unique_ptr impl_; +}; + +} // namespace eddy::nemotron diff --git a/include/eddy/models/nemotron/nemotron_featurizer.hpp b/include/eddy/models/nemotron/nemotron_featurizer.hpp new file mode 100644 index 0000000..0a4333e --- /dev/null +++ b/include/eddy/models/nemotron/nemotron_featurizer.hpp @@ -0,0 +1,60 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 +// +// Native C++ log-mel featurizer for the Nemotron streaming model — a drop-in +// replacement for the `nemotron_preprocessor.xml` OpenVINO IR. +// +// It reproduces NeMo's AudioToMelSpectrogramPreprocessor exactly (verified +// against the IR to fp16-storage precision): preemphasis 0.97 -> center pad +// n_fft/2 zeros -> framed STFT (Hann(win_length) centred in n_fft, hop) -> +// power spectrum -> slaney mel filterbank -> log(x + guard). The IR carries no +// per-feature normalisation; frames past the valid audio length are zeroed +// (matching the IR's length mask). + +#pragma once + +#include +#include + +namespace eddy::nemotron { + +class MelFeaturizer { + public: + // Nemotron defaults: 16 kHz, 128 mels, 25 ms Hann window (400 samples), + // 10 ms hop (160), 512-pt FFT, preemphasis 0.97, log guard 6e-8 (the value + // stored in the exported IR; NeMo's 2^-24 rounds to this at fp16). + explicit MelFeaturizer(int sample_rate = 16000, int n_mels = 128); + + // Compute log-mel for `n` samples of 16 kHz mono float PCM. `valid_samples` + // is the number of non-padding samples (frames whose index >= valid_samples/ + // hop are zeroed, mirroring the OV preprocessor). Fills `out_mel` with + // [n_mels * frames] in bin-major layout (out_mel[bin*frames + t]) and sets + // `out_frames`. + void compute(const float* audio, std::size_t n, int valid_samples, + std::vector& out_mel, std::size_t& out_frames) const; + + int n_mels() const { return n_mels_; } + int sample_rate() const { return sample_rate_; } + + private: + int sample_rate_; + int n_mels_; + int n_fft_; + int hop_; + int win_length_; + int n_freq_; // n_fft_/2 + 1 + float preemph_; + float log_guard_; + + std::vector window_; // [n_fft_]: Hann(win_length_) centred, 0 elsewhere + std::vector mel_fb_; // [n_mels_ * n_freq_], row-major (slaney) + + // Radix-2 FFT precomputed tables (size n_fft_). + std::vector bitrev_; + std::vector tw_cos_; // [n_fft_/2] + std::vector tw_sin_; // [n_fft_/2] + + void fft(std::vector& re, std::vector& im) const; +}; + +} // namespace eddy::nemotron diff --git a/include/eddy/utils/ensure_models.hpp b/include/eddy/utils/ensure_models.hpp index 92651a6..664bbda 100644 --- a/include/eddy/utils/ensure_models.hpp +++ b/include/eddy/utils/ensure_models.hpp @@ -1,4 +1,5 @@ -// Centralized helper to check and download Parakeet OpenVINO model files. +// Centralized helper to check and download OpenVINO model files. +// Model-agnostic: operates on any eddy::ModelConfig (Parakeet, Nemotron, ...). #pragma once @@ -8,7 +9,7 @@ #include #include -namespace eddy::parakeet { +namespace eddy::model_utils { // Checks if all required model files exist in target_dir. // Returns true if all files are present, false otherwise. @@ -39,5 +40,5 @@ using DownloadProgressCallback = std::functionsecond; // Create progress callback wrapper - eddy::parakeet::DownloadProgressCallback cpp_callback = nullptr; + eddy::model_utils::DownloadProgressCallback cpp_callback = nullptr; if (progress_callback) { cpp_callback = [progress_callback, user_data](const std::string& filename, int current, int total) { progress_callback(filename.c_str(), current, total, user_data); @@ -74,7 +75,7 @@ EddyError eddy_download_parakeet_models( // Download models std::string last_error; - bool success = eddy::parakeet::download_models( + bool success = eddy::model_utils::download_models( config, std::filesystem::path(target_dir), &last_error, @@ -397,7 +398,7 @@ EDDY_API EddyParakeetModel eddy_parakeet_create(EddyParakeetConfig config, char* } std::string err; - (void)eddy::parakeet::check_models_available(model_dir, &err); + (void)eddy::model_utils::check_models_available(model_dir, &err); #if defined(_WIN32) if (!std::filesystem::exists(model_dir)) { auto legacy = eddy::get_app_data_dir() / "cache" / "models" / "parakeet-v2" / "files"; @@ -510,4 +511,154 @@ EDDY_API char* eddy_parakeet_decode_tokens(EddyParakeetModel handle, const int* return copy_string(txt); } +// ----------------------------- +// Nemotron streaming C API +// ----------------------------- + +static constexpr const char* kNemotronModelName = "nemotron-streaming"; + +struct CNemotron { + std::unique_ptr model; +}; + +EDDY_API void eddy_nemotron_free_result(EddyNemotronResult* result) { + if (!result) return; + if (result->text) { delete[] result->text; result->text = nullptr; } + if (result->detected_language) { delete[] result->detected_language; result->detected_language = nullptr; } + if (result->token_ids) { delete[] result->token_ids; result->token_ids = nullptr; } + result->num_tokens = 0; +} + +EDDY_API EddyNemotronModel eddy_nemotron_create(EddyNemotronConfig config, char** error_message) { + try { + const std::string device = config.device ? config.device : "CPU"; + const std::string language = config.language ? config.language : "auto"; + + // Resolve model directory: explicit dir, else (NULL/"cache") the Eddy cache. + const std::string md = config.model_dir ? config.model_dir : ""; + std::filesystem::path model_dir = + (!md.empty() && md != "cache") ? std::filesystem::path(md) + : eddy::get_model_assets_dir(kNemotronModelName); + + auto backend = std::make_shared( + eddy::OpenVINOOptions{ .device = device, + .cache_dir = eddy::get_model_dir(kNemotronModelName).string() } + ); + + // Fail fast at create time if the model files are missing, matching the + // eddy_parakeet_create contract. Otherwise the first failure only surfaces + // deep inside ensure_compiled() on the initial transcribe()/warmup() call. + { + std::string check_err; + if (!eddy::model_utils::check_models_available( + model_dir, &check_err, eddy::model_configs::NEMOTRON_FILES)) { + throw std::runtime_error( + "Nemotron model files not available in '" + model_dir.string() + + "': " + check_err); + } + } + + eddy::nemotron::ModelPaths paths{ + .preprocessor = (model_dir / "nemotron_preprocessor.xml").string(), + .encoder = (model_dir / "nemotron_encoder.xml").string(), + .decoder = (model_dir / "nemotron_decoder.xml").string(), + .joint = (model_dir / "nemotron_joint.xml").string(), + .vocab_json = (model_dir / "nemotron_vocab.json").string(), + .metadata_json = (model_dir / "metadata.json").string(), + }; + + eddy::nemotron::Config cfg; + cfg.device = device; + cfg.language = language; + + auto handle = std::make_unique(); + handle->model = std::make_unique(backend, paths, cfg); + return static_cast(handle.release()); + } catch (const std::exception& e) { + if (error_message) *error_message = capture_exception(e); + return nullptr; + } catch (...) { + if (error_message) *error_message = copy_string("[Eddy Error] Unknown exception in nemotron create"); + return nullptr; + } +} + +EDDY_API void eddy_nemotron_destroy(EddyNemotronModel handle) { + if (!handle) return; + delete static_cast(handle); +} + +static EddyError nemotron_fill_result(const eddy::nemotron::TranscriptionResult& res, EddyNemotronResult* out) { + out->text = copy_string(res.text); + out->detected_language = copy_string(res.detected_language); + out->prompt_id_used = res.prompt_id_used; + out->latency_ms = res.latency_ms; + out->num_tokens = res.token_ids.size(); + if (out->num_tokens > 0) { + out->token_ids = new int[out->num_tokens]; + for (size_t i = 0; i < out->num_tokens; ++i) out->token_ids[i] = res.token_ids[i]; + } else { + out->token_ids = nullptr; + } + return EDDY_OK; +} + +EDDY_API EddyError eddy_nemotron_infer_buffer(EddyNemotronModel handle, const float* pcm, size_t length, + int sample_rate, EddyNemotronResult* out, char** err) { + if (!handle || !pcm || !out) { + if (err) *err = copy_string("[Eddy Error] Invalid argument: null pointer"); + return EDDY_ERROR_INVALID_ARGUMENT; + } + // Zero-init before any other early return so callers that follow the + // "always safe to eddy_nemotron_free_result" contract never delete[] + // uninitialized pointers (and so a throw mid-fill is cleaned up in catch). + *out = EddyNemotronResult{}; + if (sample_rate != 16000) { + if (err) *err = copy_string("[Eddy Error] Nemotron expects 16kHz mono audio"); + return EDDY_ERROR_INVALID_ARGUMENT; + } + try { + auto* h = static_cast(handle); + std::vector samples(pcm, pcm + length); + return nemotron_fill_result(h->model->transcribe(samples), out); + } catch (const std::exception& e) { + eddy_nemotron_free_result(out); + if (err) *err = capture_exception(e); + return EDDY_ERROR_INFERENCE_FAILED; + } catch (...) { + eddy_nemotron_free_result(out); + if (err) *err = copy_string("[Eddy Error] Unknown exception during nemotron inference"); + return EDDY_ERROR_UNKNOWN; + } +} + +EDDY_API EddyError eddy_nemotron_infer_file(EddyNemotronModel handle, const char* wav_path, + EddyNemotronResult* out, char** err) { + if (!handle || !wav_path || !out) { + if (err) *err = copy_string("[Eddy Error] Invalid argument: null pointer"); + return EDDY_ERROR_INVALID_ARGUMENT; + } + // Zero-init on every path (FILE_NOT_FOUND, a read_wav throw on a malformed + // file, ...) so a caller that frees *out after any error never delete[]s + // uninitialized pointers. + *out = EddyNemotronResult{}; + // Only a genuinely missing file is FILE_NOT_FOUND; read_wav also throws for + // format/channel/sample-rate/decode errors, which are not filesystem issues. + std::error_code ec; + if (!std::filesystem::exists(wav_path, ec)) { + if (err) *err = copy_string("[Eddy Error] WAV file not found: " + std::string(wav_path)); + return EDDY_ERROR_FILE_NOT_FOUND; + } + try { + auto pcm = eddy::audio::read_wav(wav_path); + return eddy_nemotron_infer_buffer(handle, pcm.data(), pcm.size(), 16000, out, err); + } catch (const std::exception& e) { + if (err) *err = capture_exception(e); + return EDDY_ERROR_INFERENCE_FAILED; + } catch (...) { + if (err) *err = copy_string("[Eddy Error] Unknown exception in nemotron infer_file"); + return EDDY_ERROR_UNKNOWN; + } +} + } // extern "C" diff --git a/src/models/nemotron/nemotron_featurizer.cpp b/src/models/nemotron/nemotron_featurizer.cpp new file mode 100644 index 0000000..d894968 --- /dev/null +++ b/src/models/nemotron/nemotron_featurizer.cpp @@ -0,0 +1,198 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 + +#include "eddy/models/nemotron/nemotron_featurizer.hpp" + +#define _USE_MATH_DEFINES +#include + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +#include +#include +#include + +namespace eddy::nemotron { + +namespace { + +// librosa slaney hz<->mel (htk=False). +inline double hz_to_mel(double hz) { + const double f_sp = 200.0 / 3.0; + const double min_log_hz = 1000.0; + const double min_log_mel = min_log_hz / f_sp; + const double logstep = std::log(6.4) / 27.0; + if (hz < min_log_hz) return hz / f_sp; + return min_log_mel + std::log(hz / min_log_hz) / logstep; +} + +inline double mel_to_hz(double mel) { + const double f_sp = 200.0 / 3.0; + const double min_log_hz = 1000.0; + const double min_log_mel = min_log_hz / f_sp; + const double logstep = std::log(6.4) / 27.0; + if (mel < min_log_mel) return f_sp * mel; + return min_log_hz * std::exp(logstep * (mel - min_log_mel)); +} + +} // namespace + +MelFeaturizer::MelFeaturizer(int sample_rate, int n_mels) + : sample_rate_(sample_rate), + n_mels_(n_mels), + n_fft_(512), + hop_(static_cast(sample_rate * 0.01 + 0.5)), // 10 ms -> 160 + win_length_(static_cast(sample_rate * 0.025 + 0.5)), // 25 ms -> 400 + n_freq_(512 / 2 + 1), + preemph_(0.97f), + log_guard_(6e-8f) { + // The featurizer is calibrated for NeMo's 16 kHz Nemotron config (25 ms / + // 10 ms framing -> win 400 / hop 160, 512-pt FFT). A 512 FFT only fits the + // window if win_length <= n_fft; guard so an unexpected sample rate fails + // loudly instead of silently producing garbage mel. + if (win_length_ > n_fft_) { + throw std::runtime_error( + "MelFeaturizer: window length " + std::to_string(win_length_) + + " exceeds n_fft " + std::to_string(n_fft_) + + " (sample_rate " + std::to_string(sample_rate_) + + " unsupported by the 512-pt featurizer)."); + } + + // Hann window (periodic=False) of win_length_, centred in the n_fft_ frame + // (torch pads (n_fft - win_length)/2 on the left). Zeros elsewhere. + window_.assign(n_fft_, 0.0f); + const int off = (n_fft_ - win_length_) / 2; + for (int i = 0; i < win_length_; ++i) { + const double w = 0.5 - 0.5 * std::cos(2.0 * M_PI * i / (win_length_ - 1)); + window_[off + i] = static_cast(w); + } + + // Slaney mel filterbank [n_mels_, n_freq_], norm='slaney', fmin=0, fmax=sr/2. + const double fmin = 0.0; + const double fmax = sample_rate_ / 2.0; + std::vector f_pts(n_mels_ + 2); + { + const double mmin = hz_to_mel(fmin); + const double mmax = hz_to_mel(fmax); + for (int i = 0; i < n_mels_ + 2; ++i) { + const double mel = mmin + (mmax - mmin) * i / (n_mels_ + 1); + f_pts[i] = mel_to_hz(mel); + } + } + std::vector fft_freqs(n_freq_); + for (int k = 0; k < n_freq_; ++k) { + fft_freqs[k] = (sample_rate_ / 2.0) * k / (n_freq_ - 1); + } + mel_fb_.assign(static_cast(n_mels_) * n_freq_, 0.0f); + for (int i = 0; i < n_mels_; ++i) { + const double lo = f_pts[i], ce = f_pts[i + 1], hi = f_pts[i + 2]; + const double enorm = 2.0 / (hi - lo); // slaney normalization + for (int k = 0; k < n_freq_; ++k) { + const double left = (fft_freqs[k] - lo) / (ce - lo); + const double right = (hi - fft_freqs[k]) / (hi - ce); + double v = std::min(left, right); + if (v < 0.0) v = 0.0; + mel_fb_[static_cast(i) * n_freq_ + k] = static_cast(v * enorm); + } + } + + // Radix-2 FFT tables: bit-reversal permutation + twiddle factors. + bitrev_.resize(n_fft_); + int log2n = 0; + while ((1 << log2n) < n_fft_) ++log2n; + for (int i = 0; i < n_fft_; ++i) { + int r = 0; + for (int b = 0; b < log2n; ++b) + if (i & (1 << b)) r |= 1 << (log2n - 1 - b); + bitrev_[i] = r; + } + tw_cos_.resize(n_fft_ / 2); + tw_sin_.resize(n_fft_ / 2); + for (int i = 0; i < n_fft_ / 2; ++i) { + const double ang = -2.0 * M_PI * i / n_fft_; + tw_cos_[i] = static_cast(std::cos(ang)); + tw_sin_[i] = static_cast(std::sin(ang)); + } +} + +// In-place iterative radix-2 Cooley-Tukey FFT, size n_fft_. +void MelFeaturizer::fft(std::vector& re, std::vector& im) const { + const int n = n_fft_; + for (int i = 0; i < n; ++i) { + const int j = bitrev_[i]; + if (j > i) { + std::swap(re[i], re[j]); + std::swap(im[i], im[j]); + } + } + for (int len = 2; len <= n; len <<= 1) { + const int half = len >> 1; + const int step = n / len; // twiddle stride + for (int base = 0; base < n; base += len) { + for (int k = 0; k < half; ++k) { + const float wc = tw_cos_[k * step]; + const float ws = tw_sin_[k * step]; + const int a = base + k; + const int b = base + k + half; + const float br = re[b] * wc - im[b] * ws; + const float bi = re[b] * ws + im[b] * wc; + re[b] = re[a] - br; + im[b] = im[a] - bi; + re[a] += br; + im[a] += bi; + } + } + } +} + +void MelFeaturizer::compute(const float* audio, std::size_t n, int valid_samples, + std::vector& out_mel, std::size_t& out_frames) const { + // Preemphasis: y[0]=x[0]; y[i]=x[i]-0.97*x[i-1]. + std::vector y(n); + if (n > 0) y[0] = audio[0]; + for (std::size_t i = 1; i < n; ++i) y[i] = audio[i] - preemph_ * audio[i - 1]; + + // Center pad n_fft/2 zeros each side, then frame with hop. The padded length + // is n + n_fft; #frames = 1 + (padded - n_fft)/hop = 1 + n/hop. + const int pad = n_fft_ / 2; + const std::size_t frames = 1 + n / static_cast(hop_); + out_frames = frames; + out_mel.assign(static_cast(n_mels_) * frames, 0.0f); + + // Frames at index >= valid_samples/hop are zeroed. This intentionally has no + // "+1" (unlike `frames` above): it mirrors the OV preprocessor's length mask, + // whose mel_length = audio_length/hop (verified — for a full chunk it zeroes + // exactly the trailing frame, which the encoder-input assembly then trims). + const std::size_t valid_frames = + static_cast(valid_samples) / static_cast(hop_); + + std::vector re(n_fft_), im(n_fft_), power(n_freq_); + for (std::size_t f = 0; f < frames; ++f) { + if (f >= valid_frames) continue; // length mask: zero (already zeroed) + + // Frame covers padded[f*hop : f*hop+n_fft]; padded index j maps to y[j-pad]. + // ptrdiff_t (not long, which is 32-bit on Win64) to avoid overflow on long audio. + const std::ptrdiff_t start = static_cast(f * static_cast(hop_)) - pad; + for (int t = 0; t < n_fft_; ++t) { + const std::ptrdiff_t src = start + t; + const float s = (src >= 0 && src < static_cast(n)) ? y[static_cast(src)] : 0.0f; + re[t] = s * window_[t]; + im[t] = 0.0f; + } + + fft(re, im); + for (int k = 0; k < n_freq_; ++k) power[k] = re[k] * re[k] + im[k] * im[k]; + + // mel = mel_fb (n_mels x n_freq) @ power; then log(mel + guard). + for (int mbin = 0; mbin < n_mels_; ++mbin) { + const float* row = &mel_fb_[static_cast(mbin) * n_freq_]; + float acc = 0.0f; + for (int k = 0; k < n_freq_; ++k) acc += row[k] * power[k]; + out_mel[static_cast(mbin) * frames + f] = std::log(acc + log_guard_); + } + } +} + +} // namespace eddy::nemotron diff --git a/src/models/nemotron/nemotron_openvino.cpp b/src/models/nemotron/nemotron_openvino.cpp new file mode 100644 index 0000000..4b87c3b --- /dev/null +++ b/src/models/nemotron/nemotron_openvino.cpp @@ -0,0 +1,554 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 +// +// Cache-aware streaming inference for NVIDIA Nemotron FastConformer-RNNT ASR. +// Serves both the 3.5-ASR-Streaming-Multilingual 0.6B model (per-chunk prompt_id +// language conditioning) and the English speech-streaming 0.6B model (no prompt, +// auto-detected from the encoder's inputs). +// +// Pipeline per chunk: native C++ mel featurizer -> cache-aware encoder +// (+ prompt_id when present) -> greedy RNNT decode, carrying the encoder caches +// and decoder LSTM state across chunks. + +#include "eddy/models/nemotron/nemotron.hpp" + +#include "eddy/backends/openvino_backend.hpp" +#include "eddy/models/nemotron/nemotron_featurizer.hpp" + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eddy::nemotron { + +namespace { + +// Mel feature buffer in [bins, frames] row-major (bin-major, matching the +// [1, bins, T] tensor layout the encoder expects). +struct MelBuf { + std::vector data; // size = bins * frames + size_t bins = 0; + size_t frames = 0; +}; + +// The OpenVINO NPU plugin miscompiles BitwiseNot on a boolean tensor: it does +// an integer bitwise complement, so ~0 = -1 and ~1 = -2 are *both* nonzero +// ("true"). The FastConformer attention mask is built with a `~` over a bool, +// so on NPU the mask becomes all-true -> every key is masked -> uniform softmax +// -> the encoder output collapses to ~0 and every transcript is empty. Replace +// BitwiseNot(bool) with the semantically identical LogicalNot, which the NPU +// compiles correctly; it is a no-op on CPU/GPU. Returns the (possibly rewritten) +// model ready to compile. +std::shared_ptr load_npu_safe(ov::Core& core, const std::string& xml) { + auto model = core.read_model(xml); + bool changed = false; + for (const auto& node : model->get_ordered_ops()) { + // Only a BitwiseNot over a boolean is equivalent to LogicalNot. Guard on the + // input element type so a future IR with an integer BitwiseNot isn't silently + // miscompiled (LogicalNot would change both semantics and output dtype). + if (ov::as_type_ptr(node) && + node->get_input_element_type(0) == ov::element::boolean) { + auto repl = std::make_shared(node->input_value(0)); + repl->set_friendly_name(node->get_friendly_name()); + ov::copy_runtime_info(node, repl); + ov::replace_node(node, repl); + changed = true; + } + } + if (changed) model->validate_nodes_and_infer_types(); + return model; +} + +ov::Tensor make_i32(int value) { + ov::Tensor t(ov::element::i32, ov::Shape{1}); + t.data()[0] = value; + return t; +} + +// SentencePiece word boundary marker (U+2581 "▁"). Unlike Parakeet, +// Nemotron's multilingual tokenizer emits standalone ▁ tokens, so the +// faithful decode is "concatenate pieces, then replace ▁ with space" +// (matches the validated Python reference), not per-piece prefix logic. +constexpr std::string_view kWordBoundary = "\xE2\x96\x81"; + +std::string finalize_text(std::string s) { + // Replace every ▁ with a space. + std::string out; + out.reserve(s.size()); + for (size_t i = 0; i < s.size();) { + if (s.compare(i, kWordBoundary.size(), kWordBoundary) == 0) { + out.push_back(' '); + i += kWordBoundary.size(); + } else { + out.push_back(s[i]); + ++i; + } + } + // Trim leading/trailing whitespace. + const auto b = out.find_first_not_of(" \t\n\r"); + const auto e = out.find_last_not_of(" \t\n\r"); + if (b == std::string::npos) return ""; + return out.substr(b, e - b + 1); +} + +} // namespace + +struct OpenVINONemotron::Impl { + std::shared_ptr backend; + ModelPaths paths; + Config config; + + std::vector vocab; // id -> piece (raw, ▁-marked) + + ov::CompiledModel encoder, decoder, joint; + ov::InferRequest encoder_req, decoder_req, joint_req; + + // Native C++ log-mel featurizer replacing the nemotron_preprocessor.xml IR. + std::unique_ptr featurizer; + + // Metadata + int sample_rate = 16000; + int mel_features = 128; + int chunk_mel_frames = 112; + int pre_encode_cache = 9; + int total_mel_frames = 121; + int blank_idx = 13087; + int vocab_size = 13087; + int decoder_hidden = 640; + int decoder_layers = 2; + int default_prompt_id = 101; + ov::Shape cache_channel_shape; + ov::Shape cache_time_shape; + std::map prompt_dictionary; + std::set lang_tag_token_ids; + + // Whether the encoder takes a `prompt_id` input. True for the multilingual + // model (per-chunk language conditioning); false for the monolingual English + // speech-streaming model. Auto-detected from the encoder's input ports so one + // backend serves both variants. + bool has_prompt = false; + + ov::element::Type token_et = ov::element::i32; + + std::once_flag compile_once; + std::mutex infer_guard; + + size_t chunk_samples() const { + // mel hop is 10 ms => frames * sample_rate / 100. Pure integer math avoids + // the rounding hazard of multiplying by the non-representable 0.01. + return static_cast(chunk_mel_frames) * static_cast(sample_rate) / 100; + } +}; + +OpenVINONemotron::OpenVINONemotron(std::shared_ptr backend, + ModelPaths paths, Config config) + : impl_(std::make_unique()) { + if (!backend) { + throw std::invalid_argument("OpenVINO backend is null"); + } + impl_->backend = std::move(backend); + impl_->paths = std::move(paths); + impl_->config = std::move(config); +} + +OpenVINONemotron::~OpenVINONemotron() = default; + +void OpenVINONemotron::warmup() { ensure_compiled(); } + +int OpenVINONemotron::resolve_prompt_id(const std::string& language) const { + // prompt_dictionary is populated by ensure_compiled(); make this safe to call + // standalone (before transcribe()/warmup()). ensure_compiled() is const (it only + // mutates *impl_, reachable through the unique_ptr in a const method) and + // std::call_once guarded, so this is a cheap no-op once compiled. + ensure_compiled(); + const auto& dict = impl_->prompt_dictionary; + auto it = dict.find(language); + if (it != dict.end()) { + return it->second; + } + if (language.size() == 2) { + const std::string prefix = language + "-"; + for (const auto& [k, v] : dict) { + if (k.size() >= prefix.size() && + std::equal(prefix.begin(), prefix.end(), k.begin(), + [](char a, char b) { return std::tolower(a) == std::tolower(b); })) { + return v; + } + } + } + return impl_->default_prompt_id; +} + +void OpenVINONemotron::ensure_compiled() const { + std::call_once(impl_->compile_once, [this]() { + auto& core = impl_->backend->core(); + const std::string device = impl_->config.device.empty() ? "AUTO" : impl_->config.device; + + // --- Load metadata.json --- + { + std::ifstream f(impl_->paths.metadata_json); + if (!f.good()) { + throw std::runtime_error("Failed to open Nemotron metadata: " + impl_->paths.metadata_json); + } + nlohmann::json m; + f >> m; + impl_->sample_rate = m.value("sample_rate", 16000); + impl_->mel_features = m.value("mel_features", 128); + impl_->chunk_mel_frames = m.value("chunk_mel_frames", 112); + impl_->pre_encode_cache = m.value("pre_encode_cache", 9); + impl_->total_mel_frames = m.value("total_mel_frames", 121); + impl_->blank_idx = m.value("blank_idx", 13087); + impl_->vocab_size = m.value("vocab_size", 13087); + impl_->decoder_hidden = m.value("decoder_hidden", 640); + impl_->decoder_layers = m.value("decoder_layers", 2); + impl_->default_prompt_id = m.value("default_prompt_id", 101); + + auto to_shape = [](const nlohmann::json& arr) { + ov::Shape s; + for (const auto& d : arr) s.push_back(d.get()); + return s; + }; + // These two keys have no sensible default (they size the encoder caches), + // so require them explicitly with a path-aware message rather than letting + // nlohmann's bare "key not found" propagate from a truncated metadata.json. + if (!m.contains("cache_channel_shape") || !m.contains("cache_time_shape")) { + throw std::runtime_error( + "Nemotron metadata missing required cache shape keys " + "(cache_channel_shape / cache_time_shape): " + impl_->paths.metadata_json); + } + impl_->cache_channel_shape = to_shape(m.at("cache_channel_shape")); + impl_->cache_time_shape = to_shape(m.at("cache_time_shape")); + + if (m.contains("prompt_dictionary")) { + for (auto& [k, v] : m["prompt_dictionary"].items()) { + impl_->prompt_dictionary[k] = v.get(); + } + } + if (m.contains("lang_tag_token_ids")) { + for (const auto& id : m["lang_tag_token_ids"]) { + impl_->lang_tag_token_ids.insert(id.get()); + } + } + } + + // --- Mel featurizer (native C++, replaces nemotron_preprocessor.xml) --- + // The IR preprocessor is dynamic-shaped (NPU-incompatible) and adds an + // OV inference per chunk; the native featurizer reproduces it exactly + // (validated to fp16-storage precision). paths.preprocessor is unused. + impl_->featurizer = std::make_unique(impl_->sample_rate, impl_->mel_features); + // Guard: the featurizer's framing must match the model's expected geometry. + // A full chunk (chunk_mel_frames * sample_rate/100 samples) must yield + // chunk_mel_frames + 1 mel frames; otherwise metadata (sample_rate / + // chunk_mel_frames) disagrees with the hardcoded 10 ms hop and the mel would + // silently misalign with the encoder. + { + const size_t cs = impl_->chunk_samples(); + std::vector probe(cs, 0.0f); + std::vector mel_probe; + size_t probe_frames = 0; + impl_->featurizer->compute(probe.data(), cs, static_cast(cs), mel_probe, probe_frames); + const size_t expected = static_cast(impl_->chunk_mel_frames) + 1; + if (probe_frames != expected) { + throw std::runtime_error( + "Nemotron C++ featurizer geometry mismatch: produced " + + std::to_string(probe_frames) + " frames for a chunk, expected " + + std::to_string(expected) + " (chunk_mel_frames=" + + std::to_string(impl_->chunk_mel_frames) + ", sample_rate=" + + std::to_string(impl_->sample_rate) + "). The model's featurizer " + "config differs from the hardcoded 25 ms/10 ms framing."); + } + } + + // --- Compile models on the chosen device --- + // The encoder carries the attention-mask BitwiseNot that the NPU plugin + // miscompiles, so load it through the rewrite (no-op on CPU/GPU). + impl_->encoder = core.compile_model(load_npu_safe(core, impl_->paths.encoder), device); + impl_->decoder = core.compile_model(impl_->paths.decoder, device); + impl_->joint = core.compile_model(impl_->paths.joint, device); + + impl_->encoder_req = impl_->encoder.create_infer_request(); + impl_->decoder_req = impl_->decoder.create_infer_request(); + impl_->joint_req = impl_->joint.create_infer_request(); + + // Detect prompt conditioning from the encoder's input ports: the + // multilingual encoder has a "prompt_id" input; the English speech-streaming + // encoder does not. Drives whether transcribe() feeds a prompt_id tensor. + impl_->has_prompt = false; + for (const auto& in : impl_->encoder.inputs()) { + if (in.get_names().count("prompt_id")) { impl_->has_prompt = true; break; } + } + + impl_->token_et = impl_->decoder.input("token").get_element_type(); + + // --- Vocab (id -> piece). Flat {"0":"piece", ...} format. --- + { + std::ifstream f(impl_->paths.vocab_json); + if (!f.good()) { + throw std::runtime_error("Failed to open Nemotron vocab: " + impl_->paths.vocab_json); + } + nlohmann::json v; + f >> v; + size_t max_id = 0; + for (auto& [k, _] : v.items()) { + max_id = std::max(max_id, static_cast(std::stoul(k))); + } + impl_->vocab.assign(max_id + 1, std::string{}); + for (auto& [k, val] : v.items()) { + impl_->vocab[std::stoul(k)] = val.get(); + } + } + }); +} + +TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) { + ensure_compiled(); + std::lock_guard lock(impl_->infer_guard); + + const auto t_start = std::chrono::steady_clock::now(); + + auto& I = *impl_; + const size_t bins = static_cast(I.mel_features); + const size_t total = static_cast(I.total_mel_frames); + const size_t pre_cache = static_cast(I.pre_encode_cache); + const size_t chunk_samples = I.chunk_samples(); + + // prompt_id only applies to the multilingual (prompt-conditioned) encoder. + const int prompt_id = I.has_prompt ? resolve_prompt_id(I.config.language) : 0; + + // Persistent encoder caches (carried across chunks). + ov::Tensor cache_channel(ov::element::f32, I.cache_channel_shape); + ov::Tensor cache_time(ov::element::f32, I.cache_time_shape); + std::memset(cache_channel.data(), 0, cache_channel.get_byte_size()); + std::memset(cache_time.data(), 0, cache_time.get_byte_size()); + ov::Tensor cache_len = make_i32(0); + + // Persistent LSTM state. + const ov::Shape lstm_shape{static_cast(I.decoder_layers), 1, + static_cast(I.decoder_hidden)}; + ov::Tensor h(ov::element::f32, lstm_shape); + ov::Tensor c(ov::element::f32, lstm_shape); + std::memset(h.data(), 0, h.get_byte_size()); + std::memset(c.data(), 0, c.get_byte_size()); + + int last_token = I.blank_idx; + std::vector all_tokens; + + MelBuf mel_cache; // last pre_encode_cache frames of previous chunk's mel + std::vector mel_scratch; // per-chunk featurizer output [bins * t_mel] + + const ov::Tensor prompt_tensor = make_i32(prompt_id); // unused when !has_prompt + + // Hot-path tensors whose shapes/values are fixed for the whole call — allocate + // once and reuse across chunks and the inner RNNT loop instead of re-allocating + // every iteration (the inner `token`/`token_length` allocs dominate otherwise). + ov::Tensor audio(ov::element::f32, ov::Shape{1, chunk_samples}); + ov::Tensor mel_in(ov::element::f32, ov::Shape{1, bins, total}); + ov::Tensor token(I.token_et, ov::Shape{1, 1}); + const ov::Tensor token_length = make_i32(1); + const ov::Tensor mel_length = make_i32(static_cast(total)); + + size_t off = 0; + while (off < pcm.size()) { + const size_t end = std::min(off + chunk_samples, pcm.size()); + + // Raw audio chunk, padded to chunk_samples (reusing the hoisted tensor). + float* adst = audio.data(); + std::memset(adst, 0, audio.get_byte_size()); + std::copy(pcm.begin() + static_cast(off), pcm.begin() + static_cast(end), adst); + + // Preprocessor: audio -> mel [bins, T_mel] (native C++ featurizer). + // audio_length is intentionally the FULL padded chunk size on every chunk + // (mirrors the WER-validated reference: the chunk is zero-padded to + // chunk_samples and that full length is passed), so the length mask zeros + // only the trailing frame, which the assembly below trims anyway. + size_t t_mel = 0; + I.featurizer->compute(adst, chunk_samples, static_cast(chunk_samples), + mel_scratch, t_mel); + const float* mel_src = mel_scratch.data(); // bin-major [bins * t_mel] + + // Build encoder mel input [1, bins, total]: prepend cache (or zero + // pre_encode_cache on first chunk), then pad/trim to total. (reusing the + // hoisted tensor; fully overwritten via memset + fills below.) + float* mdst = mel_in.data(); + std::memset(mdst, 0, mel_in.get_byte_size()); + + const size_t cache_frames = mel_cache.frames; // 0 on first chunk + const size_t lead = (cache_frames > 0) ? cache_frames : pre_cache; // zero-pad lead on first chunk + for (size_t bin = 0; bin < bins; ++bin) { + float* row = mdst + bin * total; + size_t col = 0; + // leading cache frames + for (size_t t = 0; t < lead && col < total; ++t, ++col) { + if (cache_frames > 0) { + row[col] = mel_cache.data[bin * cache_frames + t]; + } // else zero (already memset) + } + // current chunk mel frames + for (size_t t = 0; t < t_mel && col < total; ++t, ++col) { + row[col] = mel_src[bin * t_mel + t]; + } + } + + // Update mel_cache = last pre_encode_cache frames of current chunk mel. + const size_t keep = std::min(pre_cache, t_mel); + mel_cache.bins = bins; + mel_cache.frames = keep; + // resize, not assign: every element is unconditionally overwritten by the + // loop below (bin*keep + t is a bijection over [0, bins*keep)), so the + // zero-fill assign() would do is pure waste on this per-chunk hot buffer. + mel_cache.data.resize(bins * keep); + for (size_t bin = 0; bin < bins; ++bin) { + for (size_t t = 0; t < keep; ++t) { + mel_cache.data[bin * keep + t] = mel_src[bin * t_mel + (t_mel - keep + t)]; + } + } + + // Encoder: mel + caches + prompt_id -> encoded + caches + I.encoder_req.set_tensor("mel", mel_in); + I.encoder_req.set_tensor("mel_length", mel_length); + I.encoder_req.set_tensor("cache_channel", cache_channel); + I.encoder_req.set_tensor("cache_time", cache_time); + I.encoder_req.set_tensor("cache_len", cache_len); + if (I.has_prompt) I.encoder_req.set_tensor("prompt_id", prompt_tensor); + I.encoder_req.infer(); + + const ov::Tensor encoded = I.encoder_req.get_tensor("encoded"); // [1, D, T_enc] + // Persist updated caches (copy out before next infer overwrites them). + { + const ov::Tensor cc = I.encoder_req.get_tensor("cache_channel_out"); + const ov::Tensor ctt = I.encoder_req.get_tensor("cache_time_out"); + const ov::Tensor cl = I.encoder_req.get_tensor("cache_len_out"); + // Cache-aware streaming: the *_out caches are the same fixed shape as the + // input caches (the ring buffer is re-filled in place), so we copy back + // into the pre-allocated input tensors. Check the byte sizes agree so a + // mismatched IR export throws here instead of silently over-/under-reading. + // Runtime check (not assert): Release builds define NDEBUG. + if (cc.get_byte_size() != cache_channel.get_byte_size() || + ctt.get_byte_size() != cache_time.get_byte_size() || + cl.get_element_type() != ov::element::i32) { + throw std::runtime_error( + "Nemotron encoder cache_*_out shape/type differs from the pre-allocated " + "input cache; the model IR does not match metadata.json cache shapes."); + } + std::memcpy(cache_channel.data(), cc.data(), cache_channel.get_byte_size()); + std::memcpy(cache_time.data(), ctt.data(), cache_time.get_byte_size()); + cache_len.data()[0] = cl.data()[0]; + } + + const ov::Shape enc_shape = encoded.get_shape(); // [1, D, T_enc] + const size_t enc_d = enc_shape[1]; + const size_t t_enc = enc_shape[2]; + const float* enc_data = encoded.data(); + + // Greedy RNNT decode over encoder frames. + ov::Tensor enc_step(ov::element::f32, ov::Shape{1, enc_d, 1}); + for (size_t t = 0; t < t_enc; ++t) { + float* es = enc_step.data(); + for (size_t ch = 0; ch < enc_d; ++ch) { + es[ch] = enc_data[ch * t_enc + t]; + } + + for (size_t sym = 0; sym < I.config.max_symbols_per_frame; ++sym) { + // Decoder (token/token_length tensors hoisted above the loops; just + // overwrite the scalar token value each iteration). + if (I.token_et == ov::element::i64) { + token.data()[0] = last_token; + } else { + token.data()[0] = last_token; + } + I.decoder_req.set_tensor("token", token); + I.decoder_req.set_tensor("token_length", token_length); + I.decoder_req.set_tensor("h_in", h); + I.decoder_req.set_tensor("c_in", c); + I.decoder_req.infer(); + const ov::Tensor dec_out = I.decoder_req.get_tensor("decoder_out"); // [1, H, 1] + + // Joint + I.joint_req.set_tensor("encoder", enc_step); + I.joint_req.set_tensor("decoder", dec_out); + I.joint_req.infer(); + const ov::Tensor logits = I.joint_req.get_tensor("logits"); // [1,1,1,V] + const float* lg = logits.data(); + const size_t vsz = logits.get_size(); + + int best = 0; + float best_score = lg[0]; + for (size_t i = 1; i < vsz; ++i) { + if (lg[i] > best_score) { + best_score = lg[i]; + best = static_cast(i); + } + } + + if (best == I.blank_idx) { + break; + } + all_tokens.push_back(best); + last_token = best; + // Advance LSTM state on emission. h_out/c_out are the same fixed shape as + // h_in/c_in by construction; guard so a mismatched decoder IR throws + // instead of corrupting the fixed-size state tensors. + const ov::Tensor h_out = I.decoder_req.get_tensor("h_out"); + const ov::Tensor c_out = I.decoder_req.get_tensor("c_out"); + if (h_out.get_byte_size() != h.get_byte_size() || + c_out.get_byte_size() != c.get_byte_size()) { + throw std::runtime_error( + "Nemotron decoder h_out/c_out byte size differs from the LSTM state " + "tensors; the decoder IR does not match the expected layer/hidden dims."); + } + std::memcpy(h.data(), h_out.data(), h.get_byte_size()); + std::memcpy(c.data(), c_out.data(), c.get_byte_size()); + } + } + + off += chunk_samples; + } + + // Strip blank / out-of-range / language-tag tokens; concatenate pieces. + auto piece = [&](int tok) -> const std::string& { + static const std::string empty; + return (tok >= 0 && tok < static_cast(I.vocab.size())) ? I.vocab[tok] : empty; + }; + + TranscriptionResult result; + result.prompt_id_used = prompt_id; + result.token_ids = all_tokens; + std::string body; + for (int tok : all_tokens) { + // blank intentionally sits at index vocab_size (== blank_idx), so the + // explicit blank check is redundant with `tok >= vocab_size`; kept for + // clarity since the two are configured independently from metadata.json. + if (tok == I.blank_idx || tok >= I.vocab_size) continue; + if (I.lang_tag_token_ids.count(tok)) { + if (result.detected_language.empty()) { + result.detected_language = finalize_text(piece(tok)); + } + continue; + } + body += piece(tok); + } + result.text = finalize_text(body); + + const auto t_end = std::chrono::steady_clock::now(); + result.latency_ms = std::chrono::duration(t_end - t_start).count(); + return result; +} + +} // namespace eddy::nemotron diff --git a/src/utils/ensure_models.cpp b/src/utils/ensure_models.cpp index 125ffc5..f6af35d 100644 --- a/src/utils/ensure_models.cpp +++ b/src/utils/ensure_models.cpp @@ -1,12 +1,13 @@ -// Centralized check and download for Parakeet model files. +// Centralized check and download for OpenVINO model files (model-agnostic). #include "eddy/utils/ensure_models.hpp" +#include #include #include #include -namespace eddy::parakeet { +namespace eddy::model_utils { static bool file_nonempty(const std::filesystem::path& p) { std::error_code ec; @@ -15,9 +16,56 @@ static bool file_nonempty(const std::filesystem::path& p) { std::filesystem::file_size(p, ec) > 0; } +// The download shells out via std::system, so any interpolated component must +// be free of characters that could break out of the double-quoted argument. +// Today every field is a compile-time constant, but ModelConfig is caller- +// supplied, so reject anything outside a conservative charset rather than risk +// command injection. The long-term fix is to drop std::system for a direct +// libcurl call and eliminate this class of concern entirely. +// +// The URL and the local output path get *separate* allowlists: the URL is the +// real injection surface (it's assembled from caller-supplied ModelConfig +// fields) so it stays tight — only the characters an https HuggingFace URL +// needs, and notably no '~'. The output path is application-controlled (the +// Eddy cache dir) but must also tolerate native Windows paths, so it +// additionally allows the native separator '\\' and spaces (the drive-letter +// ':' is already covered by the shared charset). +static bool charset_ok(const std::string& s, bool allow_path_chars) { + for (const unsigned char c : s) { + bool ok = std::isalnum(c) || c == '.' || c == '_' || c == '-' || + c == '/' || c == ':'; + if (!ok && allow_path_chars) { + // Native Windows paths: backslash separators and spaces inside the + // double-quoted argument. None of these can break out of the quotes. + ok = (c == '\\' || c == ' '); + } + if (!ok) return false; + } + return true; +} + +static bool is_url_safe(const std::string& s) { return charset_ok(s, /*allow_path_chars=*/false); } +static bool is_path_safe(const std::string& s) { return charset_ok(s, /*allow_path_chars=*/true); } + static bool download_single_file(const std::string& url, const std::filesystem::path& output_path, std::string* error_msg = nullptr) { + // Refuse to build a shell command from unsafe components. + if (!is_url_safe(url) || !is_path_safe(output_path.string())) { + if (error_msg) { + *error_msg = "Refusing to download: unsafe characters in URL or path: " + url; + } + return false; + } + // The charset permits '.' and '/', so reject ".." components explicitly to + // stop a caller-supplied filename from writing outside the target directory. + for (const auto& part : output_path) { + if (part == "..") { + if (error_msg) *error_msg = "Refusing to download: path traversal in " + output_path.string(); + return false; + } + } + // Create parent directory std::error_code ec; std::filesystem::create_directories(output_path.parent_path(), ec); @@ -35,6 +83,10 @@ static bool download_single_file(const std::string& url, // Execute download int ret = std::system(curl_cmd.c_str()); if (ret != 0) { + // Remove any partial file: an interrupted transfer leaves a truncated file + // that file_nonempty() would later accept, silently skipping a re-download + // and loading a corrupt model. + std::filesystem::remove(output_path, ec); if (error_msg) { *error_msg = "curl failed with exit code " + std::to_string(ret) + " for URL: " + url; } @@ -44,6 +96,7 @@ static bool download_single_file(const std::string& url, // Verify downloaded file auto size = std::filesystem::file_size(output_path, ec); if (ec || size == 0) { + std::filesystem::remove(output_path, ec); if (error_msg) { *error_msg = "Downloaded file is missing or empty: " + output_path.string(); } @@ -108,8 +161,11 @@ bool download_models(const eddy::ModelConfig& config, continue; } - // Construct HuggingFace URL - const std::string url = "https://huggingface.co/" + config.repo_id + "/resolve/main/" + filename; + // Construct HuggingFace URL. When repo_subdir is set, files live in a + // subfolder of the repo (e.g. "fp16/") but are still stored flat locally. + const std::string remote_rel = + config.repo_subdir.empty() ? filename : config.repo_subdir + "/" + filename; + const std::string url = "https://huggingface.co/" + config.repo_id + "/resolve/main/" + remote_rel; // Notify progress if (progress_callback) { @@ -142,4 +198,4 @@ bool download_models(const eddy::ModelConfig& config, return true; } -} // namespace eddy::parakeet +} // namespace eddy::model_utils