diff --git a/README.md b/README.md index 16c752a..7466c80 100644 --- a/README.md +++ b/README.md @@ -111,12 +111,16 @@ await init(); Sentence-level normalization scans for normalizable spans within a larger sentence: ```rust -use text_processing_rs::{normalize_sentence, tn_normalize_sentence}; +use text_processing_rs::{normalize_sentence, normalize_sentence_lang, tn_normalize_sentence}; // ITN sentence mode let result = normalize_sentence("I have twenty one apples"); assert_eq!(result, "I have 21 apples"); +// ITN sentence mode, language-aware ("en", "fr", "es", "de", "zh", "hi", "ja") +let result = normalize_sentence_lang("j'ai vingt et un ans", "fr"); +assert_eq!(result, "j'ai 21 ans"); + // TN sentence mode let result = tn_normalize_sentence("I paid $5 for 23 items"); assert_eq!(result, "I paid five dollars for twenty three items"); @@ -139,6 +143,10 @@ let spoken = NemoTextProcessing.tnNormalize("$5.50") let itn = NemoTextProcessing.normalizeSentence("I have twenty one apples") // "I have 21 apples" +// Language-aware ITN sentence mode ("en", "fr", "es", "de", "zh", "hi", "ja") +let itnFr = NemoTextProcessing.normalizeSentence("j'ai vingt et un ans", language: "fr") +// "j'ai 21 ans" + let tn = NemoTextProcessing.tnNormalizeSentence("I paid $5 for 23 items") // "I paid five dollars for twenty three items" ``` diff --git a/src/ffi.rs b/src/ffi.rs index a480125..b35926e 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -4,9 +4,9 @@ use std::ffi::{c_char, CStr, CString}; use std::ptr; use crate::{ - custom_rules, normalize, normalize_sentence, normalize_sentence_with_options, - normalize_with_options, tn_normalize, tn_normalize_lang, tn_normalize_sentence, - tn_normalize_sentence_lang, tn_normalize_sentence_with_max_span, + custom_rules, normalize, normalize_sentence, normalize_sentence_lang, + normalize_sentence_with_options, normalize_with_options, tn_normalize, tn_normalize_lang, + tn_normalize_sentence, tn_normalize_sentence_lang, tn_normalize_sentence_with_max_span, tn_normalize_sentence_with_max_span_lang, NormalizeOptions, }; @@ -168,6 +168,41 @@ pub unsafe extern "C" fn nemo_normalize_sentence_with_options( } } +/// Normalize a full sentence (ITN, spoken → written) for a specific language. +/// +/// Supported language codes: "en", "fr", "es", "de", "zh", "hi", "ja". +/// Falls back to English for unrecognized codes. The ITN counterpart of +/// [`nemo_tn_normalize_sentence_lang`]. +/// +/// # Safety +/// - `input` and `lang` must be valid null-terminated UTF-8 strings +/// - Returns a newly allocated string that must be freed with `nemo_free_string` +#[no_mangle] +pub unsafe extern "C" fn nemo_normalize_sentence_lang( + input: *const c_char, + lang: *const c_char, +) -> *mut c_char { + if input.is_null() || lang.is_null() { + return ptr::null_mut(); + } + + let input_str = match CStr::from_ptr(input).to_str() { + Ok(s) => s, + Err(_) => return ptr::null_mut(), + }; + let lang_str = match CStr::from_ptr(lang).to_str() { + Ok(s) => s, + Err(_) => return ptr::null_mut(), + }; + + let result = normalize_sentence_lang(input_str, lang_str); + + match CString::new(result) { + Ok(c_string) => c_string.into_raw(), + Err(_) => ptr::null_mut(), + } +} + /// Free a string allocated by nemo_normalize or nemo_normalize_sentence. /// /// # Safety @@ -564,4 +599,34 @@ mod tests { nemo_free_string(opted_in); } } + + #[test] + fn test_ffi_normalize_sentence_lang() { + unsafe { + // Sliding-window language (fr): span within a larger sentence. + let input = CString::new("j'ai vingt et un ans").unwrap(); + let lang = CString::new("fr").unwrap(); + let result = nemo_normalize_sentence_lang(input.as_ptr(), lang.as_ptr()); + assert!(!result.is_null()); + assert_eq!(CStr::from_ptr(result).to_str().unwrap(), "j'ai 21 ans"); + nemo_free_string(result); + + // Scanning language (zh): whole-sentence in-place replacement. + let zh_input = CString::new("我有二十一个苹果").unwrap(); + let zh_lang = CString::new("zh").unwrap(); + let zh_result = nemo_normalize_sentence_lang(zh_input.as_ptr(), zh_lang.as_ptr()); + assert_eq!(CStr::from_ptr(zh_result).to_str().unwrap(), "我有21个苹果"); + nemo_free_string(zh_result); + } + } + + #[test] + fn test_ffi_normalize_sentence_lang_null() { + unsafe { + let lang = CString::new("fr").unwrap(); + assert!(nemo_normalize_sentence_lang(ptr::null(), lang.as_ptr()).is_null()); + let input = CString::new("vingt et un").unwrap(); + assert!(nemo_normalize_sentence_lang(input.as_ptr(), ptr::null()).is_null()); + } + } } diff --git a/src/lib.rs b/src/lib.rs index aa6d489..b838b91 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1042,6 +1042,83 @@ pub fn normalize_sentence_with_options(input: &str, options: NormalizeOptions) - ) } +/// Normalize a full sentence (ITN, spoken → written) for a specific language. +/// +/// Supported language codes: "en", "fr", "es", "de", "zh", "hi", "ja". +/// Falls back to English for unrecognized codes. +/// +/// Unlike [`normalize_with_lang`], which treats the whole input as a single +/// expression, this scans for normalizable spans within a larger sentence — +/// the ITN counterpart of [`tn_normalize_sentence_lang`]. +/// +/// ``` +/// use text_processing_rs::normalize_sentence_lang; +/// +/// assert_eq!( +/// normalize_sentence_lang("j'ai vingt et un ans", "fr"), +/// "j'ai 21 ans" +/// ); +/// ``` +pub fn normalize_sentence_lang(input: &str, lang: &str) -> String { + normalize_sentence_with_max_span_lang(input, lang, DEFAULT_MAX_SPAN_TOKENS) +} + +/// Normalize a full sentence (ITN) for a specific language with a configurable +/// max span size. +/// +/// Two tagger architectures are dispatched here: +/// - `en`/`fr`/`de`/`es` use expression-level taggers, so the input is scanned +/// with the shared sliding-window [`sentence_loop`]. +/// - `hi`/`ja`/`zh` taggers already scan the whole sentence in place, so +/// [`normalize_with_lang`] is the sentence-level entry point for them and +/// `max_span_tokens` does not apply. +/// +/// Unrecognized codes fall back to English sentence normalization. +pub fn normalize_sentence_with_max_span_lang( + input: &str, + lang: &str, + max_span_tokens: usize, +) -> String { + match lang { + // Scanning-based ITN: processors already replace spans across the whole + // sentence in place, so the sliding window is unnecessary. + "hi" | "ja" | "zh" => normalize_with_lang(input, lang), + // Expression-level taggers: scan the sentence span by span. + "fr" | "de" | "es" => { + let trimmed = input.trim(); + if trimmed.is_empty() { + return trimmed.to_string(); + } + let pretokens = pretokenize(trimmed); + sentence_loop(&pretokens, max_span_tokens, |span| { + parse_span_lang(span, lang) + }) + } + // "en", "", and any unrecognized code use English sentence mode. + _ => normalize_sentence_inner(input, max_span_tokens, false, false), + } +} + +/// Span-level ITN dispatch for expression-tagger languages (`fr`/`de`/`es`). +/// +/// Each `try_*_taggers` helper already tries its language's taggers in priority +/// order and returns the first match, so a single constant priority is +/// sufficient here: [`sentence_loop`] only consults the score to break ties +/// between equal-length spans, and this path yields at most one candidate per +/// span. Returns `None` for languages without expression taggers. +fn parse_span_lang(span: &str, lang: &str) -> Option<(String, u8)> { + if span.split_whitespace().count() == 0 { + return None; + } + let result = match lang { + "fr" => try_fr_taggers(span), + "de" => try_de_taggers(span), + "es" => try_es_taggers(span), + _ => None, + }?; + Some((result, 70)) +} + /// Per-pretoken record: the token text plus the original separator that /// preceded it (`" "` for whitespace, `""` if the token came from a /// punctuation split or starts the input). diff --git a/src/wasm.rs b/src/wasm.rs index 13f475b..7763abb 100644 --- a/src/wasm.rs +++ b/src/wasm.rs @@ -3,10 +3,11 @@ use wasm_bindgen::prelude::*; use crate::{ - custom_rules, normalize, normalize_sentence, normalize_sentence_with_options, - normalize_with_lang, normalize_with_options, tn_normalize, tn_normalize_lang, - tn_normalize_sentence, tn_normalize_sentence_lang, tn_normalize_sentence_with_max_span, - tn_normalize_sentence_with_max_span_lang, NormalizeOptions, + custom_rules, normalize, normalize_sentence, normalize_sentence_lang, + normalize_sentence_with_options, normalize_with_lang, normalize_with_options, tn_normalize, + tn_normalize_lang, tn_normalize_sentence, tn_normalize_sentence_lang, + tn_normalize_sentence_with_max_span, tn_normalize_sentence_with_max_span_lang, + NormalizeOptions, }; /// Build [`NormalizeOptions`] from JS-friendly primitives. @@ -50,6 +51,11 @@ pub fn normalize_sentence_js(input: &str) -> String { normalize_sentence(input) } +#[wasm_bindgen(js_name = normalizeSentenceLang)] +pub fn normalize_sentence_lang_js(input: &str, lang: &str) -> String { + normalize_sentence_lang(input, lang) +} + /// Unified single-expression normalize. `concatCompoundNumbers=true` reads /// consecutive number words as concatenation rather than addition, e.g. /// `"thirty five sixty two"` → `"3562"`, `"seven eighty eight"` → `"788"`. diff --git a/swift/NemoTextProcessing.swift b/swift/NemoTextProcessing.swift index 605d3b0..1768010 100644 --- a/swift/NemoTextProcessing.swift +++ b/swift/NemoTextProcessing.swift @@ -95,6 +95,29 @@ public enum NemoTextProcessing { return String(cString: resultPtr) } + /// Normalize a full sentence (ITN) for a specific language, replacing + /// spoken-form spans with written form. + /// + /// Supported language codes: `"en"`, `"fr"`, `"es"`, `"de"`, `"zh"`, + /// `"hi"`, `"ja"`. Falls back to English for unrecognized codes. The ITN + /// counterpart of `tnNormalizeSentence(_:language:)`. + /// + /// - Parameters: + /// - input: Sentence containing spoken-form spans + /// - language: ISO 639-1 language code + /// - Returns: Sentence with spoken-form spans replaced with written form + public static func normalizeSentence(_ input: String, language: String) -> String { + guard let inputC = input.cString(using: .utf8), + let langC = language.cString(using: .utf8) else { + return input + } + guard let resultPtr = nemo_normalize_sentence_lang(inputC, langC) else { + return input + } + defer { nemo_free_string(resultPtr) } + return String(cString: resultPtr) + } + /// Normalize a single spoken-form expression with caller-specified options. /// /// - Parameters: diff --git a/swift/include/nemo_text_processing.h b/swift/include/nemo_text_processing.h index 8009635..496e39c 100644 --- a/swift/include/nemo_text_processing.h +++ b/swift/include/nemo_text_processing.h @@ -69,6 +69,19 @@ char* nemo_normalize_sentence_with_options( uint32_t disable_bare_second ); +/** + * Normalize a full sentence (ITN, spoken -> written) for a specific language. + * + * Supported language codes: "en", "fr", "es", "de", "zh", "hi", "ja". + * Falls back to English for unrecognized codes. The ITN counterpart of + * nemo_tn_normalize_sentence_lang. + * + * @param input Null-terminated UTF-8 string + * @param lang Null-terminated language code (e.g. "de", "fr") + * @return Newly allocated string, must be freed with nemo_free_string(). + */ +char* nemo_normalize_sentence_lang(const char* input, const char* lang); + /** * Add a custom spoken-to-written normalization rule. * Custom rules have the highest priority, checked before all built-in taggers. diff --git a/tests/multilang_itn_tests.rs b/tests/multilang_itn_tests.rs new file mode 100644 index 0000000..f45e033 --- /dev/null +++ b/tests/multilang_itn_tests.rs @@ -0,0 +1,144 @@ +//! Multi-language inverse text normalization integration tests. +//! +//! Tests the ITN language dispatch API: normalize_sentence_lang(). Ensures all +//! 7 languages are wired up and producing correct output through the public +//! sentence-level API (the ITN counterpart of tn_normalize_sentence_lang). +//! +//! Two tagger architectures are exercised: +//! - en/fr/de/es: expression-level taggers scanned with the sliding window. +//! - hi/ja/zh: whole-sentence scanning taggers (sentence == single-expression). + +use text_processing_rs::{normalize_sentence_lang, normalize_sentence_with_max_span_lang}; + +// ── French (sliding-window taggers) ────────────────────────────────── + +#[test] +fn test_fr_sentence_single_expression() { + assert_eq!( + normalize_sentence_lang("j'ai vingt et un ans", "fr"), + "j'ai 21 ans" + ); +} + +#[test] +fn test_fr_sentence_multiple_expressions() { + // Two separate number spans in one sentence — the whole-input + // normalize_with_lang would leave this untouched; the sliding window + // catches each span. + assert_eq!( + normalize_sentence_lang("j'ai vingt et un ans et trente chiens", "fr"), + "j'ai 21 ans et 30 chiens" + ); +} + +#[test] +fn test_fr_sentence_passthrough() { + assert_eq!( + normalize_sentence_lang("bonjour le monde", "fr"), + "bonjour le monde" + ); +} + +// ── German (sliding-window taggers) ────────────────────────────────── + +#[test] +fn test_de_sentence_single_expression() { + assert_eq!( + normalize_sentence_lang("ich habe einundzwanzig Katzen", "de"), + "ich habe 21 Katzen" + ); +} + +#[test] +fn test_de_sentence_multiple_expressions() { + assert_eq!( + normalize_sentence_lang("ich habe dreiundzwanzig Äpfel und zehn Birnen", "de"), + "ich habe 23 Äpfel und 10 Birnen" + ); +} + +// ── Spanish (sliding-window taggers) ───────────────────────────────── + +#[test] +fn test_es_sentence_single_expression() { + assert_eq!( + normalize_sentence_lang("tengo veintiuno años", "es"), + "tengo 21 años" + ); +} + +// ── Chinese (whole-sentence scanning) ──────────────────────────────── + +#[test] +fn test_zh_sentence() { + assert_eq!( + normalize_sentence_lang("我有二十一个苹果", "zh"), + "我有21个苹果" + ); +} + +// ── Japanese (whole-sentence scanning) ─────────────────────────────── + +#[test] +fn test_ja_sentence() { + assert_eq!( + normalize_sentence_lang("私は二十一個のりんごを持っています", "ja"), + "私は21個のりんごを持っています" + ); +} + +// ── Hindi (whole-sentence scanning, Devanagari digits) ─────────────── + +#[test] +fn test_hi_sentence() { + assert_eq!( + normalize_sentence_lang("मेरे पास इक्कीस सेब हैं", "hi"), + "मेरे पास २१ सेब हैं" + ); +} + +// ── English + fallback ─────────────────────────────────────────────── + +#[test] +fn test_en_sentence() { + assert_eq!( + normalize_sentence_lang("I have twenty one apples", "en"), + "I have 21 apples" + ); +} + +#[test] +fn test_unknown_lang_falls_back_to_english() { + assert_eq!( + normalize_sentence_lang("I have twenty one apples", "xx"), + "I have 21 apples" + ); + // Empty language code also routes through English sentence mode. + assert_eq!( + normalize_sentence_lang("I have twenty one apples", ""), + "I have 21 apples" + ); +} + +// ── Edge cases ─────────────────────────────────────────────────────── + +#[test] +fn test_empty_and_whitespace() { + for lang in ["en", "fr", "de", "es", "hi", "ja", "zh"] { + assert_eq!(normalize_sentence_lang("", lang), ""); + assert_eq!(normalize_sentence_lang(" ", lang), ""); + } +} + +#[test] +fn test_max_span_zero_uses_single_token_spans() { + // max_span_tokens == 0 clamps to 1 in the sliding window, so multi-token + // French cardinals ("vingt et un") no longer coalesce. + let out = normalize_sentence_with_max_span_lang("j'ai vingt et un ans", "fr", 0); + assert_ne!(out, "j'ai 21 ans"); + // Scanning languages ignore max_span entirely. + assert_eq!( + normalize_sentence_with_max_span_lang("我有二十一个苹果", "zh", 0), + "我有21个苹果" + ); +} diff --git a/wasm-tests/node-smoke.mjs b/wasm-tests/node-smoke.mjs index d13cf43..81c8a4a 100644 --- a/wasm-tests/node-smoke.mjs +++ b/wasm-tests/node-smoke.mjs @@ -23,6 +23,11 @@ assertEqual( 'I have 21 apples', 'normalizeSentence should convert spans' ); +assertEqual( + wasm.normalizeSentenceLang("j'ai vingt et un ans", 'fr'), + "j'ai 21 ans", + 'normalizeSentenceLang should convert spans per language' +); assertEqual(wasm.tnNormalize('$5.50'), 'five dollars fifty cents', 'tnNormalize should work'); assertEqual( wasm.tnNormalizeSentence('I paid $5 for 23 items'),