feat(ffi): add normalize_sentence_lang for multi-language ITN (#83)

README.md

@@ -111,12 +111,16 @@ await init();
 Sentence-level normalization scans for normalizable spans within a larger sentence:
 
 ```rust
-use text_processing_rs::{normalize_sentence, tn_normalize_sentence};
+use text_processing_rs::{normalize_sentence, normalize_sentence_lang, tn_normalize_sentence};
 
 // ITN sentence mode
 let result = normalize_sentence("I have twenty one apples");
 assert_eq!(result, "I have 21 apples");
 
+// ITN sentence mode, language-aware ("en", "fr", "es", "de", "zh", "hi", "ja")
+let result = normalize_sentence_lang("j'ai vingt et un ans", "fr");
+assert_eq!(result, "j'ai 21 ans");
+
 // TN sentence mode
 let result = tn_normalize_sentence("I paid $5 for 23 items");
 assert_eq!(result, "I paid five dollars for twenty three items");
@@ -139,6 +143,10 @@ let spoken = NemoTextProcessing.tnNormalize("$5.50")
 let itn = NemoTextProcessing.normalizeSentence("I have twenty one apples")
 // "I have 21 apples"
 
+// Language-aware ITN sentence mode ("en", "fr", "es", "de", "zh", "hi", "ja")
+let itnFr = NemoTextProcessing.normalizeSentence("j'ai vingt et un ans", language: "fr")
+// "j'ai 21 ans"
+
 let tn = NemoTextProcessing.tnNormalizeSentence("I paid $5 for 23 items")
 // "I paid five dollars for twenty three items"
 ```

src/ffi.rs

@@ -4,9 +4,9 @@ use std::ffi::{c_char, CStr, CString};
 use std::ptr;
 
 use crate::{
-    custom_rules, normalize, normalize_sentence, normalize_sentence_with_options,
-    normalize_with_options, tn_normalize, tn_normalize_lang, tn_normalize_sentence,
-    tn_normalize_sentence_lang, tn_normalize_sentence_with_max_span,
+    custom_rules, normalize, normalize_sentence, normalize_sentence_lang,
+    normalize_sentence_with_options, normalize_with_options, tn_normalize, tn_normalize_lang,
+    tn_normalize_sentence, tn_normalize_sentence_lang, tn_normalize_sentence_with_max_span,
     tn_normalize_sentence_with_max_span_lang, NormalizeOptions,
 };
 
@@ -168,6 +168,41 @@ pub unsafe extern "C" fn nemo_normalize_sentence_with_options(
     }
 }
 
+/// Normalize a full sentence (ITN, spoken → written) for a specific language.
+///
+/// Supported language codes: "en", "fr", "es", "de", "zh", "hi", "ja".
+/// Falls back to English for unrecognized codes. The ITN counterpart of
+/// [`nemo_tn_normalize_sentence_lang`].
+///
+/// # Safety
+/// - `input` and `lang` must be valid null-terminated UTF-8 strings
+/// - Returns a newly allocated string that must be freed with `nemo_free_string`
+#[no_mangle]
+pub unsafe extern "C" fn nemo_normalize_sentence_lang(
+    input: *const c_char,
+    lang: *const c_char,
+) -> *mut c_char {
+    if input.is_null() || lang.is_null() {
+        return ptr::null_mut();
+    }
+
+    let input_str = match CStr::from_ptr(input).to_str() {
+        Ok(s) => s,
+        Err(_) => return ptr::null_mut(),
+    };
+    let lang_str = match CStr::from_ptr(lang).to_str() {
+        Ok(s) => s,
+        Err(_) => return ptr::null_mut(),
+    };
+
+    let result = normalize_sentence_lang(input_str, lang_str);
+
+    match CString::new(result) {
+        Ok(c_string) => c_string.into_raw(),
+        Err(_) => ptr::null_mut(),
+    }
+}
+
 /// Free a string allocated by nemo_normalize or nemo_normalize_sentence.
 ///
 /// # Safety
@@ -564,4 +599,34 @@ mod tests {
             nemo_free_string(opted_in);
         }
     }
+
+    #[test]
+    fn test_ffi_normalize_sentence_lang() {
+        unsafe {
+            // Sliding-window language (fr): span within a larger sentence.
+            let input = CString::new("j'ai vingt et un ans").unwrap();
+            let lang = CString::new("fr").unwrap();
+            let result = nemo_normalize_sentence_lang(input.as_ptr(), lang.as_ptr());
+            assert!(!result.is_null());
+            assert_eq!(CStr::from_ptr(result).to_str().unwrap(), "j'ai 21 ans");
+            nemo_free_string(result);
+
+            // Scanning language (zh): whole-sentence in-place replacement.
+            let zh_input = CString::new("我有二十一个苹果").unwrap();
+            let zh_lang = CString::new("zh").unwrap();
+            let zh_result = nemo_normalize_sentence_lang(zh_input.as_ptr(), zh_lang.as_ptr());
+            assert_eq!(CStr::from_ptr(zh_result).to_str().unwrap(), "我有21个苹果");
+            nemo_free_string(zh_result);
+        }
+    }
+
+    #[test]
+    fn test_ffi_normalize_sentence_lang_null() {
+        unsafe {
+            let lang = CString::new("fr").unwrap();
+            assert!(nemo_normalize_sentence_lang(ptr::null(), lang.as_ptr()).is_null());
+            let input = CString::new("vingt et un").unwrap();
+            assert!(nemo_normalize_sentence_lang(input.as_ptr(), ptr::null()).is_null());
+        }
+    }
 }

src/lib.rs

@@ -1042,6 +1042,83 @@ pub fn normalize_sentence_with_options(input: &str, options: NormalizeOptions) -
     )
 }
 
+/// Normalize a full sentence (ITN, spoken → written) for a specific language.
+///
+/// Supported language codes: "en", "fr", "es", "de", "zh", "hi", "ja".
+/// Falls back to English for unrecognized codes.
+///
+/// Unlike [`normalize_with_lang`], which treats the whole input as a single
+/// expression, this scans for normalizable spans within a larger sentence —
+/// the ITN counterpart of [`tn_normalize_sentence_lang`].
+///
+/// ```
+/// use text_processing_rs::normalize_sentence_lang;
+///
+/// assert_eq!(
+///     normalize_sentence_lang("j'ai vingt et un ans", "fr"),
+///     "j'ai 21 ans"
+/// );
+/// ```
+pub fn normalize_sentence_lang(input: &str, lang: &str) -> String {
+    normalize_sentence_with_max_span_lang(input, lang, DEFAULT_MAX_SPAN_TOKENS)
+}
+
+/// Normalize a full sentence (ITN) for a specific language with a configurable
+/// max span size.
+///
+/// Two tagger architectures are dispatched here:
+/// - `en`/`fr`/`de`/`es` use expression-level taggers, so the input is scanned
+///   with the shared sliding-window [`sentence_loop`].
+/// - `hi`/`ja`/`zh` taggers already scan the whole sentence in place, so
+///   [`normalize_with_lang`] is the sentence-level entry point for them and
+///   `max_span_tokens` does not apply.
+///
+/// Unrecognized codes fall back to English sentence normalization.
+pub fn normalize_sentence_with_max_span_lang(
+    input: &str,
+    lang: &str,
+    max_span_tokens: usize,
+) -> String {
+    match lang {
+        // Scanning-based ITN: processors already replace spans across the whole
+        // sentence in place, so the sliding window is unnecessary.
+        "hi" | "ja" | "zh" => normalize_with_lang(input, lang),
+        // Expression-level taggers: scan the sentence span by span.
+        "fr" | "de" | "es" => {
+            let trimmed = input.trim();
+            if trimmed.is_empty() {
+                return trimmed.to_string();
+            }
+            let pretokens = pretokenize(trimmed);
+            sentence_loop(&pretokens, max_span_tokens, |span| {
+                parse_span_lang(span, lang)
+            })
+        }
+        // "en", "", and any unrecognized code use English sentence mode.
+        _ => normalize_sentence_inner(input, max_span_tokens, false, false),
+    }
+}
+
+/// Span-level ITN dispatch for expression-tagger languages (`fr`/`de`/`es`).
+///
+/// Each `try_*_taggers` helper already tries its language's taggers in priority
+/// order and returns the first match, so a single constant priority is
+/// sufficient here: [`sentence_loop`] only consults the score to break ties
+/// between equal-length spans, and this path yields at most one candidate per
+/// span. Returns `None` for languages without expression taggers.
+fn parse_span_lang(span: &str, lang: &str) -> Option<(String, u8)> {
+    if span.split_whitespace().count() == 0 {
+        return None;
+    }
+    let result = match lang {
+        "fr" => try_fr_taggers(span),
+        "de" => try_de_taggers(span),
+        "es" => try_es_taggers(span),
+        _ => None,
+    }?;
+    Some((result, 70))
+}
+
 /// Per-pretoken record: the token text plus the original separator that
 /// preceded it (`" "` for whitespace, `""` if the token came from a
 /// punctuation split or starts the input).

src/wasm.rs

@@ -3,10 +3,11 @@
 use wasm_bindgen::prelude::*;
 
 use crate::{
-    custom_rules, normalize, normalize_sentence, normalize_sentence_with_options,
-    normalize_with_lang, normalize_with_options, tn_normalize, tn_normalize_lang,
-    tn_normalize_sentence, tn_normalize_sentence_lang, tn_normalize_sentence_with_max_span,
-    tn_normalize_sentence_with_max_span_lang, NormalizeOptions,
+    custom_rules, normalize, normalize_sentence, normalize_sentence_lang,
+    normalize_sentence_with_options, normalize_with_lang, normalize_with_options, tn_normalize,
+    tn_normalize_lang, tn_normalize_sentence, tn_normalize_sentence_lang,
+    tn_normalize_sentence_with_max_span, tn_normalize_sentence_with_max_span_lang,
+    NormalizeOptions,
 };
 
 /// Build [`NormalizeOptions`] from JS-friendly primitives.
@@ -50,6 +51,11 @@ pub fn normalize_sentence_js(input: &str) -> String {
     normalize_sentence(input)
 }
 
+#[wasm_bindgen(js_name = normalizeSentenceLang)]
+pub fn normalize_sentence_lang_js(input: &str, lang: &str) -> String {
+    normalize_sentence_lang(input, lang)
+}
+
 /// Unified single-expression normalize. `concatCompoundNumbers=true` reads
 /// consecutive number words as concatenation rather than addition, e.g.
 /// `"thirty five sixty two"` → `"3562"`, `"seven eighty eight"` → `"788"`.

swift/NemoTextProcessing.swift

@@ -95,6 +95,29 @@ public enum NemoTextProcessing {
         return String(cString: resultPtr)
     }
 
+    /// Normalize a full sentence (ITN) for a specific language, replacing
+    /// spoken-form spans with written form.
+    ///
+    /// Supported language codes: `"en"`, `"fr"`, `"es"`, `"de"`, `"zh"`,
+    /// `"hi"`, `"ja"`. Falls back to English for unrecognized codes. The ITN
+    /// counterpart of `tnNormalizeSentence(_:language:)`.
+    ///
+    /// - Parameters:
+    ///   - input: Sentence containing spoken-form spans
+    ///   - language: ISO 639-1 language code
+    /// - Returns: Sentence with spoken-form spans replaced with written form
+    public static func normalizeSentence(_ input: String, language: String) -> String {
+        guard let inputC = input.cString(using: .utf8),
+              let langC = language.cString(using: .utf8) else {
+            return input
+        }
+        guard let resultPtr = nemo_normalize_sentence_lang(inputC, langC) else {
+            return input
+        }
+        defer { nemo_free_string(resultPtr) }
+        return String(cString: resultPtr)
+    }
+
     /// Normalize a single spoken-form expression with caller-specified options.
     ///
     /// - Parameters:

swift/include/nemo_text_processing.h

@@ -69,6 +69,19 @@ char* nemo_normalize_sentence_with_options(
     uint32_t disable_bare_second
 );
 
+/**
+ * Normalize a full sentence (ITN, spoken -> written) for a specific language.
+ *
+ * Supported language codes: "en", "fr", "es", "de", "zh", "hi", "ja".
+ * Falls back to English for unrecognized codes. The ITN counterpart of
+ * nemo_tn_normalize_sentence_lang.
+ *
+ * @param input Null-terminated UTF-8 string
+ * @param lang  Null-terminated language code (e.g. "de", "fr")
+ * @return Newly allocated string, must be freed with nemo_free_string().
+ */
+char* nemo_normalize_sentence_lang(const char* input, const char* lang);
+
 /**
  * Add a custom spoken-to-written normalization rule.
  * Custom rules have the highest priority, checked before all built-in taggers.