Apply CheckBidi to every label in a Bidi domain

src/to_ascii.cpp

@@ -84,6 +84,53 @@ static bool is_ace_prefix(std::u32string_view label) noexcept {
          label[2] == U'-' && label[3] == U'-';
 }
 
+// RFC 5893: a Bidi domain name contains at least one RTL label. CheckBidi then
+// applies to every label in the domain, so this must be known before the first
+// label is validated. ACE labels are decoded; a decode failure is left for the
+// conversion pass, which rejects the domain.
+static bool mapped_domain_is_bidi(std::u32string_view mapped) {
+  const char32_t* p = mapped.data();
+  const char32_t* const end = p + mapped.size();
+  std::string puny;
+  std::u32string decoded;
+  while (p < end) {
+    const char32_t* begin = p;
+    while (p < end && *p != U'.') {
+      ++p;
+    }
+    const std::u32string_view label(begin, static_cast<size_t>(p - begin));
+    if (p < end) {
+      ++p;
+    }
+    if (label.empty()) {
+      continue;
+    }
+    if (is_ace_prefix(label)) {
+      bool ace_ascii = true;
+      for (char32_t c : label) {
+        if (c >= 0x80) {
+          ace_ascii = false;
+          break;
+        }
+      }
+      if (!ace_ascii) {
+        continue;
+      }
+      puny.resize(label.size() - 4);
+      for (size_t i = 0; i < puny.size(); i++) {
+        puny[i] = static_cast<char>(label[i + 4]);
+      }
+      decoded.clear();
+      if (punycode_to_utf32(puny, decoded) && label_contains_rtl(decoded)) {
+        return true;
+      }
+    } else if (label_contains_rtl(label)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 [[nodiscard]] bool to_ascii(std::string_view ut8_string, std::string& out) {
   out.clear();
   // The ASCII path is a single same-size copy, so it needs no length bound.
@@ -136,6 +183,9 @@ static bool is_ace_prefix(std::u32string_view label) noexcept {
     }
   }
 
+  // CheckBidi applies to every label once any label is RTL (RFC 5893).
+  const bool bidi_domain = mapped_domain_is_bidi(mapped);
+
   // Estimate ASCII output size (punycode may expand non-ASCII labels).
   out.reserve(mapped.size() + 8);
 
@@ -188,14 +238,20 @@ static bool is_ace_prefix(std::u32string_view label) noexcept {
           return false;
         }
       }
-      if (post_map.empty() || !is_label_valid(post_map)) {
+      if (post_map.empty() || !check_label(post_map, bidi_domain)) {
         out.clear();
         return false;
       }
     } else if (is_ascii(label_view)) {
+      // All-ASCII domains never reach here (they return above). An ASCII label
+      // in a Bidi domain must still satisfy RFC 5893 rules 1, 5, and 6.
+      if (bidi_domain && !check_label(label_view, true)) {
+        out.clear();
+        return false;
+      }
       append_ascii_label(out, label_view);
     } else {
-      if (!is_label_valid(label_view)) {
+      if (!check_label(label_view, bidi_domain)) {
         out.clear();
         return false;
       }

src/to_unicode.cpp

@@ -17,11 +17,79 @@
 
 namespace ada::idna {
 
+// True when an all-ASCII label is valid on its own but violates RFC 5893
+// because the domain contains an RTL label (for example a label starting
+// with a digit). ToUnicode then returns the original input (UTS #46).
+static bool ascii_label_fails_bidi_domain(std::string_view label) {
+  std::u32string utf32(label.size(), 0);
+  for (size_t i = 0; i < label.size(); i++) {
+    utf32[i] = static_cast<unsigned char>(label[i]);
+  }
+  return check_label(utf32, false) && !check_label(utf32, true);
+}
+
+static bool utf8_domain_is_bidi(std::string_view input) {
+  std::u32string decoded;
+  size_t label_start = 0;
+  while (label_start < input.size()) {
+    const size_t loc_dot = input.find('.', label_start);
+    const bool is_last_label = (loc_dot == std::string_view::npos);
+    const size_t label_size =
+        is_last_label ? input.size() - label_start : loc_dot - label_start;
+    const std::string_view label = input.substr(label_start, label_size);
+    if (label.starts_with("xn--") && is_ascii(label)) {
+      decoded.clear();
+      if (label.size() > 4 && punycode_to_utf32(label.substr(4), decoded) &&
+          label_contains_rtl(decoded)) {
+        return true;
+      }
+    } else if (!is_ascii(label)) {
+      const size_t utf32_length =
+          utf32_length_from_utf8(label.data(), label.size());
+      if (utf32_length != 0) {
+        decoded.assign(utf32_length, 0);
+        const size_t written =
+            utf8_to_utf32(label.data(), label.size(), decoded.data());
+        if (written == utf32_length && label_contains_rtl(decoded)) {
+          return true;
+        }
+      }
+    }
+    if (is_last_label) {
+      break;
+    }
+    label_start += label_size + 1;
+  }
+  return false;
+}
+
 [[nodiscard]] bool to_unicode(std::string_view input, std::string& output) {
   output.clear();
   if (input.size() > max_domain_input_bytes) {
     return false;
   }
+  // An ASCII label that is only illegal because the domain is a Bidi domain
+  // fails ToUnicode for the whole input (UTS #46), not just that label.
+  const bool bidi_domain = utf8_domain_is_bidi(input);
+  if (bidi_domain) {
+    size_t label_start = 0;
+    while (label_start < input.size()) {
+      const size_t loc_dot = input.find('.', label_start);
+      const bool is_last_label = (loc_dot == std::string_view::npos);
+      const size_t label_size =
+          is_last_label ? input.size() - label_start : loc_dot - label_start;
+      const std::string_view label = input.substr(label_start, label_size);
+      if (!label.empty() && is_ascii(label) && !label.starts_with("xn--") &&
+          ascii_label_fails_bidi_domain(label)) {
+        output.assign(input);
+        return true;
+      }
+      if (is_last_label) {
+        break;
+      }
+      label_start += label_size + 1;
+    }
+  }
   output.reserve(input.size());
 
   size_t label_start = 0;
@@ -50,7 +118,7 @@ namespace ada::idna {
             accept_decoded = false;
           } else if (!ada::idna::normalize(post_map) ||
                      post_map != tmp_buffer || post_map.empty() ||
-                     !ada::idna::is_label_valid(post_map)) {
+                     !check_label(post_map, bidi_domain)) {
             accept_decoded = false;
           }
         }

src/validity.cpp

@@ -128,7 +128,9 @@ inline static size_t find_last_not_of_nsm(
 }
 
 // An RTL label is a label that contains at least one character of type R, AL,
-// or AN. https://www.rfc-editor.org/rfc/rfc5893#section-2
+// or AN. A domain containing one is a Bidi domain name: RFC 5893 then applies
+// to every label, including labels that are themselves all LTR.
+// https://www.rfc-editor.org/rfc/rfc5893#section-2
 inline static bool is_rtl_label(const std::u32string_view label) noexcept {
   const size_t mask =
       (1u << direction::R) | (1u << direction::AL) | (1u << direction::AN);
@@ -140,7 +142,21 @@ inline static bool is_rtl_label(const std::u32string_view label) noexcept {
   return (directions & mask) != 0;
 }
 
-bool is_label_valid(const std::u32string_view label) {
+// Same as is_rtl_label, but loads the Bidi tables first. Visible to to_ascii
+// and to_unicode, which are part of this translation unit.
+inline static bool label_contains_rtl(std::u32string_view label) noexcept {
+  if (label.empty() || !ensure_tables() || dir_start == nullptr ||
+      dir_final == nullptr || dir_value == nullptr) {
+    return false;
+  }
+  return is_rtl_label(label);
+}
+
+// `bidi_domain` is true when the domain name contains at least one RTL label.
+// CheckBidi then applies to this label even if the label itself has no R, AL,
+// or AN character (RFC 5893). The one-argument is_label_valid() passes false,
+// which preserves the per-label check for callers that do not know yet.
+static bool check_label(const std::u32string_view label, bool bidi_domain) {
   if (label.empty()) {
     return true;
   }
@@ -231,8 +247,8 @@ bool is_label_valid(const std::u32string_view label) {
 
   // A "Bidi domain name" is a domain name that contains at least one RTL label.
   // The following rule, consisting of six conditions, applies to labels in Bidi
-  // domain names.
-  if (is_rtl_label(label)) {
+  // domain names: every label, not only the RTL ones.
+  if (bidi_domain || is_rtl_label(label)) {
     // The first character must be a character with Bidi property L, R,
     // or AL. If it has the R or AL property, it is an RTL label; if it
     // has the L property, it is an LTR label.
@@ -301,4 +317,8 @@ bool is_label_valid(const std::u32string_view label) {
   return true;
 }
 
+bool is_label_valid(const std::u32string_view label) {
+  return check_label(label, false);
+}
+
 }  // namespace ada::idna

tests/to_ascii_tests.cpp

@@ -218,9 +218,9 @@ TEST(to_ascii_tests, contextj_joiner_does_not_bypass_bidi) {
 }
 
 TEST(to_ascii_tests, contextj_checks_every_joiner_and_bidi_rule_1) {
-  // Ported from ada-url/ada#1276. A virama before a joiner settles only that
-  // joiner: later joiners in the label still need their own ContextJ check,
-  // and the RFC 5893 Bidi rule must still run on the label.
+  // Ported from ada-url/ada#1276 and ada-url/ada#1277. A virama before a joiner
+  // settles only that joiner: later joiners in the label still need their own
+  // ContextJ check, and the RFC 5893 Bidi rule must still run on the label.
   // 'a' (L) + virama + ZWNJ + ZWJ: the ZWJ has no virama before it.
   EXPECT_TRUE(ada::idna::to_ascii("a\u094D\u200C\u200D.example").empty())
       << "a joiner after a valid virama+ZWNJ must still be checked";
@@ -232,6 +232,50 @@ TEST(to_ascii_tests, contextj_checks_every_joiner_and_bidi_rule_1) {
   EXPECT_EQ(ada::idna::to_ascii("\u0915\u094D\u200C\u0915.example"),
             "xn--11ba1ow90g.example")
       << "valid ZWNJ label must still convert";
+
+  // Regression tests from ada-url/ada#1277:
+  // 1) U+0644 U+200C U+0073 U+06EF ("\u0644\u200Cs\u06EF"):
+  //    ZWNJ is between U+0644 (LAM, D) and U+0073 ('s', U).
+  //    Before is D, but after is U (not R or D), violating ContextJ.
+  EXPECT_TRUE(ada::idna::to_ascii("\u0644\u200Cs\u06EF").empty())
+      << "ZWNJ followed by latin 's' violates ContextJ";
+  // 2) U+0061 U+0644 U+200C U+06EF ("a\u0644\u200C\u06EF"):
+  //    Label starts with 'a' (L), but contains U+0644 (AL) and U+06EF (AL),
+  //    violating Bidi rule 5.
+  EXPECT_TRUE(ada::idna::to_ascii("a\u0644\u200C\u06EF").empty())
+      << "LTR label containing Arabic letters violates Bidi rule 5";
+  // 3) U+0627 U+0BCD U+200D ("\u0627\u0BCD\u200D"):
+  //    U+0627 (AL) + U+0BCD (TAMIL SIGN VIRAMA) + U+200D (ZWJ).
+  //    ZWJ preceded by virama satisfies ContextJ, but Bidi rule 3 requires
+  //    the RTL label to end with R, AL, EN, or AN followed by zero or more NSM.
+  //    ZWJ (U+200D) has Bidi property BN, not NSM/R/AL/EN/AN, so it violates
+  //    Bidi rule 3.
+  EXPECT_TRUE(ada::idna::to_ascii("\u0627\u0BCD\u200D").empty())
+      << "RTL label ending with ZWJ violates Bidi rule 3";
+
+  // Hangul LV + T composition regression test from ada-url/idna#104:
+  // U+AC00 (가, LV) + U+11A8 (ᆨ, T) -> U+AC01 (각) under NFC.
+  // Must produce xn--p39a, not xn--rud7310f.
+  EXPECT_EQ(ada::idna::to_ascii("\uAC00\u11A8.example"), "xn--p39a.example");
+}
+
+TEST(to_ascii_tests, bidi_domain_checks_every_label) {
+  // RFC 5893 applies to every label of a domain that contains an RTL label,
+  // including labels that have no R, AL, or AN character of their own.
+  // ada-url/ada#1277.
+  EXPECT_TRUE(ada::idna::to_ascii("1.\u0627").empty())
+      << "EN-leading label in a Bidi domain violates rule 1";
+  EXPECT_TRUE(ada::idna::to_ascii("\u0627.1a").empty())
+      << "EN-leading label in a Bidi domain violates rule 1";
+  EXPECT_TRUE(ada::idna::to_ascii("1a.\u0627").empty());
+  // An LTR label that starts with L and ends with L or EN is allowed.
+  EXPECT_EQ(ada::idna::to_ascii("example.\u0627"), "example.xn--mgb");
+  EXPECT_EQ(ada::idna::to_ascii("a1.\u0627"), "a1.xn--mgb");
+  EXPECT_EQ(ada::idna::to_ascii("\u0627.example"), "xn--mgb.example");
+  // Not a Bidi domain, so a leading digit is fine.
+  EXPECT_EQ(ada::idna::to_ascii("1.example"), "1.example");
+  // All-ASCII input, including punycode, stays on the WHATWG fast path.
+  EXPECT_EQ(ada::idna::to_ascii("1.xn--mgb"), "1.xn--mgb");
 }
 
 TEST(to_ascii_tests, contextj_viramas_from_newer_unicode) {

tests/to_unicode_tests.cpp

@@ -80,3 +80,13 @@ TEST_F(IdnaToUnicodeTest, AlternatingFileConversions) {
     ASSERT_EQ(ada::idna::to_unicode(input), output);
   }
 }
+
+TEST_F(IdnaToUnicodeTest, BidiDomainKeepsOriginalWhenAsciiLabelFails) {
+  // "1.xn--mgb" decodes to "1." + U+0627. The digit label violates RFC 5893
+  // rule 1 because the domain is a Bidi domain, so ToUnicode keeps the input.
+  EXPECT_EQ(ada::idna::to_unicode("1.xn--mgb"), "1.xn--mgb");
+  EXPECT_EQ(ada::idna::to_unicode("xn--mgb.1a"), "xn--mgb.1a");
+  // "example" satisfies the LTR Bidi rules, so the RTL label is converted.
+  EXPECT_EQ(ada::idna::to_unicode("example.xn--mgb"), "example.\u0627");
+  EXPECT_EQ(ada::idna::to_unicode("xn--mgb.example"), "\u0627.example");
+}