Skip to content

Commit c051104

Browse files
iliaaliliaal
committed
ext/intl: Use byte offsets in IntlDateFormatter parsing
IntlDateFormatter::parse(), localtime() and parseToCalendar() passed the PHP byte offset to ICU, which expects a UTF-16 code-unit position, and returned ICU's position unchanged. With multibyte text before the date, parsing started at the wrong place and the returned offset was too small. Convert the offset in both directions as GH-23094 did for NumberFormatter, and reject an offset that splits a UTF-8 sequence. A returned position inside a surrogate pair, which a partially matched literal can produce, maps to the start of that character. The offset helpers move into intl_convert.c and NumberFormatter now uses them too.
1 parent aaa8707 commit c051104

6 files changed

Lines changed: 146 additions & 40 deletions

File tree

‎NEWS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,9 @@ PHP NEWS
6464
(DanielEScherzer, vapier)
6565

6666
- Intl:
67+
. Fixed IntlDateFormatter::parse(), localtime() and parseToCalendar()
68+
treating the offset as UTF-16 code units instead of bytes.
69+
(Ilia Alshanetsky)
6770
. Fixed cloning IntlDateFormatter and MessageFormatter losing PHP-side state
6871
such as dateType, timeType, calendar and the message pattern.
6972
(Ilia Alshanetsky)

‎ext/intl/dateformat/dateformat_parse.c‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,29 @@ static void internal_parse_to_timestamp(IntlDateFormatter_object *dfo, char* tex
4141
/* Convert timezone to UTF-16. */
4242
intl_convert_utf8_to_utf16(&text_utf16, &text_utf16_len, text_to_parse, text_len, &INTL_DATA_ERROR_CODE(dfo));
4343
INTL_METHOD_CHECK_STATUS(dfo, "Error converting timezone to UTF-16" );
44+
if (parse_pos && !intl_convert_utf8_offset_to_utf16(text_to_parse, text_len, parse_pos, &INTL_DATA_ERROR_CODE(dfo))) {
45+
if (text_utf16) {
46+
efree(text_utf16);
47+
}
48+
INTL_METHOD_CHECK_STATUS(dfo, "Invalid UTF-8 offset" );
49+
}
4450

4551
if (UNEXPECTED(update_calendar)) {
4652
UCalendar *parsed_calendar = (UCalendar *)udat_getCalendar(DATE_FORMAT_OBJECT(dfo));
4753
udat_parseCalendar(DATE_FORMAT_OBJECT(dfo), parsed_calendar, text_utf16, text_utf16_len, parse_pos, &INTL_DATA_ERROR_CODE(dfo));
54+
if (parse_pos) {
55+
*parse_pos = intl_convert_utf16_offset_to_utf8(text_utf16, text_utf16_len, *parse_pos);
56+
}
4857
if (text_utf16) {
4958
efree(text_utf16);
5059
}
5160
INTL_METHOD_CHECK_STATUS( dfo, "Calendar parsing failed" );
5261
timestamp = ucal_getMillis( parsed_calendar, &INTL_DATA_ERROR_CODE(dfo));
5362
} else {
5463
timestamp = udat_parse(DATE_FORMAT_OBJECT(dfo), text_utf16, text_utf16_len, parse_pos, &INTL_DATA_ERROR_CODE(dfo));
64+
if (parse_pos) {
65+
*parse_pos = intl_convert_utf16_offset_to_utf8(text_utf16, text_utf16_len, *parse_pos);
66+
}
5567
if (text_utf16) {
5668
efree(text_utf16);
5769
}
@@ -95,9 +107,18 @@ static void internal_parse_to_localtime(IntlDateFormatter_object *dfo, char* tex
95107
/* Convert timezone to UTF-16. */
96108
intl_convert_utf8_to_utf16(&text_utf16, &text_utf16_len, text_to_parse, text_len, &INTL_DATA_ERROR_CODE(dfo));
97109
INTL_METHOD_CHECK_STATUS(dfo, "Error converting timezone to UTF-16" );
110+
if (parse_pos && !intl_convert_utf8_offset_to_utf16(text_to_parse, text_len, parse_pos, &INTL_DATA_ERROR_CODE(dfo))) {
111+
if (text_utf16) {
112+
efree(text_utf16);
113+
}
114+
INTL_METHOD_CHECK_STATUS(dfo, "Invalid UTF-8 offset" );
115+
}
98116

99117
parsed_calendar = (UCalendar *)udat_getCalendar(DATE_FORMAT_OBJECT(dfo));
100118
udat_parseCalendar( DATE_FORMAT_OBJECT(dfo), parsed_calendar, text_utf16, text_utf16_len, parse_pos, &INTL_DATA_ERROR_CODE(dfo));
119+
if (parse_pos) {
120+
*parse_pos = intl_convert_utf16_offset_to_utf8(text_utf16, text_utf16_len, *parse_pos);
121+
}
101122

102123
if (text_utf16) {
103124
efree(text_utf16);

‎ext/intl/formatter/formatter_parse.c‎

Lines changed: 4 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -27,42 +27,6 @@
2727

2828
#define ICU_LOCALE_BUG 1
2929

30-
static bool numfmt_utf8_offset_to_utf16(const char *str, size_t str_len, int32_t *position, UErrorCode *status)
31-
{
32-
int32_t utf16_position;
33-
34-
if (*position < 0 || (size_t) *position > str_len) {
35-
return true;
36-
}
37-
38-
*status = U_ZERO_ERROR;
39-
u_strFromUTF8(NULL, 0, &utf16_position, str, *position, status);
40-
if (*status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(*status)) {
41-
return false;
42-
}
43-
*status = U_ZERO_ERROR;
44-
45-
*position = utf16_position;
46-
return true;
47-
}
48-
49-
static int32_t numfmt_utf16_offset_to_utf8(const UChar *str, int32_t str_len, int32_t position)
50-
{
51-
int32_t utf8_position;
52-
UErrorCode status = U_ZERO_ERROR;
53-
54-
if (position < 0 || position > str_len) {
55-
return position;
56-
}
57-
58-
u_strToUTF8(NULL, 0, &utf8_position, str, position, &status);
59-
if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) {
60-
return position;
61-
}
62-
63-
return utf8_position;
64-
}
65-
6630
/* {{{ Parse a number. */
6731
PHP_FUNCTION( numfmt_parse )
6832
{
@@ -97,7 +61,7 @@ PHP_FUNCTION( numfmt_parse )
9761
/* Convert given string to UTF-16. */
9862
intl_convert_utf8_to_utf16(&sstr, &sstr_len, str, str_len, &INTL_DATA_ERROR_CODE(nfo));
9963
INTL_METHOD_CHECK_STATUS( nfo, "String conversion to UTF-16 failed" );
100-
if (zposition && !numfmt_utf8_offset_to_utf16(str, str_len, &position, &INTL_DATA_ERROR_CODE(nfo))) {
64+
if (zposition && !intl_convert_utf8_offset_to_utf16(str, str_len, &position, &INTL_DATA_ERROR_CODE(nfo))) {
10165
efree(sstr);
10266
INTL_METHOD_CHECK_STATUS(nfo, "Invalid UTF-8 offset");
10367
}
@@ -141,7 +105,7 @@ PHP_FUNCTION( numfmt_parse )
141105
}
142106

143107
if (zposition) {
144-
position = numfmt_utf16_offset_to_utf8(sstr, sstr_len, position);
108+
position = intl_convert_utf16_offset_to_utf8(sstr, sstr_len, position);
145109
ZEND_TRY_ASSIGN_REF_LONG(zposition, position);
146110
}
147111

@@ -191,7 +155,7 @@ PHP_FUNCTION( numfmt_parse_currency )
191155

192156
if(zposition) {
193157
position = (int32_t) zval_get_long(zposition);
194-
if (!numfmt_utf8_offset_to_utf16(str, str_len, &position, &INTL_DATA_ERROR_CODE(nfo))) {
158+
if (!intl_convert_utf8_offset_to_utf16(str, str_len, &position, &INTL_DATA_ERROR_CODE(nfo))) {
195159
efree(sstr);
196160
INTL_METHOD_CHECK_STATUS(nfo, "Invalid UTF-8 offset");
197161
}
@@ -200,7 +164,7 @@ PHP_FUNCTION( numfmt_parse_currency )
200164

201165
number = unum_parseDoubleCurrency(FORMATTER_OBJECT(nfo), sstr, sstr_len, position_p, currency, &INTL_DATA_ERROR_CODE(nfo));
202166
if(zposition) {
203-
position = numfmt_utf16_offset_to_utf8(sstr, sstr_len, position);
167+
position = intl_convert_utf16_offset_to_utf8(sstr, sstr_len, position);
204168
ZEND_TRY_ASSIGN_REF_LONG(zposition, position);
205169
}
206170
if (sstr) {

‎ext/intl/intl_convert.c‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
#include "intl_common.h"
2323
#include "intl_convert.h"
2424

25+
#include <unicode/utf16.h>
26+
2527
/* {{{ intl_convert_utf8_to_utf16
2628
* Convert given string from UTF-8 to UTF-16 to *target buffer.
2729
*
@@ -150,3 +152,43 @@ zend_string* intl_convert_utf16_to_utf8(
150152
return dst;
151153
}
152154
/* }}} */
155+
156+
bool intl_convert_utf8_offset_to_utf16(const char *str, size_t str_len, int32_t *position, UErrorCode *status)
157+
{
158+
int32_t utf16_position;
159+
160+
if (*position < 0 || (size_t) *position > str_len) {
161+
return true;
162+
}
163+
164+
*status = U_ZERO_ERROR;
165+
u_strFromUTF8(NULL, 0, &utf16_position, str, *position, status);
166+
if (*status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(*status)) {
167+
return false;
168+
}
169+
*status = U_ZERO_ERROR;
170+
171+
*position = utf16_position;
172+
return true;
173+
}
174+
175+
int32_t intl_convert_utf16_offset_to_utf8(const UChar *str, int32_t str_len, int32_t position)
176+
{
177+
int32_t utf8_position;
178+
UErrorCode status = U_ZERO_ERROR;
179+
180+
if (position < 0 || position > str_len) {
181+
return position;
182+
}
183+
184+
if (position > 0 && position < str_len && U16_IS_LEAD(str[position - 1]) && U16_IS_TRAIL(str[position])) {
185+
position--;
186+
}
187+
188+
u_strToUTF8(NULL, 0, &utf8_position, str, position, &status);
189+
if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) {
190+
return position;
191+
}
192+
193+
return utf8_position;
194+
}

‎ext/intl/intl_convert.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,12 @@ zend_string* intl_convert_utf16_to_utf8(
2727
const UChar* src, int32_t src_len,
2828
UErrorCode* status );
2929

30+
bool intl_convert_utf8_offset_to_utf16(
31+
const char* str, size_t str_len,
32+
int32_t* position, UErrorCode* status );
33+
34+
int32_t intl_convert_utf16_offset_to_utf8(
35+
const UChar* str, int32_t str_len,
36+
int32_t position );
37+
3038
#endif // INTL_CONVERT_H

‎ext/intl/tests/dateformat_parse_utf8_offset.phpt‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
--TEST--
2+
IntlDateFormatter parsing uses UTF-8 byte offsets
3+
--EXTENSIONS--
4+
intl
5+
--FILE--
6+
<?php
7+
$formatter = new IntlDateFormatter('en_US', IntlDateFormatter::NONE, IntlDateFormatter::NONE, 'UTC', IntlDateFormatter::GREGORIAN, 'yyyy-MM-dd');
8+
$prefix = "\u{1F600}";
9+
$text = $prefix . '2017-10-12 tail';
10+
11+
echo "parse():\n";
12+
$offset = strlen($prefix);
13+
var_dump(gmdate('Y-m-d', $formatter->parse($text, $offset)), $offset);
14+
15+
echo "localtime():\n";
16+
$offset = strlen($prefix);
17+
$tm = $formatter->localtime($text, $offset);
18+
var_dump($tm['tm_year'] + 1900, $tm['tm_mon'] + 1, $tm['tm_mday'], $offset);
19+
20+
echo "parseToCalendar():\n";
21+
$offset = strlen($prefix);
22+
var_dump(gmdate('Y-m-d', $formatter->parseToCalendar($text, $offset)), $offset);
23+
24+
echo "failed parse:\n";
25+
$offset = strlen($prefix);
26+
var_dump($formatter->parse($prefix . '2017-xx-12', $offset), $offset);
27+
28+
echo "offset inside a character:\n";
29+
$offset = 1;
30+
var_dump($formatter->parse($text, $offset), $offset, intl_get_error_message());
31+
32+
echo "partial match inside a surrogate pair:\n";
33+
$formatter = new IntlDateFormatter('en_US', IntlDateFormatter::NONE, IntlDateFormatter::NONE, 'UTC', IntlDateFormatter::GREGORIAN, "\u{20AC}\u{1F600}yyyy-MM-dd");
34+
$offset = 0;
35+
var_dump($formatter->parse("\u{20AC}\u{1F601}2017-10-12", $offset), $offset);
36+
$formatter = new IntlDateFormatter('en_US', IntlDateFormatter::NONE, IntlDateFormatter::NONE, 'UTC', IntlDateFormatter::GREGORIAN, "yyyy-MM-dd\u{1F600}");
37+
foreach (['parse', 'localtime', 'parseToCalendar'] as $method) {
38+
$offset = 0;
39+
$formatter->$method("2017-10-12\u{1F601}", $offset);
40+
echo "$method(): ";
41+
var_dump($offset);
42+
}
43+
?>
44+
--EXPECT--
45+
parse():
46+
string(10) "2017-10-12"
47+
int(14)
48+
localtime():
49+
int(2017)
50+
int(10)
51+
int(12)
52+
int(14)
53+
parseToCalendar():
54+
string(10) "2017-10-12"
55+
int(14)
56+
failed parse:
57+
bool(false)
58+
int(9)
59+
offset inside a character:
60+
bool(false)
61+
int(1)
62+
string(42) "Invalid UTF-8 offset: U_INVALID_CHAR_FOUND"
63+
partial match inside a surrogate pair:
64+
bool(false)
65+
int(3)
66+
parse(): int(10)
67+
localtime(): int(10)
68+
parseToCalendar(): int(10)

0 commit comments

Comments
 (0)