From 10a7a2e447aa88aa64b9bdbe426fd24d9973cfd7 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Sun, 15 Nov 2026 16:23:42 -0500 Subject: [PATCH 1/3] ext/mbstring: Faster UTF-8 validation following simdutf Use the "lookup" algorithm of Keiser and Lemire, "Validating UTF-8 In Less Than One Instruction Per Byte", Software: Practice and Experience 51 (5), 2021, as implemented in the simdutf library, for mb_check_encoding() and the other functions which validate UTF-8: - AVX2: full lookup algorithm (the previous version only adopted part of it), errors accumulated over blocks, 64-byte ASCII fast path, and no copying for the tail of the string or for short strings. - AVX-512 (new): selected at runtime when available, on x86-64. - ARM64: NEON implementation of the same algorithm, replacing the emulated SSE2 code. Long strings are validated about twice as fast (UTF-8 text) to three times as fast (ASCII); short strings are not slower. --- ext/mbstring/mbstring.c | 660 +++++++++++------- .../tests/mb_check_encoding_utf8_blocks.phpt | 46 ++ 2 files changed, 440 insertions(+), 266 deletions(-) create mode 100644 ext/mbstring/tests/mb_check_encoding_utf8_blocks.phpt diff --git a/ext/mbstring/mbstring.c b/ext/mbstring/mbstring.c index 16e969a3ea32..fe65b9989463 100644 --- a/ext/mbstring/mbstring.c +++ b/ext/mbstring/mbstring.c @@ -5105,15 +5105,192 @@ MBSTRING_API bool php_mb_check_encoding(const char *input, size_t length, const # define MBSTRING_BROKEN_X86_MSVC_INTRINSICS #endif +#if defined(ZEND_INTRIN_AVX2_NATIVE) || defined(ZEND_INTRIN_AVX2_RESOLVER) || defined(__aarch64__) || defined(_M_ARM64) + +/* UTF-8 validation following the "lookup" algorithm of + * John Keiser and Daniel Lemire, "Validating UTF-8 In Less Than One Instruction Per Byte", + * Software: Practice and Experience 51 (5), 2021. https://doi.org/10.1002/spe.2940 + * The code is adapted from the simdutf library (https://github.com/simdutf/simdutf). + * + * Each byte is checked together with the byte which precedes it: three table lookups, + * on the high nibble of the previous byte, the low nibble of the previous byte and the + * high nibble of the current byte, each return a set of possible errors. A bit which is + * set in all three results is an actual error. The only error which cannot be detected + * from two successive bytes is a missing third or fourth byte of a multi-byte sequence; + * it is found by checking the bytes two and three positions back. */ +#define MB_UTF8_TOO_SHORT (1 << 0) /* 11______ 0_______ or 11______ 11______ */ +#define MB_UTF8_TOO_LONG (1 << 1) /* 0_______ 10______ */ +#define MB_UTF8_OVERLONG_3 (1 << 2) /* 11100000 100_____ */ +#define MB_UTF8_TOO_LARGE (1 << 3) /* 11110100 1001____ and above */ +#define MB_UTF8_SURROGATE (1 << 4) /* 11101101 101_____ */ +#define MB_UTF8_OVERLONG_2 (1 << 5) /* 1100000_ 10______ */ +#define MB_UTF8_TOO_LARGE_1000 (1 << 6) /* 11110101 1000____ and above */ +#define MB_UTF8_OVERLONG_4 (1 << 6) /* 11110000 1000____ */ +/* Bit 7 (0x80); written as -128 since _mm_setr_epi8 and friends take signed chars */ +#define MB_UTF8_TWO_CONTS (-128) /* 10______ 10______ */ +#define MB_UTF8_CARRY (MB_UTF8_TOO_SHORT | MB_UTF8_TOO_LONG | MB_UTF8_TWO_CONTS) + +/* Lookup table indexed by the high nibble of the previous byte */ +#define MB_UTF8_BYTE_1_HIGH \ + MB_UTF8_TOO_LONG, MB_UTF8_TOO_LONG, MB_UTF8_TOO_LONG, MB_UTF8_TOO_LONG, \ + MB_UTF8_TOO_LONG, MB_UTF8_TOO_LONG, MB_UTF8_TOO_LONG, MB_UTF8_TOO_LONG, \ + MB_UTF8_TWO_CONTS, MB_UTF8_TWO_CONTS, MB_UTF8_TWO_CONTS, MB_UTF8_TWO_CONTS, \ + MB_UTF8_TOO_SHORT | MB_UTF8_OVERLONG_2, \ + MB_UTF8_TOO_SHORT, \ + MB_UTF8_TOO_SHORT | MB_UTF8_OVERLONG_3 | MB_UTF8_SURROGATE, \ + MB_UTF8_TOO_SHORT | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000 | MB_UTF8_OVERLONG_4 + +/* Lookup table indexed by the low nibble of the previous byte */ +#define MB_UTF8_BYTE_1_LOW \ + MB_UTF8_CARRY | MB_UTF8_OVERLONG_3 | MB_UTF8_OVERLONG_2 | MB_UTF8_OVERLONG_4, \ + MB_UTF8_CARRY | MB_UTF8_OVERLONG_2, \ + MB_UTF8_CARRY, \ + MB_UTF8_CARRY, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000 | MB_UTF8_SURROGATE, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000, \ + MB_UTF8_CARRY | MB_UTF8_TOO_LARGE | MB_UTF8_TOO_LARGE_1000 + +/* Lookup table indexed by the high nibble of the current byte */ +#define MB_UTF8_BYTE_2_HIGH \ + MB_UTF8_TOO_SHORT, MB_UTF8_TOO_SHORT, MB_UTF8_TOO_SHORT, MB_UTF8_TOO_SHORT, \ + MB_UTF8_TOO_SHORT, MB_UTF8_TOO_SHORT, MB_UTF8_TOO_SHORT, MB_UTF8_TOO_SHORT, \ + MB_UTF8_TOO_LONG | MB_UTF8_OVERLONG_2 | MB_UTF8_TWO_CONTS | MB_UTF8_OVERLONG_3 | MB_UTF8_TOO_LARGE_1000 | MB_UTF8_OVERLONG_4, \ + MB_UTF8_TOO_LONG | MB_UTF8_OVERLONG_2 | MB_UTF8_TWO_CONTS | MB_UTF8_OVERLONG_3 | MB_UTF8_TOO_LARGE, \ + MB_UTF8_TOO_LONG | MB_UTF8_OVERLONG_2 | MB_UTF8_TWO_CONTS | MB_UTF8_SURROGATE | MB_UTF8_TOO_LARGE, \ + MB_UTF8_TOO_LONG | MB_UTF8_OVERLONG_2 | MB_UTF8_TWO_CONTS | MB_UTF8_SURROGATE | MB_UTF8_TOO_LARGE, \ + MB_UTF8_TOO_SHORT, MB_UTF8_TOO_SHORT, MB_UTF8_TOO_SHORT, MB_UTF8_TOO_SHORT + +/* mb_utf8_keep_mask[i] is 0xFF if i >= 64, 0 otherwise; loading a vector from + * mb_utf8_keep_mask + k gives a mask which clears its first 64 - k bytes */ +static const unsigned char mb_utf8_keep_mask[96] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF +}; + +#endif + /* If we are building an AVX2-only binary, don't compile the next function */ #ifndef ZEND_INTRIN_AVX2_NATIVE -/* SSE2-based function for validating UTF-8 strings - * A faster implementation which uses AVX2 instructions follows */ +/* Function for validating UTF-8 strings, using NEON on ARM64 or SSE2 on x86 + * Faster implementations which use AVX2 or AVX-512 instructions follow */ static bool mb_fast_check_utf8_default(zend_string *str) { unsigned char *p = (unsigned char*)ZSTR_VAL(str); -# ifdef XSSE2 +# if defined(__aarch64__) || defined(_M_ARM64) + /* NEON version of the "lookup" algorithm; see mb_fast_check_utf8_avx2 */ +# define MB_UTF8_CHECK_NEON(input, prev) do { \ + uint8x16_t prev1 = vextq_u8((prev), (input), 15); \ + uint8x16_t sc = vandq_u8(vandq_u8( \ + vqtbl1q_u8(byte_1_high, vshrq_n_u8(prev1, 4)), \ + vqtbl1q_u8(byte_1_low, vandq_u8(prev1, v_0f))), \ + vqtbl1q_u8(byte_2_high, vshrq_n_u8((input), 4))); \ + uint8x16_t must23 = veorq_u8( \ + vcgeq_u8(vextq_u8((prev), (input), 14), v_e0), \ + vcgeq_u8(vextq_u8((prev), (input), 13), v_f0)); \ + error = vorrq_u8(error, veorq_u8(vandq_u8(must23, v_80), sc)); \ + } while (0) + + static const int8_t byte_1_high_table[16] = { MB_UTF8_BYTE_1_HIGH }; + static const int8_t byte_1_low_table[16] = { MB_UTF8_BYTE_1_LOW }; + static const int8_t byte_2_high_table[16] = { MB_UTF8_BYTE_2_HIGH }; + /* A block ending with any of these is incomplete: ... 1111____ 111_____ 11______ */ + static const uint8_t max_value_table[16] = { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0xF0 - 1, 0xE0 - 1, 0xC0 - 1 }; + + const unsigned char *e = p + ZSTR_LEN(str); + const uint8x16_t byte_1_high = vreinterpretq_u8_s8(vld1q_s8(byte_1_high_table)); + const uint8x16_t byte_1_low = vreinterpretq_u8_s8(vld1q_s8(byte_1_low_table)); + const uint8x16_t byte_2_high = vreinterpretq_u8_s8(vld1q_s8(byte_2_high_table)); + const uint8x16_t max_value = vld1q_u8(max_value_table); + const uint8x16_t v_0f = vdupq_n_u8(0x0F); + const uint8x16_t v_80 = vdupq_n_u8(0x80); + const uint8x16_t v_e0 = vdupq_n_u8(0xE0); + const uint8x16_t v_f0 = vdupq_n_u8(0xF0); + + uint8x16_t error = vdupq_n_u8(0); + uint8x16_t prev_input = vdupq_n_u8(0); + uint8x16_t prev_incomplete = vdupq_n_u8(0); + + while (e - p >= 64) { + /* The error vector is sticky, so it suffices to test it once in a while */ + const unsigned char *stop = (e - p >= 1024 + 64) ? p + 1024 : e - 63; + do { + uint8x16_t in0 = vld1q_u8(p); + uint8x16_t in1 = vld1q_u8(p + 16); + uint8x16_t in2 = vld1q_u8(p + 32); + uint8x16_t in3 = vld1q_u8(p + 48); + if (vmaxvq_u8(vorrq_u8(vorrq_u8(in0, in1), vorrq_u8(in2, in3))) < 0x80) { + /* All ASCII; only need to check that the previous block was complete */ + error = vorrq_u8(error, prev_incomplete); + } else { + MB_UTF8_CHECK_NEON(in0, prev_input); + MB_UTF8_CHECK_NEON(in1, in0); + MB_UTF8_CHECK_NEON(in2, in1); + MB_UTF8_CHECK_NEON(in3, in2); + prev_incomplete = vqsubq_u8(in3, max_value); + prev_input = in3; + } + p += 64; + } while (p < stop); + if (vmaxvq_u8(error)) { + return false; + } + } + + while (e - p >= 16) { + uint8x16_t in = vld1q_u8(p); + if (vmaxvq_u8(in) < 0x80) { + error = vorrq_u8(error, prev_incomplete); + } else { + MB_UTF8_CHECK_NEON(in, prev_input); + prev_incomplete = vqsubq_u8(in, max_value); + prev_input = in; + } + p += 16; + } + + if (p < e) { + /* Check the last 16 bytes of the string, using the 16 bytes before them as context. + * Some of these bytes may have been checked already; checking them again is harmless. */ + size_t len = ZSTR_LEN(str); + uint8x16_t in, prev; + if (len >= 32) { + in = vld1q_u8(e - 16); + prev = vld1q_u8(e - 32); + } else { + /* Short string: we may read up to 16 bytes before the start of the string, which + * belong to the zend_string header, and clear them */ + in = vld1q_u8(e - 16); + if (len >= 16) { + prev = vandq_u8(vld1q_u8(e - 32), vld1q_u8(mb_utf8_keep_mask + 32 + len)); + } else { + in = vandq_u8(in, vld1q_u8(mb_utf8_keep_mask + 48 + len)); + prev = vdupq_n_u8(0); + } + } + if (vmaxvq_u8(in) < 0x80) { + error = vorrq_u8(error, prev_incomplete); + } else { + MB_UTF8_CHECK_NEON(in, prev); + prev_incomplete = vqsubq_u8(in, max_value); + } + } + + error = vorrq_u8(error, prev_incomplete); + return vmaxvq_u8(error) == 0; +# undef MB_UTF8_CHECK_NEON +# elif defined(XSSE2) /* `e` points 1 byte past the last full 16-byte block of string content * Note that we include the terminating null byte which is included in each zend_string * as part of the content to check; this ensures that multi-byte characters which are @@ -5387,6 +5564,14 @@ static bool mb_fast_check_utf8_default(zend_string *str) #endif /* #ifndef ZEND_INTRIN_AVX2_NATIVE */ +/* Only use the AVX-512 function when the AVX2 function is also selected at runtime */ +#if (defined(__x86_64__) || defined(_M_X64)) && defined(ZEND_INTRIN_AVX2_FUNC_PROTO) && defined(ZEND_INTRIN_AVX512_FUNC_PROTO) +# define MBSTRING_INTRIN_AVX512_FUNC_PROTO 1 +#endif +#if (defined(__x86_64__) || defined(_M_X64)) && defined(ZEND_INTRIN_AVX2_FUNC_PTR) && defined(ZEND_INTRIN_AVX512_FUNC_PTR) +# define MBSTRING_INTRIN_AVX512_FUNC_PTR 1 +#endif + #ifdef ZEND_INTRIN_AVX2_NATIVE /* We are building AVX2-only binary */ @@ -5405,6 +5590,9 @@ static bool mb_fast_check_utf8_default(zend_string *str) * resolve symbols accordingly */ ZEND_INTRIN_AVX2_FUNC_DECL(bool mb_fast_check_utf8_avx2(zend_string *str)); +# ifdef MBSTRING_INTRIN_AVX512_FUNC_PROTO +ZEND_INTRIN_AVX512_FUNC_DECL(bool mb_fast_check_utf8_avx512(zend_string *str)); +# endif bool mb_fast_check_utf8(zend_string *str) __attribute__((ifunc("resolve_check_utf8"))); @@ -5414,6 +5602,11 @@ ZEND_NO_SANITIZE_ADDRESS ZEND_ATTRIBUTE_UNUSED static check_utf8_func_t resolve_check_utf8(void) { +# ifdef MBSTRING_INTRIN_AVX512_FUNC_PROTO + if (zend_cpu_supports_avx512()) { + return mb_fast_check_utf8_avx512; + } +# endif if (zend_cpu_supports_avx2()) { return mb_fast_check_utf8_avx2; } @@ -5428,8 +5621,14 @@ static check_utf8_func_t resolve_check_utf8(void) #ifdef HAVE_FUNC_ATTRIBUTE_TARGET static bool mb_fast_check_utf8_avx2(zend_string *str) __attribute__((target("avx2"))); +# ifdef MBSTRING_INTRIN_AVX512_FUNC_PTR +static bool mb_fast_check_utf8_avx512(zend_string *str) __attribute__((target("avx512f,avx512cd,avx512vl,avx512dq,avx512bw"))); +# endif #else static bool mb_fast_check_utf8_avx2(zend_string *str); +# ifdef MBSTRING_INTRIN_AVX512_FUNC_PTR +static bool mb_fast_check_utf8_avx512(zend_string *str); +# endif #endif static bool (*check_utf8_ptr)(zend_string *str) = NULL; @@ -5441,6 +5640,12 @@ static bool mb_fast_check_utf8(zend_string *str) static void init_check_utf8(void) { +#ifdef MBSTRING_INTRIN_AVX512_FUNC_PTR + if (zend_cpu_supports_avx512()) { + check_utf8_ptr = mb_fast_check_utf8_avx512; + return; + } +#endif if (zend_cpu_supports_avx2()) { check_utf8_ptr = mb_fast_check_utf8_avx2; } else { @@ -5458,295 +5663,218 @@ static void init_check_utf8(void) #if defined(ZEND_INTRIN_AVX2_NATIVE) || defined(ZEND_INTRIN_AVX2_RESOLVER) -/* GCC prior to version 8 does not define all intrinsics. See GH-11514. - * Use a workaround from https://stackoverflow.com/questions/32630458/setting-m256i-to-the-value-of-two-m128i-values */ -#if defined(__GNUC__) && !defined(__llvm__) && !defined(__INTEL_COMPILER) && __GNUC__ < 8 -# define _mm256_set_m128i(v0, v1) _mm256_insertf128_si256(_mm256_castsi128_si256(v1), (v0), 1) -#endif +/* The 32 bytes ending N bytes before the start of `input`, taken from `prev` and `input` */ +#define MB_UTF8_PREV_AVX2(input, prev, N) \ + _mm256_alignr_epi8((input), _mm256_permute2x128_si256((prev), (input), 0x21), 16 - (N)) + +/* OR the errors found in the 32-byte block `input` into `error`; `prev` is the preceding block */ +#define MB_UTF8_CHECK_AVX2(input, prev) do { \ + __m256i prev1 = MB_UTF8_PREV_AVX2(input, prev, 1); \ + __m256i sc = _mm256_and_si256(_mm256_and_si256( \ + _mm256_shuffle_epi8(byte_1_high, _mm256_and_si256(_mm256_srli_epi16(prev1, 4), v_0f)), \ + _mm256_shuffle_epi8(byte_1_low, _mm256_and_si256(prev1, v_0f))), \ + _mm256_shuffle_epi8(byte_2_high, _mm256_and_si256(_mm256_srli_epi16(input, 4), v_0f))); \ + /* A byte must be a continuation byte if it is the 3rd byte of a 3/4-byte sequence or \ + * the 4th byte of a 4-byte sequence; only such positions get the high bit set here */ \ + __m256i must23 = _mm256_or_si256( \ + _mm256_subs_epu8(MB_UTF8_PREV_AVX2(input, prev, 2), v_third), \ + _mm256_subs_epu8(MB_UTF8_PREV_AVX2(input, prev, 3), v_fourth)); \ + error = _mm256_or_si256(error, _mm256_xor_si256(_mm256_and_si256(must23, v_80), sc)); \ + } while (0) -/* Take (256-bit) `hi` and `lo` as a 512-bit value, shift down by some - * number of bytes, then take the low 256 bits - * This is used to take some number of trailing bytes from the previous 32-byte - * block followed by some number of leading bytes from the current 32-byte block - * - * _mm256_alignr_epi8 (VPALIGNR) is used to shift out bytes from a 256-bit - * YMM register while shifting in bytes from another YMM register... but - * it works separately on respective 128-bit halves of the YMM registers, - * which is not what we want. - * To make it work as desired, we first do _mm256_permute2x128_si256 - * (VPERM2I128) to combine the low 128 bits from the previous block and - * the high 128 bits of the current block in one YMM register. - * Then VPALIGNR will do what is needed. */ -#define _mm256_shift_epi8(hi, lo, shift) _mm256_alignr_epi8(lo, _mm256_permute2x128_si256(hi, lo, 33), 16 - shift) - -/* AVX2-based UTF-8 validation function; validates text in 32-byte chunks - * - * Some parts of this function are the same as `mb_fast_check_utf8`; code comments - * are not repeated, so consult `mb_fast_check_utf8` for information on uncommented - * sections. */ #ifdef ZEND_INTRIN_AVX2_FUNC_PROTO ZEND_API bool mb_fast_check_utf8_avx2(zend_string *str) #else static bool mb_fast_check_utf8_avx2(zend_string *str) #endif { - unsigned char *p = (unsigned char*)ZSTR_VAL(str); - unsigned char *e = p + ((ZSTR_LEN(str) + 1) & ~(sizeof(__m256i) - 1)); - - /* The algorithm used here for UTF-8 validation is partially adapted from the - * paper "Validating UTF-8 In Less Than One Instruction Per Byte", by John Keiser - * and Daniel Lemire. - * Ref: https://arxiv.org/pdf/2010.03090.pdf - * - * Most types of invalid UTF-8 text can be detected by examining pairs of - * successive bytes. Specifically: - * - * • Overlong 2-byte code units start with 0xC0 or 0xC1. - * No valid UTF-8 string ever uses these byte values. - * • Overlong 3-byte code units start with 0xE0, followed by a byte < 0xA0. - * • Overlong 4-byte code units start with 0xF0, followed by a byte < 0x90. - * • 5-byte or 6-byte code units, which should never be used, start with - * 0xF8-FE. - * • A codepoint value higher than U+10FFFF, which is the highest value for - * any Unicode codepoint, would either start with 0xF4, followed by a - * byte >= 0x90, or else would start with 0xF5-F7, followed by any value. - * • A codepoint value from U+D800-DFFF, which are reserved and should never - * be used, would start with 0xED, followed by a byte >= 0xA0. - * • The byte value 0xFF is also illegal and is never used in valid UTF-8. - * - * To detect all these problems, for each pair of successive bytes, we do - * table lookups using the high nibble of the first byte, the low nibble of - * the first byte, and the high nibble of the second byte. Each table lookup - * retrieves a bitmask, in which each 1 bit indicates a possible invalid - * combination; AND those three bitmasks together, and any 1 bit in the result - * will indicate an actual invalid byte combination was found. - */ - -#define BAD_BYTE 0x1 -#define OVERLONG_2BYTE 0x2 -#define _1BYTE (BAD_BYTE | OVERLONG_2BYTE) -#define OVERLONG_3BYTE 0x4 -#define SURROGATE 0x8 -#define OVERLONG_4BYTE 0x10 -#define INVALID_CP 0x20 - - /* Each of these are 16-entry tables, repeated twice; this is required by the - * VPSHUFB instruction which we use to perform 32 table lookups in parallel - * The first entry is for 0xF, the second is for 0xE, and so on down to 0x0 - * - * So, for example, notice that the 4th entry in the 1st table is OVERLONG_2BYTE; - * that means that high nibble 0xC is consistent with the byte pair being part of - * an overlong 2-byte code unit */ - const __m256i bad_hi_nibble2 = _mm256_set_epi8( - BAD_BYTE | OVERLONG_4BYTE | INVALID_CP, OVERLONG_3BYTE | SURROGATE, 0, OVERLONG_2BYTE, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - BAD_BYTE | OVERLONG_4BYTE | INVALID_CP, OVERLONG_3BYTE | SURROGATE, 0, OVERLONG_2BYTE, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0); - const __m256i bad_lo_nibble2 = _mm256_set_epi8( - BAD_BYTE, BAD_BYTE, BAD_BYTE | SURROGATE, BAD_BYTE, - BAD_BYTE, BAD_BYTE, BAD_BYTE, BAD_BYTE, - BAD_BYTE, BAD_BYTE, BAD_BYTE, INVALID_CP, - 0, 0, OVERLONG_2BYTE, OVERLONG_2BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, - BAD_BYTE, BAD_BYTE, BAD_BYTE | SURROGATE, BAD_BYTE, - BAD_BYTE, BAD_BYTE, BAD_BYTE, BAD_BYTE, - BAD_BYTE, BAD_BYTE, BAD_BYTE, INVALID_CP, - 0, 0, OVERLONG_2BYTE, OVERLONG_2BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE); - const __m256i bad_hi_nibble = _mm256_set_epi8( - _1BYTE | SURROGATE | INVALID_CP, _1BYTE | SURROGATE | INVALID_CP, - _1BYTE | SURROGATE | INVALID_CP, _1BYTE | SURROGATE | INVALID_CP, - _1BYTE | SURROGATE | INVALID_CP, _1BYTE | SURROGATE | INVALID_CP, - _1BYTE | OVERLONG_3BYTE | INVALID_CP, _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, - _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, - _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, - _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, - _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, - _1BYTE | SURROGATE | INVALID_CP, _1BYTE | SURROGATE | INVALID_CP, - _1BYTE | SURROGATE | INVALID_CP, _1BYTE | SURROGATE | INVALID_CP, - _1BYTE | SURROGATE | INVALID_CP, _1BYTE | SURROGATE | INVALID_CP, - _1BYTE | OVERLONG_3BYTE | INVALID_CP, _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, - _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, - _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, - _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, - _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE, _1BYTE | OVERLONG_3BYTE | OVERLONG_4BYTE); - - const __m256i find_continuation = _mm256_set1_epi8(-64); - const __m256i _b = _mm256_set1_epi8(0xB); - const __m256i _d = _mm256_set1_epi8(0xD); - const __m256i _f = _mm256_set1_epi8(0xF); - - __m256i last_hi_nibbles = _mm256_setzero_si256(), last_lo_nibbles = _mm256_setzero_si256(); - __m256i operand; - - while (p < e) { - operand = _mm256_loadu_si256((__m256i*)p); - -check_operand: - if (!_mm256_movemask_epi8(operand)) { - /* Entire 32-byte block is ASCII characters; the only thing we need to validate is that - * the previous block didn't end with an incomplete multi-byte character - * (This will also confirm that the previous block didn't end with a bad byte like 0xFF) */ - __m256i bad_mask = _mm256_set_epi8(0xB, 0xD, 0xE, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127); - __m256i bad = _mm256_cmpgt_epi8(last_hi_nibbles, bad_mask); - if (_mm256_movemask_epi8(bad)) { - return false; + const unsigned char *p = (const unsigned char*)ZSTR_VAL(str); + const unsigned char *e = p + ZSTR_LEN(str); + + const __m256i byte_1_high = _mm256_setr_epi8(MB_UTF8_BYTE_1_HIGH, MB_UTF8_BYTE_1_HIGH); + const __m256i byte_1_low = _mm256_setr_epi8(MB_UTF8_BYTE_1_LOW, MB_UTF8_BYTE_1_LOW); + const __m256i byte_2_high = _mm256_setr_epi8(MB_UTF8_BYTE_2_HIGH, MB_UTF8_BYTE_2_HIGH); + const __m256i v_0f = _mm256_set1_epi8(0x0F); + const __m256i v_80 = _mm256_set1_epi8((char)0x80); + const __m256i v_third = _mm256_set1_epi8((char)(0xE0 - 0x80)); /* only 111_____ will be >= 0x80 */ + const __m256i v_fourth = _mm256_set1_epi8((char)(0xF0 - 0x80)); /* only 1111____ will be >= 0x80 */ + /* A block ending with any of these is incomplete: ... 1111____ 111_____ 11______ */ + const __m256i max_value = _mm256_setr_epi8( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + (char)(0xF0 - 1), (char)(0xE0 - 1), (char)(0xC0 - 1)); + + __m256i error = _mm256_setzero_si256(); + __m256i prev_input = _mm256_setzero_si256(); + __m256i prev_incomplete = _mm256_setzero_si256(); + + while (e - p >= 64) { + /* The error vector is sticky, so it suffices to test it once in a while */ + const unsigned char *stop = (e - p >= 1024 + 64) ? p + 1024 : e - 63; + do { + __m256i in0 = _mm256_loadu_si256((const __m256i*)p); + __m256i in1 = _mm256_loadu_si256((const __m256i*)(p + 32)); + if (!_mm256_movemask_epi8(_mm256_or_si256(in0, in1))) { + /* All ASCII; only need to check that the previous block was complete. + * `prev_input` is left alone: if it ends with an incomplete character, an error + * is recorded here; otherwise it gives the same results as ASCII context would */ + error = _mm256_or_si256(error, prev_incomplete); + } else { + MB_UTF8_CHECK_AVX2(in0, prev_input); + MB_UTF8_CHECK_AVX2(in1, in0); + prev_incomplete = _mm256_subs_epu8(in1, max_value); + prev_input = in1; } + p += 64; + } while (p < stop); + if (!_mm256_testz_si256(error, error)) { + return false; + } + } - /* Consume as many full blocks of single-byte characters as we can */ - while (true) { - p += sizeof(__m256i); - if (p >= e) { - goto finish_up_remaining_bytes; - } - operand = _mm256_loadu_si256((__m256i*)p); - if (_mm256_movemask_epi8(operand)) { - break; + if (e - p >= 32) { + __m256i in = _mm256_loadu_si256((const __m256i*)p); + if (!_mm256_movemask_epi8(in)) { + error = _mm256_or_si256(error, prev_incomplete); + } else { + MB_UTF8_CHECK_AVX2(in, prev_input); + prev_incomplete = _mm256_subs_epu8(in, max_value); + prev_input = in; + } + p += 32; + } + + if (p < e) { + /* Check the last 32 bytes of the string, using the 32 bytes before them as context. + * Some of these bytes may have been checked already; checking them again is harmless. */ + size_t len = ZSTR_LEN(str); + __m256i in, prev; + if (len >= 64) { + in = _mm256_loadu_si256((const __m256i*)(e - 32)); + prev = _mm256_loadu_si256((const __m256i*)(e - 64)); + } else { + /* Short string: we may read up to 16 bytes before the start of the string, which + * belong to the zend_string header, and clear them */ + if (len >= 32) { + in = _mm256_loadu_si256((const __m256i*)(e - 32)); + /* Only the last 3 bytes of the context matter */ + uint32_t ctx; + memcpy(&ctx, e - 36, sizeof(ctx)); + prev = _mm256_inserti128_si256(_mm256_setzero_si256(), _mm_slli_si128(_mm_cvtsi32_si128(ctx), 12), 1); + prev = _mm256_and_si256(prev, _mm256_loadu_si256((const __m256i*)(mb_utf8_keep_mask + len))); + } else { + if (len >= 16) { + in = _mm256_loadu_si256((const __m256i*)(e - 32)); + } else { + in = _mm256_inserti128_si256(_mm256_setzero_si256(), _mm_loadu_si128((const __m128i*)(e - 16)), 1); } + in = _mm256_and_si256(in, _mm256_loadu_si256((const __m256i*)(mb_utf8_keep_mask + 32 + len))); + prev = _mm256_setzero_si256(); } } + if (!_mm256_movemask_epi8(in)) { + error = _mm256_or_si256(error, prev_incomplete); + prev_incomplete = _mm256_setzero_si256(); + } else { + MB_UTF8_CHECK_AVX2(in, prev); + prev_incomplete = _mm256_subs_epu8(in, max_value); + } + } - __m256i hi_nibbles = _mm256_and_si256(_mm256_srli_epi16(operand, 4), _f); - __m256i lo_nibbles = _mm256_and_si256(operand, _f); - - __m256i lo_nibbles2 = _mm256_shift_epi8(last_lo_nibbles, lo_nibbles, 1); - __m256i hi_nibbles2 = _mm256_shift_epi8(last_hi_nibbles, hi_nibbles, 1); + error = _mm256_or_si256(error, prev_incomplete); + return _mm256_testz_si256(error, error); +} - /* Do parallel table lookups in all 3 tables */ - __m256i bad = _mm256_cmpgt_epi8( - _mm256_and_si256( - _mm256_and_si256( - _mm256_shuffle_epi8(bad_lo_nibble2, lo_nibbles2), - _mm256_shuffle_epi8(bad_hi_nibble2, hi_nibbles2)), - _mm256_shuffle_epi8(bad_hi_nibble, hi_nibbles)), - _mm256_setzero_si256()); +#if defined(MBSTRING_INTRIN_AVX512_FUNC_PROTO) || defined(MBSTRING_INTRIN_AVX512_FUNC_PTR) - __m256i cont_mask = _mm256_cmpgt_epi8(hi_nibbles2, _b); - __m256i hi_nibbles3 = _mm256_shift_epi8(last_hi_nibbles, hi_nibbles, 2); - cont_mask = _mm256_or_si256(cont_mask, _mm256_cmpgt_epi8(hi_nibbles3, _d)); - __m256i hi_nibbles4 = _mm256_shift_epi8(last_hi_nibbles, hi_nibbles, 3); - cont_mask = _mm256_or_si256(cont_mask, _mm256_cmpeq_epi8(hi_nibbles4, _f)); +/* The 64 bytes ending N bytes before the start of `input`, taken from `prev` and `input` */ +#define MB_UTF8_PREV_AVX512(input, prev, N) \ + _mm512_alignr_epi8((input), _mm512_permutex2var_epi32((input), prev_shuffle, (prev)), 16 - (N)) - __m256i continuation = _mm256_cmpgt_epi8(find_continuation, operand); - bad = _mm256_or_si256(bad, _mm256_xor_si256(continuation, cont_mask)); +#define MB_UTF8_CHECK_AVX512(input, prev) do { \ + __m512i prev1 = MB_UTF8_PREV_AVX512(input, prev, 1); \ + __m512i sc = _mm512_ternarylogic_epi64( \ + _mm512_shuffle_epi8(byte_1_high, _mm512_and_si512(_mm512_srli_epi16(prev1, 4), v_0f)), \ + _mm512_shuffle_epi8(byte_1_low, _mm512_and_si512(prev1, v_0f)), \ + _mm512_shuffle_epi8(byte_2_high, _mm512_and_si512(_mm512_srli_epi16(input, 4), v_0f)), \ + 0x80); /* a & b & c */ \ + __m512i must23 = _mm512_or_si512( \ + _mm512_subs_epu8(MB_UTF8_PREV_AVX512(input, prev, 2), v_third), \ + _mm512_subs_epu8(MB_UTF8_PREV_AVX512(input, prev, 3), v_fourth)); \ + /* error |= (must23 & 0x80) ^ sc */ \ + error = _mm512_or_si512(error, _mm512_ternarylogic_epi64(must23, v_80, sc, 0x6A)); \ + } while (0) - if (_mm256_movemask_epi8(bad)) { +# ifdef MBSTRING_INTRIN_AVX512_FUNC_PROTO +ZEND_API bool mb_fast_check_utf8_avx512(zend_string *str) +# else +static bool mb_fast_check_utf8_avx512(zend_string *str) +# endif +{ + const unsigned char *p = (const unsigned char*)ZSTR_VAL(str); + const unsigned char *e = p + ZSTR_LEN(str); + + const __m512i byte_1_high = _mm512_broadcast_i32x4(_mm_setr_epi8(MB_UTF8_BYTE_1_HIGH)); + const __m512i byte_1_low = _mm512_broadcast_i32x4(_mm_setr_epi8(MB_UTF8_BYTE_1_LOW)); + const __m512i byte_2_high = _mm512_broadcast_i32x4(_mm_setr_epi8(MB_UTF8_BYTE_2_HIGH)); + const __m512i v_0f = _mm512_set1_epi8(0x0F); + const __m512i v_80 = _mm512_set1_epi8((char)0x80); + const __m512i v_third = _mm512_set1_epi8((char)(0xE0 - 0x80)); + const __m512i v_fourth = _mm512_set1_epi8((char)(0xF0 - 0x80)); + const __m512i max_value = _mm512_setr_epi64(-1, -1, -1, -1, -1, -1, -1, (int64_t)0xBFDFEFFFFFFFFFFFULL); + /* Brings the last 16 bytes of `prev` in front of the first 48 bytes of `input` */ + const __m512i prev_shuffle = _mm512_setr_epi32(28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + + __m512i error = _mm512_setzero_si512(); + __m512i prev_input = _mm512_setzero_si512(); + __m512i prev_incomplete = _mm512_setzero_si512(); + + /* Align the loads on 64-byte boundaries for long strings: check one unaligned block, then + * continue from the next boundary, taking the 3 bytes before it as context */ + uintptr_t misalignment = (uintptr_t)p % 64; + if (e - p >= 2048 && misalignment != 0 && misalignment <= 61) { + __m512i in = _mm512_loadu_si512((const void*)p); + MB_UTF8_CHECK_AVX512(in, prev_input); + p += 64 - misalignment; + prev_input = _mm512_maskz_loadu_epi8(0xE000000000000000ULL, (const void*)(p - 64)); + prev_incomplete = _mm512_subs_epu8(prev_input, max_value); + } + + while (e - p >= 64) { + const unsigned char *stop = (e - p >= 1024 + 64) ? p + 1024 : e - 63; + do { + __m512i in = _mm512_loadu_si512((const void*)p); + if (!_mm512_movepi8_mask(in)) { + error = _mm512_or_si512(error, prev_incomplete); + } else { + MB_UTF8_CHECK_AVX512(in, prev_input); + prev_incomplete = _mm512_subs_epu8(in, max_value); + prev_input = in; + } + p += 64; + } while (p < stop); + if (_mm512_test_epi8_mask(error, error)) { return false; } - - last_hi_nibbles = hi_nibbles; - last_lo_nibbles = lo_nibbles; - p += sizeof(__m256i); } -finish_up_remaining_bytes: - if (p == e) { - uint8_t remaining_bytes = ZSTR_LEN(str) & (sizeof(__m256i) - 1); /* Not including terminating null */ - - switch (remaining_bytes) { - case 0: ; - /* No actual data bytes are remaining */ - __m256i bad_mask = _mm256_set_epi8(0xB, 0xD, 0xE, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127); - __m256i bad = _mm256_cmpgt_epi8(last_hi_nibbles, bad_mask); - return _mm256_movemask_epi8(bad) == 0; - case 1: - case 2: - operand = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, *((int16_t*)p)); - goto check_operand; - case 3: - case 4: - operand = _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, *((int32_t*)p)); - goto check_operand; - case 5: - operand = _mm256_set_m128i(_mm_setzero_si128(), _mm_srli_si128(_mm_loadu_si128((__m128i*)(p - 10)), 10)); - goto check_operand; - case 6: - operand = _mm256_set_m128i(_mm_setzero_si128(), _mm_srli_si128(_mm_loadu_si128((__m128i*)(p - 9)), 9)); - goto check_operand; - case 7: - case 8: -#ifdef MBSTRING_BROKEN_X86_MSVC_INTRINSICS - operand = _mm256_set_epi32(0, 0, 0, 0, 0, 0, ((int32_t*)p)[1], ((int32_t*)p)[0]); -#else - operand = _mm256_set_epi64x(0, 0, 0, *((int64_t*)p)); -#endif - goto check_operand; - case 9: - operand = _mm256_set_m128i(_mm_setzero_si128(), _mm_srli_si128(_mm_loadu_si128((__m128i*)(p - 6)), 6)); - goto check_operand; - case 10: - operand = _mm256_set_m128i(_mm_setzero_si128(), _mm_srli_si128(_mm_loadu_si128((__m128i*)(p - 5)), 5)); - goto check_operand; - case 11: - operand = _mm256_set_m128i(_mm_setzero_si128(), _mm_srli_si128(_mm_loadu_si128((__m128i*)(p - 4)), 4)); - goto check_operand; - case 12: - operand = _mm256_set_m128i(_mm_setzero_si128(), _mm_srli_si128(_mm_loadu_si128((__m128i*)(p - 3)), 3)); - goto check_operand; - case 13: - operand = _mm256_set_m128i(_mm_setzero_si128(), _mm_srli_si128(_mm_loadu_si128((__m128i*)(p - 2)), 2)); - goto check_operand; - case 14: - operand = _mm256_set_m128i(_mm_setzero_si128(), _mm_srli_si128(_mm_loadu_si128((__m128i*)(p - 1)), 1)); - goto check_operand; - case 15: - case 16: - operand = _mm256_set_m128i(_mm_setzero_si128(), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 17: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 2)), 14), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 18: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 3)), 13), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 19: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 4)), 12), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 20: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 5)), 11), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 21: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 6)), 10), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 22: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 7)), 9), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 23: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 8)), 8), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 24: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 9)), 7), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 25: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 10)), 6), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 26: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 11)), 5), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 27: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 12)), 4), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 28: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 13)), 3), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 29: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 14)), 2), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 30: - operand = _mm256_set_m128i(_mm_srli_si128(_mm_loadu_si128((__m128i*)(p + 15)), 1), _mm_loadu_si128((__m128i*)p)); - goto check_operand; - case 31: - return true; + if (p < e) { + /* Bytes past the end of the string are loaded as zero (ASCII NUL) */ + __m512i in = _mm512_maskz_loadu_epi8(~0ULL >> (64 - (e - p)), (const void*)p); + if (!_mm512_movepi8_mask(in)) { + error = _mm512_or_si512(error, prev_incomplete); + } else { + MB_UTF8_CHECK_AVX512(in, prev_input); + prev_incomplete = _mm512_subs_epu8(in, max_value); } - - ZEND_UNREACHABLE(); } - return true; + error = _mm512_or_si512(error, prev_incomplete); + return !_mm512_test_epi8_mask(error, error); } +#endif /* defined(MBSTRING_INTRIN_AVX512_FUNC_PROTO) || defined(MBSTRING_INTRIN_AVX512_FUNC_PTR) */ + #endif /* defined(ZEND_INTRIN_AVX2_NATIVE) || defined(ZEND_INTRIN_AVX2_RESOLVER) */ static bool mb_check_str_encoding(zend_string *str, const mbfl_encoding *encoding) diff --git a/ext/mbstring/tests/mb_check_encoding_utf8_blocks.phpt b/ext/mbstring/tests/mb_check_encoding_utf8_blocks.phpt new file mode 100644 index 000000000000..9d6b32e38e16 --- /dev/null +++ b/ext/mbstring/tests/mb_check_encoding_utf8_blocks.phpt @@ -0,0 +1,46 @@ +--TEST-- +mb_check_encoding() with UTF-8: errors at every position around SIMD block boundaries +--EXTENSIONS-- +mbstring +--FILE-- + 100000 ? "enough cases\n" : "too few cases: $checked\n"; +echo "mismatches: $mismatches\n"; +?> +--EXPECT-- +enough cases +mismatches: 0 From 024248d440af40dfef42103021696f6dad08dea1 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Sun, 15 Nov 2026 16:57:58 -0500 Subject: [PATCH 2/3] ext/mbstring: Fix UTF-8 validation review notes The new test called preg_match('//u') before mb_check_encoding(). That sets IS_STR_VALID_UTF8, and str_repeat($s, 1) returns the same string, so valid input never reached the new kernels. Copy the bytes with pack() first, and cover strings longer than 2048 bytes. The AVX-512 alignment prologue loaded context through a pointer before the zend_string. Read the three bytes from inside the string. OR the NEON continuation checks, matching AVX2. Record the speedup in NEWS. --- NEWS | 2 ++ ext/mbstring/mbstring.c | 17 ++++++++-- .../tests/mb_check_encoding_utf8_blocks.phpt | 31 ++++++++++++++++++- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 94d00d96dffb..486178e2101a 100644 --- a/NEWS +++ b/NEWS @@ -65,6 +65,8 @@ PHP NEWS (lacatoire) - MBString: + . Improved performance of UTF-8 validation in mb_check_encoding(), + mb_strlen(), mb_substr(), and related functions. (Daniel Lemire) . Fixed bug GH-23106 (mb_strpos() reads past the end of a haystack ending in a truncated UTF-8 sequence). (Lazizbek Ergashev) . Updated Unicode data tables to Unicode 18.0 (Yuya Hamada) diff --git a/ext/mbstring/mbstring.c b/ext/mbstring/mbstring.c index fe65b9989463..3312cd706dbf 100644 --- a/ext/mbstring/mbstring.c +++ b/ext/mbstring/mbstring.c @@ -5196,7 +5196,7 @@ static bool mb_fast_check_utf8_default(zend_string *str) vqtbl1q_u8(byte_1_high, vshrq_n_u8(prev1, 4)), \ vqtbl1q_u8(byte_1_low, vandq_u8(prev1, v_0f))), \ vqtbl1q_u8(byte_2_high, vshrq_n_u8((input), 4))); \ - uint8x16_t must23 = veorq_u8( \ + uint8x16_t must23 = vorrq_u8( \ vcgeq_u8(vextq_u8((prev), (input), 14), v_e0), \ vcgeq_u8(vextq_u8((prev), (input), 13), v_f0)); \ error = vorrq_u8(error, veorq_u8(vandq_u8(must23, v_80), sc)); \ @@ -5821,7 +5821,13 @@ static bool mb_fast_check_utf8_avx512(zend_string *str) const __m512i v_80 = _mm512_set1_epi8((char)0x80); const __m512i v_third = _mm512_set1_epi8((char)(0xE0 - 0x80)); const __m512i v_fourth = _mm512_set1_epi8((char)(0xF0 - 0x80)); - const __m512i max_value = _mm512_setr_epi64(-1, -1, -1, -1, -1, -1, -1, (int64_t)0xBFDFEFFFFFFFFFFFULL); + /* A block ending with any of these is incomplete: ... 1111____ 111_____ 11______ */ + const __m512i max_value = _mm512_setr_epi8( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + (char)(0xF0 - 1), (char)(0xE0 - 1), (char)(0xC0 - 1)); /* Brings the last 16 bytes of `prev` in front of the first 48 bytes of `input` */ const __m512i prev_shuffle = _mm512_setr_epi32(28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); @@ -5836,7 +5842,12 @@ static bool mb_fast_check_utf8_avx512(zend_string *str) __m512i in = _mm512_loadu_si512((const void*)p); MB_UTF8_CHECK_AVX512(in, prev_input); p += 64 - misalignment; - prev_input = _mm512_maskz_loadu_epi8(0xE000000000000000ULL, (const void*)(p - 64)); + /* The 3 bytes before p belong to the string: misalignment <= 61, so + * p - 3 does not step back past the start. They occupy the last 3 lanes. */ + uint32_t ctx = 0; + memcpy(&ctx, p - 3, 3); + __m128i ctxv = _mm_slli_si128(_mm_cvtsi32_si128(ctx), 13); + prev_input = _mm512_inserti64x2(_mm512_setzero_si512(), ctxv, 3); prev_incomplete = _mm512_subs_epu8(prev_input, max_value); } diff --git a/ext/mbstring/tests/mb_check_encoding_utf8_blocks.phpt b/ext/mbstring/tests/mb_check_encoding_utf8_blocks.phpt index 9d6b32e38e16..068d00714a93 100644 --- a/ext/mbstring/tests/mb_check_encoding_utf8_blocks.phpt +++ b/ext/mbstring/tests/mb_check_encoding_utf8_blocks.phpt @@ -26,7 +26,9 @@ foreach ($fillers as $filler) { foreach ([false, true] as $replace) { $s = substr($base, 0, $pos) . $seq . substr($base, $pos + ($replace ? strlen($seq) : 0)); $expected = preg_match('//u', $s) === 1; - $s = str_repeat($s, 1); // a fresh string, with no cached validity flag + // preg_match('//u') sets the UTF-8 flag, and str_repeat($s, 1) + // returns that same string. pack() allocates a new one without it. + $s = ($s === "") ? $s : pack("H*", bin2hex($s)); if (mb_check_encoding($s, 'UTF-8') !== $expected) { if ($mismatches++ < 10) { echo "Mismatch: ", bin2hex($s), " expected ", var_export($expected, true), "\n"; @@ -38,6 +40,33 @@ foreach ($fillers as $filler) { } } } +// Lengths past 2048 bytes take the AVX-512 alignment prologue. +foreach ([2048, 2112, 4095] as $len) { + foreach ($fillers as $filler) { + $base = str_repeat($filler, intdiv($len, strlen($filler)) + 1); + $base = mb_strcut($base, 0, $len, 'UTF-8'); + $probe = pack('H*', bin2hex($base)); + if (mb_check_encoding($probe, 'UTF-8') !== (preg_match('//u', $base) === 1)) { + if ($mismatches++ < 10) { + echo "Mismatch: long ", bin2hex(substr($base, 0, 16)), "... len ", strlen($base), "\n"; + } + } + $checked++; + if (strlen($base) < 65) { + continue; + } + $s = $probe; + $s[64] = "\x80"; + $expected = preg_match('//u', $s) === 1; + $s = pack('H*', bin2hex($s)); + if (mb_check_encoding($s, 'UTF-8') !== $expected) { + if ($mismatches++ < 10) { + echo "Mismatch: long mutated len ", strlen($s), "\n"; + } + } + $checked++; + } +} echo $checked > 100000 ? "enough cases\n" : "too few cases: $checked\n"; echo "mismatches: $mismatches\n"; ?> From 91c0384b83445c53ef30a6d3daf63bd212d1fd6d Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Sun, 15 Nov 2026 17:26:27 -0500 Subject: [PATCH 3/3] ext/mbstring: Fix AVX-512 build _mm512_setr_epi8 does not exist in GCC or clang. Go back to the _mm512_setr_epi64 constant, which has the same bytes. --- ext/mbstring/mbstring.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/ext/mbstring/mbstring.c b/ext/mbstring/mbstring.c index 3312cd706dbf..b1a955f124f4 100644 --- a/ext/mbstring/mbstring.c +++ b/ext/mbstring/mbstring.c @@ -5821,13 +5821,9 @@ static bool mb_fast_check_utf8_avx512(zend_string *str) const __m512i v_80 = _mm512_set1_epi8((char)0x80); const __m512i v_third = _mm512_set1_epi8((char)(0xE0 - 0x80)); const __m512i v_fourth = _mm512_set1_epi8((char)(0xF0 - 0x80)); - /* A block ending with any of these is incomplete: ... 1111____ 111_____ 11______ */ - const __m512i max_value = _mm512_setr_epi8( - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - (char)(0xF0 - 1), (char)(0xE0 - 1), (char)(0xC0 - 1)); + /* A block ending with any of these is incomplete: ... 1111____ 111_____ 11______ + * (bytes 61-63 are 0xEF, 0xDF, 0xBF; there is no _mm512_setr_epi8) */ + const __m512i max_value = _mm512_setr_epi64(-1, -1, -1, -1, -1, -1, -1, (int64_t)0xBFDFEFFFFFFFFFFFULL); /* Brings the last 16 bytes of `prev` in front of the first 48 bytes of `input` */ const __m512i prev_shuffle = _mm512_setr_epi32(28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);