diff --git a/src/Interpreters/ITokenExtractor.cpp b/src/Interpreters/ITokenExtractor.cpp index fc77a7507cbcb..a83b0858a988c 100644 --- a/src/Interpreters/ITokenExtractor.cpp +++ b/src/Interpreters/ITokenExtractor.cpp @@ -55,6 +55,8 @@ bool NgramTokenExtractor::nextInString(const char * data, size_t length, size_t size_t sz = UTF8::seqLength(static_cast(data[*token_start + *token_length])); *token_length += sz; } + /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length. + *token_length = std::min(*token_length, length - *token_start); *pos += UTF8::seqLength(static_cast(data[*pos])); return code_points == n; } @@ -80,7 +82,8 @@ bool NgramTokenExtractor::nextInStringLike(const char * data, size_t length, siz token.clear(); code_points = 0; escaped = false; - *pos = ++i; + ++i; + *pos = i; } else if (!escaped && data[i] == '\\') { @@ -90,7 +93,8 @@ bool NgramTokenExtractor::nextInStringLike(const char * data, size_t length, siz else { const size_t sz = UTF8::seqLength(static_cast(data[i])); - for (size_t j = 0; j < sz; ++j) + /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length. + for (size_t j = 0; j < sz && i + j < length; ++j) token += data[i + j]; i += sz; ++code_points; @@ -254,7 +258,8 @@ bool DefaultTokenExtractor::nextInStringLike(const char * data, size_t length, s else { const size_t sz = UTF8::seqLength(static_cast(data[*pos])); - for (size_t j = 0; j < sz; ++j) + /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length. + for (size_t j = 0; j < sz && *pos < length; ++j) { token += data[*pos]; ++*pos; diff --git a/tests/queries/0_stateless/03784_msan_token_iterator.reference b/tests/queries/0_stateless/03784_msan_token_iterator.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/03784_msan_token_iterator.sql b/tests/queries/0_stateless/03784_msan_token_iterator.sql new file mode 100644 index 0000000000000..925abbf847a8b --- /dev/null +++ b/tests/queries/0_stateless/03784_msan_token_iterator.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS t0; +CREATE TABLE t0 (c0 Int, INDEX i1 substring(sipHash128(c0), 1, 4) TYPE ngrambf_v1(1, 8, 1, 0)) ENGINE = MergeTree ORDER BY (); +INSERT INTO TABLE t0 (c0) SELECT number FROM numbers(28); +DROP TABLE t0;