Stable 25.8 Backport of #93663 - Fix trash in TokenExtractor

src/Interpreters/ITokenExtractor.cpp

@@ -55,6 +55,8 @@ bool NgramTokenExtractor::nextInString(const char * data, size_t length, size_t
         size_t sz = UTF8::seqLength(static_cast<UInt8>(data[*token_start + *token_length]));
         *token_length += sz;
     }
+    /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length.
+    *token_length = std::min(*token_length, length - *token_start);
     *pos += UTF8::seqLength(static_cast<UInt8>(data[*pos]));
     return code_points == n;
 }
@@ -80,7 +82,8 @@ bool NgramTokenExtractor::nextInStringLike(const char * data, size_t length, siz
             token.clear();
             code_points = 0;
             escaped = false;
-            *pos = ++i;
+            ++i;
+            *pos = i;
         }
         else if (!escaped && data[i] == '\\')
         {
@@ -90,7 +93,8 @@ bool NgramTokenExtractor::nextInStringLike(const char * data, size_t length, siz
         else
         {
             const size_t sz = UTF8::seqLength(static_cast<UInt8>(data[i]));
-            for (size_t j = 0; j < sz; ++j)
+            /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length.
+            for (size_t j = 0; j < sz && i + j < length; ++j)
                 token += data[i + j];
             i += sz;
             ++code_points;
@@ -254,7 +258,8 @@ bool DefaultTokenExtractor::nextInStringLike(const char * data, size_t length, s
         else
         {
             const size_t sz = UTF8::seqLength(static_cast<UInt8>(data[*pos]));
-            for (size_t j = 0; j < sz; ++j)
+            /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length.
+            for (size_t j = 0; j < sz && *pos < length; ++j)
             {
                 token += data[*pos];
                 ++*pos;

tests/queries/0_stateless/03784_msan_token_iterator.sql

@@ -0,0 +1,4 @@
+DROP TABLE IF EXISTS t0;
+CREATE TABLE t0 (c0 Int, INDEX i1 substring(sipHash128(c0), 1, 4) TYPE ngrambf_v1(1, 8, 1, 0)) ENGINE = MergeTree ORDER BY ();
+INSERT INTO TABLE t0 (c0) SELECT number FROM numbers(28);
+DROP TABLE t0;