Fix scanner keyword matching under Turkish locale

.github/workflows/main.yml

@@ -102,6 +102,10 @@ jobs:
         with:
           composer-options: "--prefer-dist"
 
+      # ScannerTest checks keyword matching under a Turkish locale
+      - name: Generate the tr_TR.UTF-8 locale
+        run: sudo locale-gen tr_TR.UTF-8
+
       - name: Run tests
         run: composer test-unit
 

resources/docker/develop/Dockerfile

@@ -56,6 +56,8 @@ install-php-extensions \
 rm -f /usr/local/etc/php/conf.d/docker-php-ext-phalcon.ini
 # Locales
 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
+# tr_TR: the scanner's keyword matching must not depend on the locale
+sed -i -e 's/# tr_TR.UTF-8 UTF-8/tr_TR.UTF-8 UTF-8/' /etc/locale.gen
 dpkg-reconfigure --frontend=noninteractive locales
 update-locale LANG=en_US.UTF-8
 # Unlimited memory for tests

src/Scanner/Scanner.php

@@ -23,7 +23,6 @@
 use function strlen;
 use function strpos;
 use function strspn;
-use function strtolower;
 use function substr;
 use function substr_count;
 
@@ -89,11 +88,11 @@ final class Scanner
 
     private const DOUBLE_OR_INTEGER = '/[0-9]+(?:\.[0-9]+)?/A';
 
-    private const ENDVERBATIM = '/\{%-?[ \t\r\n]*endverbatim(?![a-zA-Z0-9_])/Ai';
+    private const ENDVERBATIM = '/\{%-?[ \t\r\n]*[eE][nN][dD][vV][eE][rR][bB][aA][tT][iI][mM](?![a-zA-Z0-9_])/A';
 
     private const IDENTIFIER = '/\\\\?[a-zA-Z_][a-zA-Z0-9_\\\\]*/A';
 
-    private const IS_NOT_ONE_SPACE = '/is not/Ai';
+    private const IS_NOT_ONE_SPACE = '/[iI][sS] [nN][oO][tT]/A';
 
     private const IS_NOT_SPACES = '/is +not/A';
 
@@ -151,6 +150,14 @@ final class Scanner
         'with'          => Opcode::WITH,
     ];
 
+    /**
+     * Keywords and `is not` / `endverbatim` are matched case-insensitively
+     * byte by byte, as cphalcon's re2c scanner does: strtolower() follows the
+     * locale before PHP 8.2, and so does PCRE's `/i` once a locale is set -
+     * under a Turkish one `I` and `i` are not each other's case.
+     */
+    private const LOWER_CASE = 'abcdefghijklmnopqrstuvwxyz';
+
     /**
      * Punctuation and operators, longest first within each leading character
      * so the first hit is the longest match.
@@ -202,6 +209,8 @@ final class Scanner
      */
     private const TRIM = " \t\n\r\v";
 
+    private const UPPER_CASE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+
     private readonly string $code;
 
     /**
@@ -529,7 +538,7 @@ private function scanVerbatim(): ScannerStatus
      */
     private function scanWord(int $cursor, string $word): ScannerStatus
     {
-        $keyword = strtolower($word);
+        $keyword = strtr($word, self::UPPER_CASE, self::LOWER_CASE);
 
         if ($keyword === 'is') {
             $isNot = $this->matchIsNot($cursor);

tests/unit/Scanner/ScannerTest.php

@@ -157,6 +157,34 @@ public function testKeywordsAreCaseInsensitive(): void
         );
     }
 
+    /**
+     * cphalcon matches keywords byte by byte. Before PHP 8.2 strtolower()
+     * follows the locale, and PCRE's case-insensitive matching does on every
+     * version once a locale is set: under a Turkish one `I` and `i` are not
+     * each other's case. CI and the development image generate tr_TR.UTF-8;
+     * where it is missing the test runs under the current locale.
+     */
+    public function testKeywordsIgnoreTheLocale(): void
+    {
+        $previous = (string) setlocale(LC_CTYPE, '0');
+        setlocale(LC_CTYPE, 'tr_TR.UTF-8', 'tr_TR.ISO-8859-9', 'tr_TR');
+
+        try {
+            $tokens = $this->codeTokens('{% IF a IS NOT b %}{{ a IN b }}{% verbatim %}x{% ENDVERBATIM %}');
+        } finally {
+            setlocale(LC_CTYPE, $previous);
+        }
+
+        $this->assertSame(
+            [
+                ['IF', null], ['IDENTIFIER', 'a'], ['NOTEQUALS', null], ['IDENTIFIER', 'b'],
+                ['IDENTIFIER', 'a'], ['IN', null], ['IDENTIFIER', 'b'],
+                ['VERBATIM', null], ['RAW_FRAGMENT', 'x'], ['ENDVERBATIM', null],
+            ],
+            $tokens
+        );
+    }
+
     public function testLeftTrimCrossesAComment(): void
     {
         $this->assertSame(