From 7c0fcc9c92cadba5184852ce6a6467b2b3445246 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 19 May 2026 18:34:36 +0000
Subject: [PATCH] Bump tokenizers from 0.15.2 to 0.23.1

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.15.2 to 0.23.1.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.15.2...v0.23.1)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-version: 0.23.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements_cpu.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements_cpu.txt b/requirements_cpu.txt
index 2b063ff..586f0d9 100644
--- a/requirements_cpu.txt
+++ b/requirements_cpu.txt
@@ -1,6 +1,6 @@
 sentencepiece
 torch
-tokenizers==0.15.2
+tokenizers==0.23.1
 transformers==4.39.3
 accelerate>=0.24.0
 tqdm