From ce3d10304c62a6ff3cf212cc4b309c61c9794838 Mon Sep 17 00:00:00 2001 From: Paul Zhu Date: Fri, 15 May 2026 17:39:28 -0700 Subject: [PATCH 1/2] fix: caseless match for arm64 and proper check for win32 --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 65bfd26..d39999b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,11 +84,13 @@ elseif (CMAKE_SYSTEM_NAME STREQUAL "Android") CXX_${TOKENIZERS_CPP_CARGO_TARGET}=${ANDROID_TOOLCHAIN_ROOT}/bin/${TOKENIZERS_CPP_CARGO_TARGET}${ANDROID_NATIVE_API_LEVEL}-clang++ ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Windows") - if(CMAKE_SYSTEM_PROCESSOR STREQUAL "ARM64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") + string(TOUPPER "${CMAKE_SYSTEM_PROCESSOR}" _proc_upper) + if(_proc_upper STREQUAL "ARM64" OR _proc_upper STREQUAL "AARCH64") set(TOKENIZERS_CPP_CARGO_TARGET aarch64-pc-windows-msvc) else() set(TOKENIZERS_CPP_CARGO_TARGET x86_64-pc-windows-msvc) endif() + unset(_proc_upper) endif() if(WIN32) @@ -123,7 +125,7 @@ add_subdirectory(msgpack) option(MLC_ENABLE_SENTENCEPIECE_TOKENIZER "Enable SentencePiece tokenizer" ON) -if(MSVC) +if(WIN32) set(TOKENIZERS_RUST_LIB "${TOKENIZERS_CPP_CARGO_BINARY_DIR}/tokenizers_c.lib") else() set(TOKENIZERS_RUST_LIB "${TOKENIZERS_CPP_CARGO_BINARY_DIR}/libtokenizers_c.a") From cb1b346e0c5c325a485dfa86269c8fe2f533d58c Mon Sep 17 00:00:00 2001 From: Paul Zhu Date: Mon, 18 May 2026 10:24:03 -0700 Subject: [PATCH 2/2] use latest sentencepiece --- sentencepiece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentencepiece b/sentencepiece index 11051e3..a899e9a 160000 --- a/sentencepiece +++ b/sentencepiece @@ -1 +1 @@ -Subproject commit 11051e3b73b3a6222a52acd720e39805dc7545ab +Subproject commit a899e9a247791163095a331d8f16e4b7ea2068a0