From bae6378a937ce6d25f0bae393ad04a3989a3d876 Mon Sep 17 00:00:00 2001 From: Ali-Nawaz-devt Date: Sat, 14 Nov 2026 16:35:04 +0500 Subject: [PATCH 1/2] feat: add Sindhi tokenizer Python SDK --- .github/workflows/python-tests.yml | 7 +- .github/workflows/release.yml | 35 +++++++++ README.md | 52 ++++++------ pyproject.toml | 30 ++++--- sindhi_tokenizer/__init__.py | 6 ++ sindhi_tokenizer/py.typed | 0 sindhi_tokenizer/tokenizer.py | 122 +++++++++++++++++++++++++++++ tests/test_tokenizer_sdk.py | 89 +++++++++++++++++++++ 8 files changed, 305 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 sindhi_tokenizer/__init__.py create mode 100644 sindhi_tokenizer/py.typed create mode 100644 sindhi_tokenizer/tokenizer.py create mode 100644 tests/test_tokenizer_sdk.py diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index ce80364..0fe0599 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install -e ".[dev]" pip install flake8 pytest - name: Lint with flake8 @@ -37,3 +37,6 @@ jobs: - name: Test metrics import and basic sanity run: | python -c "from evaluation.metrics import word_split, _canon; assert len(word_split('سنڌ جي تاريخ')) == 3; assert _canon('سنڌ') != ''" + + - name: Test SDK + run: pytest -q diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9234cba --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,35 @@ +name: Publish Python package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + publish: + name: Build and publish to PyPI + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/sindhi-tokenizer + permissions: + id-token: write + steps: + - name: Check out release tag + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install build tools + run: python -m pip install --upgrade build + + - name: Build distributions + run: python -m build + + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/README.md b/README.md index 9a55d69..2eabf1a 100644 --- a/README.md +++ b/README.md @@ -299,42 +299,46 @@ In accordance with rigorous research standards, two negative experimental result ## Installation and Usage -### Prerequisites -- Python 3.9, 3.10, 3.11, or 3.12 -- `sentencepiece`, `transformers`, `huggingface_hub` +### Install the Python SDK + +Requires Python 3.9 or newer. The SDK only installs SentencePiece and the Hugging Face Hub client; training and demo dependencies are optional. ```bash -git clone https://github.com/alinawazcode/sindhi-tokenizer.git -cd sindhi-tokenizer -pip install -e . +pip install sindhi-tokenizer ``` -### Quickstart: Load from Hugging Face Hub -```python -from huggingface_hub import hf_hub_download -import sentencepiece as spm - -# Download the recommended 64k flagship model -model_path = hf_hub_download( - repo_id="alinawazmahar/sindhi-tokenizer", - filename="sindhi_custom_64000.model" -) +### Quickstart -# Load into SentencePiece processor -sp = spm.SentencePieceProcessor(model_file=model_path) +```python +from sindhi_tokenizer import SindhiTokenizer +# Downloads the recommended 64k model once, then reuses the Hub cache. +tokenizer = SindhiTokenizer.from_pretrained() text = "سنڌ جي تاريخ تمام قديم آهي" -tokens = sp.encode(text, out_type=str) -token_ids = sp.encode(text, out_type=int) -print("Tokens:", tokens) -# [' سنڌ', ' جي', ' تاريخ', ' تمام', ' قديم', ' آهي'] +pieces = tokenizer.tokenize(text) +token_ids = tokenizer.encode(text) + +print("Pieces:", pieces) print("Token IDs:", token_ids) -print("Decoded:", sp.decode(token_ids)) +print("Decoded:", tokenizer.decode(token_ids)) +print("Vocabulary size:", tokenizer.vocab_size) # 64000 ``` +Choose another released vocabulary with `vocab_size=32000`, `100000`, or `150000`. You can also load a local `.model` file with `SindhiTokenizer.from_file(path)`. Batch methods are `encode_batch()` and `decode_batch()`. + +To work on the repository and its training/demo tools: + +```bash +git clone https://github.com/alinawazcode/sindhi-tokenizer.git +cd sindhi-tokenizer +pip install -e ".[dev,training,demo]" +``` + +The model download is cached by Hugging Face Hub. The SDK code is MIT licensed; the model files are separately licensed under CC BY-SA 4.0. Review the model card for attribution and sharing terms before redistributing model files. + > [!CAUTION] -> **Do not use `transformers.LlamaTokenizer` to load raw `.model` files**: Hugging Face's `LlamaTokenizer` can silently fail on custom SPM Unigram files and return a 3-token fallback vocabulary. Always load using `sentencepiece.SentencePieceProcessor` or the provided [`evaluation.metrics.SPMTokenizerAdapter`](evaluation/metrics.py). +> **Do not use `transformers.LlamaTokenizer` to load raw `.model` files**: it can silently return a 3-token fallback vocabulary for these custom SentencePiece Unigram files. Use `SindhiTokenizer`, `sentencepiece.SentencePieceProcessor`, or the benchmark adapter in [`evaluation/metrics.py`](evaluation/metrics.py). --- diff --git a/pyproject.toml b/pyproject.toml index 6704623..16574bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,40 +1,47 @@ [build-system] -requires = ["setuptools>=61.0"] +requires = ["setuptools>=77.0"] build-backend = "setuptools.build_meta" [project] name = "sindhi-tokenizer" -version = "1.0.0" -description = "A SentencePiece Unigram tokenizer purpose-built for the Sindhi language with byte-fallback and full character coverage." +version = "1.1.0" +description = "A SentencePiece Unigram tokenizer and lightweight Python SDK purpose-built for Sindhi." readme = "README.md" authors = [ { name = "Ali Nawaz" }, { name = "Muqadas Fareed" }, { name = "Soonh Sadaf" } ] -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] keywords = ["sindhi", "tokenizer", "sentencepiece", "unigram", "nlp", "low-resource-languages"] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Science/Research", "Topic :: Scientific/Engineering :: Artificial Intelligence", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "sentencepiece>=0.1.99", - "transformers>=4.30.0", - "huggingface_hub>=0.16.0", + "huggingface_hub>=0.23.0", +] + +[project.optional-dependencies] +training = [ "datasets>=2.14.0", "numpy>=1.22.0", "pandas>=1.5.0", - "gradio>=4.0.0" ] +demo = [ + "gradio>=4.0.0", + "transformers>=4.30.0", +] +dev = ["build>=1.2.0", "pytest>=7.4.0", "twine>=5.0.0"] [project.urls] Homepage = "https://github.com/alinawazcode/sindhi-tokenizer" @@ -43,4 +50,7 @@ Models = "https://huggingface.co/alinawazmahar/sindhi-tokenizer" [tool.setuptools.packages.find] where = ["."] -include = ["evaluation*", "training*"] +include = ["evaluation*", "training*", "sindhi_tokenizer*"] + +[tool.setuptools.package-data] +sindhi_tokenizer = ["py.typed"] diff --git a/sindhi_tokenizer/__init__.py b/sindhi_tokenizer/__init__.py new file mode 100644 index 0000000..383c6c5 --- /dev/null +++ b/sindhi_tokenizer/__init__.py @@ -0,0 +1,6 @@ +"""Python SDK for the released Sindhi SentencePiece tokenizer.""" + +from .tokenizer import SindhiTokenizer + +__all__ = ["SindhiTokenizer"] +__version__ = "1.1.0" diff --git a/sindhi_tokenizer/py.typed b/sindhi_tokenizer/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/sindhi_tokenizer/tokenizer.py b/sindhi_tokenizer/tokenizer.py new file mode 100644 index 0000000..39164ba --- /dev/null +++ b/sindhi_tokenizer/tokenizer.py @@ -0,0 +1,122 @@ +"""Simple inference API for the released Sindhi SentencePiece models.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Literal, Sequence + +import sentencepiece as spm +from huggingface_hub import hf_hub_download + + +DEFAULT_MODEL_ID = "alinawazmahar/sindhi-tokenizer" +MODEL_FILES = { + 32_000: "sindhi_custom_32000.model", + 64_000: "sindhi_custom_64000.model", + 100_000: "sindhi_custom_100000.model", + 150_000: "sindhi_custom_150000.model", +} +TokenOutput = Literal["ids", "pieces"] + + +class SindhiTokenizer: + """Load and use one of the released Sindhi SentencePiece models. + + Use :meth:`from_pretrained` for a Hugging Face model or :meth:`from_file` + for a local ``.model`` file. The default is the recommended 64k vocabulary. + """ + + def __init__(self, model_path: str | Path) -> None: + self.model_path = Path(model_path) + if not self.model_path.is_file(): + raise FileNotFoundError(f"SentencePiece model not found: {self.model_path}") + self._processor = spm.SentencePieceProcessor(model_file=str(self.model_path)) + + @classmethod + def from_file(cls, model_path: str | Path) -> "SindhiTokenizer": + """Load a model from a local SentencePiece ``.model`` file.""" + return cls(model_path) + + @classmethod + def from_pretrained( + cls, + model_id: str = DEFAULT_MODEL_ID, + *, + vocab_size: int = 64_000, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_files_only: bool = False, + ) -> "SindhiTokenizer": + """Download or reuse a cached released vocabulary from Hugging Face. + + ``vocab_size`` must be one of 32000, 64000, 100000, or 150000. + """ + try: + filename = MODEL_FILES[vocab_size] + except KeyError as exc: + supported = ", ".join(str(size) for size in MODEL_FILES) + raise ValueError(f"Unsupported vocab_size {vocab_size}; choose one of: {supported}.") from exc + + model_path = hf_hub_download( + repo_id=model_id, + filename=filename, + revision=revision, + cache_dir=str(cache_dir) if cache_dir is not None else None, + local_files_only=local_files_only, + ) + return cls(model_path) + + @property + def vocab_size(self) -> int: + """Number of SentencePiece entries in this model.""" + return self._processor.get_piece_size() + + def tokenize(self, text: str) -> list[str]: + """Split text into SentencePiece pieces, retaining whitespace markers.""" + self._check_text(text) + return self._processor.encode(text, out_type=str) + + def encode(self, text: str, *, out_type: TokenOutput = "ids") -> list[int] | list[str]: + """Encode a string as integer IDs (default) or readable pieces.""" + self._check_text(text) + if out_type == "ids": + return self._processor.encode(text, out_type=int) + if out_type == "pieces": + return self._processor.encode(text, out_type=str) + raise ValueError("out_type must be 'ids' or 'pieces'.") + + def encode_batch( + self, texts: Sequence[str], *, out_type: TokenOutput = "ids" + ) -> list[list[int]] | list[list[str]]: + """Encode strings in order, returning one token sequence per input.""" + if isinstance(texts, (str, bytes)): + raise TypeError("texts must be a sequence of strings, not one string.") + for text in texts: + self._check_text(text) + if out_type not in ("ids", "pieces"): + raise ValueError("out_type must be 'ids' or 'pieces'.") + kind = int if out_type == "ids" else str + return [self._processor.encode(text, out_type=kind) for text in texts] + + def decode(self, tokens: Sequence[int] | Sequence[str]) -> str: + """Reconstruct text from token IDs or SentencePiece strings.""" + if isinstance(tokens, (str, bytes)): + raise TypeError("tokens must be a sequence of token IDs or pieces.") + return self._processor.decode(list(tokens)) + + def decode_batch(self, batch: Sequence[Sequence[int] | Sequence[str]]) -> list[str]: + """Decode a batch of token sequences in order.""" + return [self.decode(tokens) for tokens in batch] + + def id_to_piece(self, token_id: int) -> str: + """Return the readable SentencePiece string for an ID.""" + return self._processor.id_to_piece(token_id) + + def piece_to_id(self, piece: str) -> int: + """Return the ID for a piece, or the model's unknown ID if absent.""" + return self._processor.piece_to_id(piece) + + @staticmethod + def _check_text(text: str) -> None: + if not isinstance(text, str): + raise TypeError(f"text must be str, got {type(text).__name__}.") diff --git a/tests/test_tokenizer_sdk.py b/tests/test_tokenizer_sdk.py new file mode 100644 index 0000000..59fe01d --- /dev/null +++ b/tests/test_tokenizer_sdk.py @@ -0,0 +1,89 @@ +import pytest + +from sindhi_tokenizer import SindhiTokenizer +from sindhi_tokenizer import tokenizer as tokenizer_module + + +class FakeSentencePieceProcessor: + def __init__(self, model_file): + self.model_file = model_file + + def get_piece_size(self): + return 64_000 + + def encode(self, text, out_type=int): + values = [ord(char) for char in text] + return values if out_type is int else [f"▁{char}" for char in text] + + def decode(self, tokens): + if tokens and isinstance(tokens[0], str): + return "".join(token.removeprefix("▁") for token in tokens) + return "".join(chr(token) for token in tokens) + + def id_to_piece(self, token_id): + return f"piece-{token_id}" + + def piece_to_id(self, piece): + return 7 if piece == "known" else 0 + + +@pytest.fixture +def fake_processor(monkeypatch): + monkeypatch.setattr(tokenizer_module.spm, "SentencePieceProcessor", FakeSentencePieceProcessor) + + +def test_local_load_encode_decode_and_batches(tmp_path, fake_processor): + model_path = tmp_path / "model.model" + model_path.touch() + tokenizer = SindhiTokenizer.from_file(model_path) + + assert tokenizer.vocab_size == 64_000 + assert tokenizer.encode("سنڌ") == [ord(char) for char in "سنڌ"] + assert tokenizer.decode(tokenizer.encode("سنڌ")) == "سنڌ" + assert tokenizer.encode("سنڌ", out_type="pieces") == [f"▁{char}" for char in "سنڌ"] + assert tokenizer.decode_batch(tokenizer.encode_batch(["الف", "ب"], out_type="ids")) == ["الف", "ب"] + assert tokenizer.id_to_piece(12) == "piece-12" + assert tokenizer.piece_to_id("known") == 7 + + +def test_from_pretrained_uses_expected_hub_file(tmp_path, monkeypatch, fake_processor): + model_path = tmp_path / "downloaded.model" + model_path.touch() + call = {} + + def fake_download(**kwargs): + call.update(kwargs) + return str(model_path) + + monkeypatch.setattr(tokenizer_module, "hf_hub_download", fake_download) + tokenizer = SindhiTokenizer.from_pretrained(vocab_size=100_000, revision="main") + + assert tokenizer.vocab_size == 64_000 + assert call["repo_id"] == "alinawazmahar/sindhi-tokenizer" + assert call["filename"] == "sindhi_custom_100000.model" + assert call["revision"] == "main" + + +def test_rejects_unsupported_vocabulary_without_downloading(monkeypatch): + def fail_if_called(**kwargs): + raise AssertionError("hub download should not run for an invalid vocabulary") + + monkeypatch.setattr(tokenizer_module, "hf_hub_download", fail_if_called) + with pytest.raises(ValueError, match="Unsupported vocab_size"): + SindhiTokenizer.from_pretrained(vocab_size=42_000) + + +def test_validates_inputs_and_local_files(tmp_path, fake_processor): + with pytest.raises(FileNotFoundError): + SindhiTokenizer.from_file(tmp_path / "missing.model") + + model_path = tmp_path / "model.model" + model_path.touch() + tokenizer = SindhiTokenizer.from_file(model_path) + + with pytest.raises(TypeError, match="text must be str"): + tokenizer.encode(123) # type: ignore[arg-type] + with pytest.raises(TypeError, match="sequence of strings"): + tokenizer.encode_batch("single string") + with pytest.raises(ValueError, match="out_type"): + tokenizer.encode("text", out_type="tokens") # type: ignore[arg-type] From 4762ba617cb0ce318a5779515ae0c9920b25e11d Mon Sep 17 00:00:00 2001 From: Ali-Nawaz-devt Date: Sat, 14 Nov 2026 23:12:52 +0500 Subject: [PATCH 2/2] ci: pin tests to Ubuntu 24.04 --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 0fe0599..46299e7 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -8,7 +8,7 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"]