Upload tokenizers/TOKENIZERS_USER_GUIDE.txt with huggingface_hub

tokenizers/TOKENIZERS_USER_GUIDE.txt

@@ -0,0 +1,69 @@
+================================================================================
+  TOKENIZERS - USER GUIDE (Android Python STB) - Generated by RIMI
+================================================================================
+
+WHAT IT IS
+  HuggingFace `tokenizers` 0.23.2 - fast Rust implementation of modern
+  tokenizers (BPE, WordPiece, WordLevel, Unigram) with Python bindings
+  (PyO3). Train vocabularies and encode/decode text at high speed.
+
+WHEELS (STANDALONE folder)
+  tokenizers-0.23.2-cp312-cp312-android_24_x86_64.whl   (emulator)
+  tokenizers-0.23.2-cp312-cp312-android_24_arm64_v8a.whl (phones)
+  huggingface_hub-1.30.0-py3-none-any.whl               (pure-Python dep,
+    official PyPI wheel, unmodified)
+  Install the wheel matching your device arch, plus huggingface_hub.
+
+IMPORT
+  from tokenizers import Tokenizer
+  from tokenizers.models import BPE, WordPiece, WordLevel
+  from tokenizers.trainers import BpeTrainer, WordPieceTrainer, WordLevelTrainer
+  from tokenizers.pre_tokenizers import Whitespace, WhitespaceSplit
+  from tokenizers.normalizers import Lowercase, NFKC
+  from tokenizers.processors import BertProcessing
+  from tokenizers.decoders import BPEDecoder
+  import tokenizers
+  print(tokenizers.__version__)   # 0.23.2
+
+QUICK EXAMPLE - BPE train + roundtrip
+  from tokenizers import Tokenizer
+  from tokenizers.models import BPE
+  from tokenizers.trainers import BpeTrainer
+  from tokenizers.pre_tokenizers import Whitespace
+
+  tok = Tokenizer(BPE(unk_token="[UNK]"))
+  tok.pre_tokenizer = Whitespace()
+  trainer = BpeTrainer(vocab_size=200, special_tokens=["[UNK]"])
+  tok.train(["my_corpus.txt"], trainer)
+  enc = tok.encode("Hello world")
+  print(enc.ids, enc.tokens)
+  print(tok.decode(enc.ids))
+
+QUICK EXAMPLE - WordLevel
+  from tokenizers import Tokenizer
+  from tokenizers.models import WordLevel
+  tok = Tokenizer(WordLevel(vocab={"hello": 0, "world": 1, "[UNK]": 2},
+                            unk_token="[UNK]"))
+  print(tok.encode("hello world").ids)   # [0, 1]
+
+SAVE / LOAD
+  tok.save("/path/tok.json")
+  tok2 = Tokenizer.from_file("/path/tok.json")
+
+ANDROID NOTES
+  - Native extension: tokenizers/tokenizers.cpython-312.so (Rust cdylib,
+    16KB page-aligned, NoRELRO, stripped, DT_NEEDED includes librimi.so
+    app-lock + libc++_shared + libpython3.12.so.1.0).
+  - Requires huggingface_hub at runtime (Requires-Dist kept in METADATA).
+    huggingface_hub's optional native dep hf-xet is NOT bundled and NOT
+    needed; the hub falls back to httpx without it.
+  - No network needed for train/encode/decode; only for hub downloads.
+
+TESTED FEATURES (Test_Tokenizers.py, exit 0 = all pass)
+  import + __version__ 0.23.2; submodules import; BPE train on tiny
+  corpus; encode/decode roundtrip; encode_batch; WordLevel; save/load;
+  Lowercase normalizer; BertProcessing.
+
+================================================================================
+  Generated by RIMI
+================================================================================