Upload folder using huggingface_hub

README.md

@@ -47,4 +47,8 @@ tokens = tokenizer.encode("Hello, world!")
 ## Sample Encoding
 | Text | Tokens | Token IDs |
 |------|--------|-----------|
-| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, t, u, z` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 3, 70, 81, 77, 87, 92, 4, 81, 86, 87, 92` |
+| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, t, u, z` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 13, 70, 81, 77, 87, 92, 31, 81, 86, 87, 92` |
+
+Command used to create this tokenizer:
+```bash
+['/home/gsa/tokenizers2/flexitok/tokenizer_training/train_tokenizers.py', 'algorithm=bpe', 'vocab_size=2000', 'langs=[cmn_Hani]', 'data_dir=/scratch/gsa/data/multilingual-addition/', 'output_dir=/scratch/gsa/trained_tokenizers/multilingual_addition', 'pretokenizer=custom:addition', 'number_handling=ltr_3digit', 'add_numbers=false', 'handle_contractions=false', 'unicode_normalization=nfc', 'use_byte_level_regex=false', 'byte_fallback=false', 'strip_zero_width=false', 'cjk_char_split=false', 'add_cjk_chars=false', 'max_lines=-1', 'test_string=yirmi iki+dokuz=otuz bir\\ntwenty two+nine=thirty one', 'hf.publish_to_hf=true', 'hf_repo_prefix=flexitok/', 'hf.hf_repo_id=flexitok/maddition_cmn_Hani_2000', 'hf.collections=[flexitok/multilingual_addition_tokenizers]']

special_tokens_map.json

@@ -1,8 +1,4 @@
 {
-  "additional_special_tokens": [
-    "+",
-    "="
-  ],
   "bos_token": "<s>",
   "eos_token": "</s>",
   "pad_token": "<pad>"

tokenizer.json

@@ -29,24 +29,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 3,
-      "content": "+",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 4,
-      "content": "=",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": {
@@ -99,35 +81,35 @@
       "<s>": 0,
       "</s>": 1,
       "<pad>": 2,
-      "+": 3,
-      "=": 4,
-      "!": 5,
-      "\"": 6,
-      "#": 7,
-      "$": 8,
-      "%": 9,
-      "&": 10,
-      "'": 11,
-      "(": 12,
-      ")": 13,
-      "*": 14,
-      ",": 15,
-      "-": 16,
-      ".": 17,
-      "/": 18,
-      "0": 19,
-      "1": 20,
-      "2": 21,
-      "3": 22,
-      "4": 23,
-      "5": 24,
-      "6": 25,
-      "7": 26,
-      "8": 27,
-      "9": 28,
-      ":": 29,
-      ";": 30,
-      "<": 31,
+      "!": 3,
+      "\"": 4,
+      "#": 5,
+      "$": 6,
+      "%": 7,
+      "&": 8,
+      "'": 9,
+      "(": 10,
+      ")": 11,
+      "*": 12,
+      "+": 13,
+      ",": 14,
+      "-": 15,
+      ".": 16,
+      "/": 17,
+      "0": 18,
+      "1": 19,
+      "2": 20,
+      "3": 21,
+      "4": 22,
+      "5": 23,
+      "6": 24,
+      "7": 25,
+      "8": 26,
+      "9": 27,
+      ":": 28,
+      ";": 29,
+      "<": 30,
+      "=": 31,
       ">": 32,
       "?": 33,
       "@": 34,

tokenizer_config.json

@@ -23,28 +23,8 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "3": {
-      "content": "+",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "4": {
-      "content": "=",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
-  "additional_special_tokens": [
-    "+",
-    "="
-  ],
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",