update

README.md

@@ -4,7 +4,7 @@ emoji: ⚡
 colorFrom: red
 colorTo: gray
 sdk: gradio
-sdk_version: 3.41.2
+sdk_version: 4.28.3
 app_file: app.py
 pinned: false
 ---

app.py

@@ -7,7 +7,7 @@ from patcher.gr_interface import TabbedInterface
 
 demo = TabbedInterface(
     [tab_playground, tab_compression],
-    [" ⚔️Playground", "🏆 Compression Leaderboard",],  # 编码速度，解码速度，字符分类(zh、num等，支持正则)，支持的语言，机构，。
+    [" ⚔️ Playground", "🏆 Compression Leaderboard",],  # 编码速度，解码速度，字符分类(zh、num等，支持正则)，支持的语言，机构，。
     title='<div align="center">Tokenizer Arena ⚔️</div>',
     css="css/style.css"
 )

app_compression.py

@@ -79,9 +79,9 @@ with gr.Blocks() as demo:
             # "`num_of_trillion_tokens`  `num_of_billion_tokens`\n"
             "- `b_tokens/g_bytes` measures how many billion tokens per gigabytes corpus. \n"
             "- `t_tokens/t_bytes` measures how many trillion tokens per terabytes corpus. \n"
-            "- `n_chars/n_tokens` measures how many chars per token in the tokenized corpus. \n\n"
-            "All the above measures are depend on corpus. You can reproduce this "
-            "procedure at [github](https://github.com/xu-song/tokenizer-arena/)."
+            "- `n_chars/n_tokens` measures how many chars per token in the tokenized corpus. \n"
+            # "\nAll the above measures are depend on corpus. You can reproduce this "
+            # "procedure at [github](https://github.com/xu-song/tokenizer-arena/)."
         )
 
     gr.Markdown("## 🏆 Compression Rate Leaderboard")

app_playground.py

@@ -58,6 +58,7 @@ with gr.Blocks() as demo:
         gr.Markdown("## Input Text")
         dropdown_examples = gr.Dropdown(
             example_types,
+            value="Examples",
             type="index",
             show_label=False,
             container=False,

stats/compress_rate.json

@@ -1864,5 +1864,2423 @@
     "n_bytes": 1540504,
     "n_tokens": 476666,
     "n_chars": 1484970
+  },
+  "gpt_neox_japanese_2_7b.cc100-en": {
+    "vocab_size": 32000,
+    "n_bytes": 1124813,
+    "n_tokens": 1121413,
+    "n_chars": 1121360
+  },
+  "gpt_neox_japanese_2_7b.cc100-zh-Hans": {
+    "vocab_size": 32000,
+    "n_bytes": 2633047,
+    "n_tokens": 1049033,
+    "n_chars": 927311
+  },
+  "aya_101.cc100-ja": {
+    "vocab_size": 250100,
+    "n_bytes": 1774770,
+    "n_tokens": 300542,
+    "n_chars": 603065
+  },
+  "baichuan.cc100-ja": {
+    "vocab_size": 64000,
+    "n_bytes": 1774770,
+    "n_tokens": 591656,
+    "n_chars": 603065
+  },
+  "baichuan2.cc100-ja": {
+    "vocab_size": 125696,
+    "n_bytes": 1774770,
+    "n_tokens": 554936,
+    "n_chars": 603065
+  },
+  "bert_base_cased.cc100-ja": {
+    "vocab_size": 28996,
+    "n_bytes": 1774770,
+    "n_tokens": 410492,
+    "n_chars": 603065
+  },
+  "bert_base_chinese.cc100-ja": {
+    "vocab_size": 21128,
+    "n_bytes": 1774770,
+    "n_tokens": 396831,
+    "n_chars": 603065
+  },
+  "bert_base_uncased.cc100-ja": {
+    "vocab_size": 30522,
+    "n_bytes": 1774770,
+    "n_tokens": 580634,
+    "n_chars": 603065
+  },
+  "bloom.cc100-ja": {
+    "vocab_size": 250680,
+    "n_bytes": 1774770,
+    "n_tokens": 523592,
+    "n_chars": 603065
+  },
+  "byt5_small.cc100-ja": {
+    "vocab_size": 384,
+    "n_bytes": 1774770,
+    "n_tokens": 1784770,
+    "n_chars": 603065
+  },
+  "aya_101.cc100-ar": {
+    "vocab_size": 250100,
+    "n_bytes": 2813283,
+    "n_tokens": 631736,
+    "n_chars": 1560987
+  },
+  "baichuan.cc100-ar": {
+    "vocab_size": 64000,
+    "n_bytes": 2813283,
+    "n_tokens": 1422976,
+    "n_chars": 1560987
+  },
+  "baichuan2.cc100-ar": {
+    "vocab_size": 125696,
+    "n_bytes": 2813283,
+    "n_tokens": 1337285,
+    "n_chars": 1560987
+  },
+  "bert_base_cased.cc100-ar": {
+    "vocab_size": 28996,
+    "n_bytes": 2813283,
+    "n_tokens": 1232449,
+    "n_chars": 1560987
+  },
+  "bert_base_chinese.cc100-ar": {
+    "vocab_size": 21128,
+    "n_bytes": 2813283,
+    "n_tokens": 536389,
+    "n_chars": 1560987
+  },
+  "bert_base_uncased.cc100-ar": {
+    "vocab_size": 30522,
+    "n_bytes": 2813283,
+    "n_tokens": 1269370,
+    "n_chars": 1560987
+  },
+  "bloom.cc100-ar": {
+    "vocab_size": 250680,
+    "n_bytes": 2813283,
+    "n_tokens": 427489,
+    "n_chars": 1560987
+  },
+  "byt5_small.cc100-ar": {
+    "vocab_size": 384,
+    "n_bytes": 2813283,
+    "n_tokens": 2823283,
+    "n_chars": 1560987
+  },
+  "character_glm_6b.cc100-ar": {
+    "vocab_size": 64789,
+    "n_bytes": 2813283,
+    "n_tokens": 1441847,
+    "n_chars": 1560987
+  },
+  "chatglm2_6b.cc100-ar": {
+    "vocab_size": 64787,
+    "n_bytes": 2813283,
+    "n_tokens": 1441847,
+    "n_chars": 1560987
+  },
+  "chatglm3_6b.cc100-ar": {
+    "vocab_size": 64796,
+    "n_bytes": 2813283,
+    "n_tokens": 1441847,
+    "n_chars": 1560987
+  },
+  "chatglm_6b.cc100-ar": {
+    "vocab_size": 150344,
+    "n_bytes": 2813283,
+    "n_tokens": 1097200,
+    "n_chars": 1560987
+  },
+  "chatyuan_large_v2.cc100-ar": {
+    "vocab_size": 32128,
+    "n_bytes": 2813283,
+    "n_tokens": 1006313,
+    "n_chars": 1560987
+  },
+  "chinese_llama.cc100-ar": {
+    "vocab_size": 49953,
+    "n_bytes": 2813283,
+    "n_tokens": 1421625,
+    "n_chars": 1560987
+  },
+  "chinese_llama2.cc100-ar": {
+    "vocab_size": 55296,
+    "n_bytes": 2813283,
+    "n_tokens": 1432081,
+    "n_chars": 1560987
+  },
+  "code_davinci_002.cc100-ar": {
+    "vocab_size": 50281,
+    "n_bytes": 2813283,
+    "n_tokens": 1558111,
+    "n_chars": 1560987
+  },
+  "crystal_coder.cc100-ar": {
+    "vocab_size": 32022,
+    "n_bytes": 2813283,
+    "n_tokens": 1422081,
+    "n_chars": 1560987
+  },
+  "dbrx_instruct.cc100-ar": {
+    "vocab_size": 100280,
+    "n_bytes": 2813283,
+    "n_tokens": 1105640,
+    "n_chars": 1560987
+  },
+  "deepseek_coder_33b_instruct.cc100-ar": {
+    "vocab_size": 32022,
+    "n_bytes": 2813283,
+    "n_tokens": 1958863,
+    "n_chars": 1560987
+  },
+  "deepseek_llm_7b_base.cc100-ar": {
+    "vocab_size": 100015,
+    "n_bytes": 2813283,
+    "n_tokens": 1426103,
+    "n_chars": 1560987
+  },
+  "falcon_180b.cc100-ar": {
+    "vocab_size": 65024,
+    "n_bytes": 2813283,
+    "n_tokens": 1597443,
+    "n_chars": 1560987
+  },
+  "falcon_7b.cc100-ar": {
+    "vocab_size": 65024,
+    "n_bytes": 2813283,
+    "n_tokens": 1597443,
+    "n_chars": 1560987
+  },
+  "fastchat_t5_3b.cc100-ar": {
+    "vocab_size": 32110,
+    "n_bytes": 2813283,
+    "n_tokens": 832267,
+    "n_chars": 1560987
+  },
+  "flan_t5_base.cc100-ar": {
+    "vocab_size": 32100,
+    "n_bytes": 2813283,
+    "n_tokens": 568957,
+    "n_chars": 1560987
+  },
+  "gemma_7b.cc100-ar": {
+    "vocab_size": 256000,
+    "n_bytes": 2813283,
+    "n_tokens": 573788,
+    "n_chars": 1560987
+  },
+  "gpt2.cc100-ar": {
+    "vocab_size": 50257,
+    "n_bytes": 2813283,
+    "n_tokens": 1558111,
+    "n_chars": 1560987
+  },
+  "gpt2_chinese.cc100-ar": {
+    "vocab_size": 21128,
+    "n_bytes": 2813283,
+    "n_tokens": 617677,
+    "n_chars": 1560987
+  },
+  "gpt_35_turbo.cc100-ar": {
+    "vocab_size": 100277,
+    "n_bytes": 2813283,
+    "n_tokens": 1105640,
+    "n_chars": 1560987
+  },
+  "gpt_4.cc100-ar": {
+    "vocab_size": 100277,
+    "n_bytes": 2813283,
+    "n_tokens": 1105640,
+    "n_chars": 1560987
+  },
+  "gpt_neox_japanese_2_7b.cc100-ar": {
+    "vocab_size": 32000,
+    "n_bytes": 2813283,
+    "n_tokens": 2809195,
+    "n_chars": 1560987
+  },
+  "gpt_nexo_20b.cc100-ar": {
+    "vocab_size": 50277,
+    "n_bytes": 2813283,
+    "n_tokens": 1106277,
+    "n_chars": 1560987
+  },
+  "grok_1.cc100-ar": {
+    "vocab_size": 131072,
+    "n_bytes": 2813283,
+    "n_tokens": 1392088,
+    "n_chars": 1560987
+  },
+  "internlm2_chat_7b.cc100-ar": {
+    "vocab_size": 92544,
+    "n_bytes": 2813283,
+    "n_tokens": 1635378,
+    "n_chars": 1560987
+  },
+  "internlm2_math_7b.cc100-ar": {
+    "vocab_size": 92544,
+    "n_bytes": 2813283,
+    "n_tokens": 1635378,
+    "n_chars": 1560987
+  },
+  "internlm_chat_7b.cc100-ar": {
+    "vocab_size": 103168,
+    "n_bytes": 2813283,
+    "n_tokens": 532046,
+    "n_chars": 1560987
+  },
+  "internlm_xcomposer_7b.cc100-ar": {
+    "vocab_size": 103168,
+    "n_bytes": 2813283,
+    "n_tokens": 532046,
+    "n_chars": 1560987
+  },
+  "jamba_v0_1.cc100-ar": {
+    "vocab_size": 65536,
+    "n_bytes": 2813283,
+    "n_tokens": 727886,
+    "n_chars": 1560987
+  },
+  "kplug.cc100-ar": {
+    "vocab_size": 10261,
+    "n_bytes": 2813283,
+    "n_tokens": 331987,
+    "n_chars": 1560987
+  },
+  "llama.cc100-ar": {
+    "vocab_size": 32000,
+    "n_bytes": 2813283,
+    "n_tokens": 1432081,
+    "n_chars": 1560987
+  },
+  "llama2.cc100-ar": {
+    "vocab_size": 32001,
+    "n_bytes": 2813283,
+    "n_tokens": 1432081,
+    "n_chars": 1560987
+  },
+  "llama3.cc100-ar": {
+    "vocab_size": 128256,
+    "n_bytes": 2813283,
+    "n_tokens": 615514,
+    "n_chars": 1560987
+  },
+  "mistral_7b.cc100-ar": {
+    "vocab_size": 32000,
+    "n_bytes": 2813283,
+    "n_tokens": 1406319,
+    "n_chars": 1560987
+  },
+  "mixtral_8_7b.cc100-ar": {
+    "vocab_size": 32000,
+    "n_bytes": 2813283,
+    "n_tokens": 1406319,
+    "n_chars": 1560987
+  },
+  "mobilebert_uncased.cc100-ar": {
+    "vocab_size": 30522,
+    "n_bytes": 2813283,
+    "n_tokens": 1269370,
+    "n_chars": 1560987
+  },
+  "moss.cc100-ar": {
+    "vocab_size": 106072,
+    "n_bytes": 2813283,
+    "n_tokens": 1557671,
+    "n_chars": 1560987
+  },
+  "mt5_large.cc100-ar": {
+    "vocab_size": 250100,
+    "n_bytes": 2813283,
+    "n_tokens": 631736,
+    "n_chars": 1560987
+  },
+  "olmo_7b.cc100-ar": {
+    "vocab_size": 50280,
+    "n_bytes": 2813283,
+    "n_tokens": 1106277,
+    "n_chars": 1560987
+  },
+  "orion_14b_chat.cc100-ar": {
+    "vocab_size": 84608,
+    "n_bytes": 2813283,
+    "n_tokens": 1531053,
+    "n_chars": 1560987
+  },
+  "phi_1.cc100-ar": {
+    "vocab_size": 50295,
+    "n_bytes": 2813283,
+    "n_tokens": 1558111,
+    "n_chars": 1560987
+  },
+  "phi_2.cc100-ar": {
+    "vocab_size": 50295,
+    "n_bytes": 2813283,
+    "n_tokens": 1558111,
+    "n_chars": 1560987
+  },
+  "phi_3_mini.cc100-ar": {
+    "vocab_size": 32011,
+    "n_bytes": 2813283,
+    "n_tokens": 1432081,
+    "n_chars": 1560987
+  },
+  "pko_t5_large.cc100-ar": {
+    "vocab_size": 50358,
+    "n_bytes": 2813283,
+    "n_tokens": 2815586,
+    "n_chars": 1560987
+  },
+  "prompt_clue.cc100-ar": {
+    "vocab_size": 32128,
+    "n_bytes": 2813283,
+    "n_tokens": 1006313,
+    "n_chars": 1560987
+  },
+  "qwen1_5_14b_chat.cc100-ar": {
+    "vocab_size": 151646,
+    "n_bytes": 2813283,
+    "n_tokens": 614959,
+    "n_chars": 1560987
+  },
+  "qwen_1_8b_chat.cc100-ar": {
+    "vocab_size": 151851,
+    "n_bytes": 2813283,
+    "n_tokens": 614959,
+    "n_chars": 1560987
+  },
+  "qwen_72b_chat.cc100-ar": {
+    "vocab_size": 151851,
+    "n_bytes": 2813283,
+    "n_tokens": 614959,
+    "n_chars": 1560987
+  },
+  "qwen_7b_chat.cc100-ar": {
+    "vocab_size": 151851,
+    "n_bytes": 2813283,
+    "n_tokens": 614959,
+    "n_chars": 1560987
+  },
+  "roberta_chinese_clue.cc100-ar": {
+    "vocab_size": 8021,
+    "n_bytes": 2813283,
+    "n_tokens": 621762,
+    "n_chars": 1560987
+  },
+  "skywork_13b_base.cc100-ar": {
+    "vocab_size": 65519,
+    "n_bytes": 2813283,
+    "n_tokens": 1432065,
+    "n_chars": 1560987
+  },
+  "skywork_13b_math.cc100-ar": {
+    "vocab_size": 65519,
+    "n_bytes": 2813283,
+    "n_tokens": 1432065,
+    "n_chars": 1560987
+  },
+  "solar_10_7b.cc100-ar": {
+    "vocab_size": 32000,
+    "n_bytes": 2813283,
+    "n_tokens": 1406319,
+    "n_chars": 1560987
+  },
+  "starchat_alpha.cc100-ar": {
+    "vocab_size": 49156,
+    "n_bytes": 2813283,
+    "n_tokens": 1195640,
+    "n_chars": 1560987
+  },
+  "switch_c_2048.cc100-ar": {
+    "vocab_size": 32100,
+    "n_bytes": 2813283,
+    "n_tokens": 568855,
+    "n_chars": 1560987
+  },
+  "t5_base.cc100-ar": {
+    "vocab_size": 32100,
+    "n_bytes": 2813283,
+    "n_tokens": 568855,
+    "n_chars": 1560987
+  },
+  "t5_large.cc100-ar": {
+    "vocab_size": 32100,
+    "n_bytes": 2813283,
+    "n_tokens": 568855,
+    "n_chars": 1560987
+  },
+  "t5_small.cc100-ar": {
+    "vocab_size": 32100,
+    "n_bytes": 2813283,
+    "n_tokens": 568855,
+    "n_chars": 1560987
+  },
+  "text_davinci_003.cc100-ar": {
+    "vocab_size": 50281,
+    "n_bytes": 2813283,
+    "n_tokens": 1558111,
+    "n_chars": 1560987
+  },
+  "tigerbot_13b_chat_v2.cc100-ar": {
+    "vocab_size": 60515,
+    "n_bytes": 2813283,
+    "n_tokens": 1422070,
+    "n_chars": 1560987
+  },
+  "tigerbot_70b_chat_v4_4k.cc100-ar": {
+    "vocab_size": 65110,
+    "n_bytes": 2813283,
+    "n_tokens": 1422073,
+    "n_chars": 1560987
+  },
+  "wizardcoder_15b_v1.cc100-ar": {
+    "vocab_size": 49153,
+    "n_bytes": 2813283,
+    "n_tokens": 1195640,
+    "n_chars": 1560987
+  },
+  "wizardcoder_python_7b_v1.cc100-ar": {
+    "vocab_size": 32001,
+    "n_bytes": 2813283,
+    "n_tokens": 1432081,
+    "n_chars": 1560987
+  },
+  "wizardlm_7b_v1.cc100-ar": {
+    "vocab_size": 32001,
+    "n_bytes": 2813283,
+    "n_tokens": 1432081,
+    "n_chars": 1560987
+  },
+  "wizardmath_70b_v1.cc100-ar": {
+    "vocab_size": 32002,
+    "n_bytes": 2813283,
+    "n_tokens": 1432081,
+    "n_chars": 1560987
+  },
+  "xlm_roberta.cc100-ar": {
+    "vocab_size": 250002,
+    "n_bytes": 2813283,
+    "n_tokens": 518287,
+    "n_chars": 1560987
+  },
+  "yi_34b.cc100-ar": {
+    "vocab_size": 64000,
+    "n_bytes": 2813283,
+    "n_tokens": 1795801,
+    "n_chars": 1560987
+  },
+  "yi_6b.cc100-ar": {
+    "vocab_size": 64000,
+    "n_bytes": 2813283,
+    "n_tokens": 1795801,
+    "n_chars": 1560987
+  },
+  "yi_vl34b.cc100-ar": {
+    "vocab_size": 64000,
+    "n_bytes": 2813283,
+    "n_tokens": 1803957,
+    "n_chars": 1560987
+  },
+  "zephyr_7b_beta.cc100-ar": {
+    "vocab_size": 32000,
+    "n_bytes": 2813283,
+    "n_tokens": 1406319,
+    "n_chars": 1560987
+  },
+  "aya_101.cc100-de": {
+    "vocab_size": 250100,
+    "n_bytes": 1814876,
+    "n_tokens": 480418,
+    "n_chars": 1784021
+  },
+  "baichuan.cc100-de": {
+    "vocab_size": 64000,
+    "n_bytes": 1814876,
+    "n_tokens": 680512,
+    "n_chars": 1784021
+  },
+  "baichuan2.cc100-de": {
+    "vocab_size": 125696,
+    "n_bytes": 1814876,
+    "n_tokens": 628063,
+    "n_chars": 1784021
+  },
+  "bert_base_cased.cc100-de": {
+    "vocab_size": 28996,
+    "n_bytes": 1814876,
+    "n_tokens": 731093,
+    "n_chars": 1784021
+  },
+  "bert_base_chinese.cc100-de": {
+    "vocab_size": 21128,
+    "n_bytes": 1814876,
+    "n_tokens": 561246,
+    "n_chars": 1784021
+  },
+  "bert_base_uncased.cc100-de": {
+    "vocab_size": 30522,
+    "n_bytes": 1814876,
+    "n_tokens": 646485,
+    "n_chars": 1784021
+  },
+  "bloom.cc100-de": {
+    "vocab_size": 250680,
+    "n_bytes": 1814876,
+    "n_tokens": 541170,
+    "n_chars": 1784021
+  },
+  "byt5_small.cc100-de": {
+    "vocab_size": 384,
+    "n_bytes": 1814876,
+    "n_tokens": 1824876,
+    "n_chars": 1784021
+  },
+  "character_glm_6b.cc100-de": {
+    "vocab_size": 64789,
+    "n_bytes": 1814876,
+    "n_tokens": 639822,
+    "n_chars": 1784021
+  },
+  "chatglm2_6b.cc100-de": {
+    "vocab_size": 64787,
+    "n_bytes": 1814876,
+    "n_tokens": 639757,
+    "n_chars": 1784021
+  },
+  "chatglm3_6b.cc100-de": {
+    "vocab_size": 64796,
+    "n_bytes": 1814876,
+    "n_tokens": 639822,
+    "n_chars": 1784021
+  },
+  "chatglm_6b.cc100-de": {
+    "vocab_size": 150344,
+    "n_bytes": 1814876,
+    "n_tokens": 589464,
+    "n_chars": 1784021
+  },
+  "chatyuan_large_v2.cc100-de": {
+    "vocab_size": 32128,
+    "n_bytes": 1814876,
+    "n_tokens": 970463,
+    "n_chars": 1784021
+  },
+  "chinese_llama.cc100-de": {
+    "vocab_size": 49953,
+    "n_bytes": 1814876,
+    "n_tokens": 523859,
+    "n_chars": 1784021
+  },
+  "chinese_llama2.cc100-de": {
+    "vocab_size": 55296,
+    "n_bytes": 1814876,
+    "n_tokens": 537318,
+    "n_chars": 1784021
+  },
+  "code_davinci_002.cc100-de": {
+    "vocab_size": 50281,
+    "n_bytes": 1814876,
+    "n_tokens": 684666,
+    "n_chars": 1784021
+  },
+  "crystal_coder.cc100-de": {
+    "vocab_size": 32022,
+    "n_bytes": 1814876,
+    "n_tokens": 527320,
+    "n_chars": 1784021
+  },
+  "dbrx_instruct.cc100-de": {
+    "vocab_size": 100280,
+    "n_bytes": 1814876,
+    "n_tokens": 500870,
+    "n_chars": 1784021
+  },
+  "deepseek_coder_33b_instruct.cc100-de": {
+    "vocab_size": 32022,
+    "n_bytes": 1814876,
+    "n_tokens": 745618,
+    "n_chars": 1784021
+  },
+  "deepseek_llm_7b_base.cc100-de": {
+    "vocab_size": 100015,
+    "n_bytes": 1814876,
+    "n_tokens": 642573,
+    "n_chars": 1784021
+  },
+  "falcon_180b.cc100-de": {
+    "vocab_size": 65024,
+    "n_bytes": 1814876,
+    "n_tokens": 497054,
+    "n_chars": 1784021
+  },
+  "falcon_7b.cc100-de": {
+    "vocab_size": 65024,
+    "n_bytes": 1814876,
+    "n_tokens": 497054,
+    "n_chars": 1784021
+  },
+  "fastchat_t5_3b.cc100-de": {
+    "vocab_size": 32110,
+    "n_bytes": 1814876,
+    "n_tokens": 736989,
+    "n_chars": 1784021
+  },
+  "flan_t5_base.cc100-de": {
+    "vocab_size": 32100,
+    "n_bytes": 1814876,
+    "n_tokens": 480254,
+    "n_chars": 1784021
+  },
+  "gemma_7b.cc100-de": {
+    "vocab_size": 256000,
+    "n_bytes": 1814876,
+    "n_tokens": 416876,
+    "n_chars": 1784021
+  },
+  "gpt2.cc100-de": {
+    "vocab_size": 50257,
+    "n_bytes": 1814876,
+    "n_tokens": 684669,
+    "n_chars": 1784021
+  },
+  "gpt2_chinese.cc100-de": {
+    "vocab_size": 21128,
+    "n_bytes": 1814876,
+    "n_tokens": 786497,
+    "n_chars": 1784021
+  },
+  "gpt_35_turbo.cc100-de": {
+    "vocab_size": 100277,
+    "n_bytes": 1814876,
+    "n_tokens": 500870,
+    "n_chars": 1784021
+  },
+  "gpt_4.cc100-de": {
+    "vocab_size": 100277,
+    "n_bytes": 1814876,
+    "n_tokens": 500870,
+    "n_chars": 1784021
+  },
+  "gpt_neox_japanese_2_7b.cc100-de": {
+    "vocab_size": 32000,
+    "n_bytes": 1814876,
+    "n_tokens": 1807780,
+    "n_chars": 1784021
+  },
+  "gpt_nexo_20b.cc100-de": {
+    "vocab_size": 50277,
+    "n_bytes": 1814876,
+    "n_tokens": 583628,
+    "n_chars": 1784021
+  },
+  "grok_1.cc100-de": {
+    "vocab_size": 131072,
+    "n_bytes": 1814876,
+    "n_tokens": 505220,
+    "n_chars": 1784021
+  },
+  "internlm2_chat_7b.cc100-de": {
+    "vocab_size": 92544,
+    "n_bytes": 1814876,
+    "n_tokens": 583917,
+    "n_chars": 1784021
+  },
+  "internlm2_math_7b.cc100-de": {
+    "vocab_size": 92544,
+    "n_bytes": 1814876,
+    "n_tokens": 583917,
+    "n_chars": 1784021
+  },
+  "internlm_chat_7b.cc100-de": {
+    "vocab_size": 103168,
+    "n_bytes": 1814876,
+    "n_tokens": 580489,
+    "n_chars": 1784021
+  },
+  "internlm_xcomposer_7b.cc100-de": {
+    "vocab_size": 103168,
+    "n_bytes": 1814876,
+    "n_tokens": 580489,
+    "n_chars": 1784021
+  },
+  "jamba_v0_1.cc100-de": {
+    "vocab_size": 65536,
+    "n_bytes": 1814876,
+    "n_tokens": 535856,
+    "n_chars": 1784021
+  },
+  "kplug.cc100-de": {
+    "vocab_size": 10261,
+    "n_bytes": 1814876,
+    "n_tokens": 789053,
+    "n_chars": 1784021
+  },
+  "llama.cc100-de": {
+    "vocab_size": 32000,
+    "n_bytes": 1814876,
+    "n_tokens": 537320,
+    "n_chars": 1784021
+  },
+  "llama2.cc100-de": {
+    "vocab_size": 32001,
+    "n_bytes": 1814876,
+    "n_tokens": 537320,
+    "n_chars": 1784021
+  },
+  "llama3.cc100-de": {
+    "vocab_size": 128256,
+    "n_bytes": 1814876,
+    "n_tokens": 499766,
+    "n_chars": 1784021
+  },
+  "mistral_7b.cc100-de": {
+    "vocab_size": 32000,
+    "n_bytes": 1814876,
+    "n_tokens": 577526,
+    "n_chars": 1784021
+  },
+  "mixtral_8_7b.cc100-de": {
+    "vocab_size": 32000,
+    "n_bytes": 1814876,
+    "n_tokens": 577526,
+    "n_chars": 1784021
+  },
+  "mobilebert_uncased.cc100-de": {
+    "vocab_size": 30522,
+    "n_bytes": 1814876,
+    "n_tokens": 646485,
+    "n_chars": 1784021
+  },
+  "moss.cc100-de": {
+    "vocab_size": 106072,
+    "n_bytes": 1814876,
+    "n_tokens": 683401,
+    "n_chars": 1784021
+  },
+  "mt5_large.cc100-de": {
+    "vocab_size": 250100,
+    "n_bytes": 1814876,
+    "n_tokens": 480418,
+    "n_chars": 1784021
+  },
+  "olmo_7b.cc100-de": {
+    "vocab_size": 50280,
+    "n_bytes": 1814876,
+    "n_tokens": 583628,
+    "n_chars": 1784021
+  },
+  "orion_14b_chat.cc100-de": {
+    "vocab_size": 84608,
+    "n_bytes": 1814876,
+    "n_tokens": 744404,
+    "n_chars": 1784021
+  },
+  "phi_1.cc100-de": {
+    "vocab_size": 50295,
+    "n_bytes": 1814876,
+    "n_tokens": 684665,
+    "n_chars": 1784021
+  },
+  "phi_2.cc100-de": {
+    "vocab_size": 50295,
+    "n_bytes": 1814876,
+    "n_tokens": 684665,
+    "n_chars": 1784021
+  },
+  "phi_3_mini.cc100-de": {
+    "vocab_size": 32011,
+    "n_bytes": 1814876,
+    "n_tokens": 537320,
+    "n_chars": 1784021
+  },
+  "pko_t5_large.cc100-de": {
+    "vocab_size": 50358,
+    "n_bytes": 1814876,
+    "n_tokens": 1254350,
+    "n_chars": 1784021
+  },
+  "prompt_clue.cc100-de": {
+    "vocab_size": 32128,
+    "n_bytes": 1814876,
+    "n_tokens": 970463,
+    "n_chars": 1784021
+  },
+  "qwen1_5_14b_chat.cc100-de": {
+    "vocab_size": 151646,
+    "n_bytes": 1814876,
+    "n_tokens": 503561,
+    "n_chars": 1784021
+  },
+  "qwen_1_8b_chat.cc100-de": {
+    "vocab_size": 151851,
+    "n_bytes": 1814876,
+    "n_tokens": 503561,
+    "n_chars": 1784021
+  },
+  "qwen_72b_chat.cc100-de": {
+    "vocab_size": 151851,
+    "n_bytes": 1814876,
+    "n_tokens": 503561,
+    "n_chars": 1784021
+  },
+  "qwen_7b_chat.cc100-de": {
+    "vocab_size": 151851,
+    "n_bytes": 1814876,
+    "n_tokens": 503561,
+    "n_chars": 1784021
+  },
+  "roberta_chinese_clue.cc100-de": {
+    "vocab_size": 8021,
+    "n_bytes": 1814876,
+    "n_tokens": 915612,
+    "n_chars": 1784021
+  },
+  "skywork_13b_base.cc100-de": {
+    "vocab_size": 65519,
+    "n_bytes": 1814876,
+    "n_tokens": 537308,
+    "n_chars": 1784021
+  },
+  "skywork_13b_math.cc100-de": {
+    "vocab_size": 65519,
+    "n_bytes": 1814876,
+    "n_tokens": 537308,
+    "n_chars": 1784021
+  },
+  "solar_10_7b.cc100-de": {
+    "vocab_size": 32000,
+    "n_bytes": 1814876,
+    "n_tokens": 577526,
+    "n_chars": 1784021
+  },
+  "starchat_alpha.cc100-de": {
+    "vocab_size": 49156,
+    "n_bytes": 1814876,
+    "n_tokens": 620541,
+    "n_chars": 1784021
+  },
+  "switch_c_2048.cc100-de": {
+    "vocab_size": 32100,
+    "n_bytes": 1814876,
+    "n_tokens": 480254,
+    "n_chars": 1784021
+  },
+  "t5_base.cc100-de": {
+    "vocab_size": 32100,
+    "n_bytes": 1814876,
+    "n_tokens": 480254,
+    "n_chars": 1784021
+  },
+  "t5_large.cc100-de": {
+    "vocab_size": 32100,
+    "n_bytes": 1814876,
+    "n_tokens": 480254,
+    "n_chars": 1784021
+  },
+  "t5_small.cc100-de": {
+    "vocab_size": 32100,
+    "n_bytes": 1814876,
+    "n_tokens": 480254,
+    "n_chars": 1784021
+  },
+  "text_davinci_003.cc100-de": {
+    "vocab_size": 50281,
+    "n_bytes": 1814876,
+    "n_tokens": 684666,
+    "n_chars": 1784021
+  },
+  "tigerbot_13b_chat_v2.cc100-de": {
+    "vocab_size": 60515,
+    "n_bytes": 1814876,
+    "n_tokens": 528918,
+    "n_chars": 1784021
+  },
+  "tigerbot_70b_chat_v4_4k.cc100-de": {
+    "vocab_size": 65110,
+    "n_bytes": 1814876,
+    "n_tokens": 529170,
+    "n_chars": 1784021
+  },
+  "wizardcoder_15b_v1.cc100-de": {
+    "vocab_size": 49153,
+    "n_bytes": 1814876,
+    "n_tokens": 620541,
+    "n_chars": 1784021
+  },
+  "wizardcoder_python_7b_v1.cc100-de": {
+    "vocab_size": 32001,
+    "n_bytes": 1814876,
+    "n_tokens": 537320,
+    "n_chars": 1784021
+  },
+  "wizardlm_7b_v1.cc100-de": {
+    "vocab_size": 32001,
+    "n_bytes": 1814876,
+    "n_tokens": 537320,
+    "n_chars": 1784021
+  },
+  "wizardmath_70b_v1.cc100-de": {
+    "vocab_size": 32002,
+    "n_bytes": 1814876,
+    "n_tokens": 537320,
+    "n_chars": 1784021
+  },
+  "xlm_roberta.cc100-de": {
+    "vocab_size": 250002,
+    "n_bytes": 1814876,
+    "n_tokens": 432571,
+    "n_chars": 1784021
+  },
+  "yi_34b.cc100-de": {
+    "vocab_size": 64000,
+    "n_bytes": 1814876,
+    "n_tokens": 698366,
+    "n_chars": 1784021
+  },
+  "yi_6b.cc100-de": {
+    "vocab_size": 64000,
+    "n_bytes": 1814876,
+    "n_tokens": 698366,
+    "n_chars": 1784021
+  },
+  "yi_vl34b.cc100-de": {
+    "vocab_size": 64000,
+    "n_bytes": 1814876,
+    "n_tokens": 697065,
+    "n_chars": 1784021
+  },
+  "zephyr_7b_beta.cc100-de": {
+    "vocab_size": 32000,
+    "n_bytes": 1814876,
+    "n_tokens": 577526,
+    "n_chars": 1784021
+  },
+  "gpt_neox_japanese_2_7b.cc100-es": {
+    "vocab_size": 32000,
+    "n_bytes": 1664455,
+    "n_tokens": 1658946,
+    "n_chars": 1630297
+  },
+  "gpt_neox_japanese_2_7b.cc100-fr": {
+    "vocab_size": 32000,
+    "n_bytes": 1540504,
+    "n_tokens": 1524129,
+    "n_chars": 1484970
+  },
+  "character_glm_6b.cc100-ja": {
+    "vocab_size": 64789,
+    "n_bytes": 1774770,
+    "n_tokens": 601380,
+    "n_chars": 603065
+  },
+  "chatglm2_6b.cc100-ja": {
+    "vocab_size": 64787,
+    "n_bytes": 1774770,
+    "n_tokens": 601380,
+    "n_chars": 603065
+  },
+  "chatglm3_6b.cc100-ja": {
+    "vocab_size": 64796,
+    "n_bytes": 1774770,
+    "n_tokens": 601380,
+    "n_chars": 603065
+  },
+  "chatglm_6b.cc100-ja": {
+    "vocab_size": 150344,
+    "n_bytes": 1774770,
+    "n_tokens": 489930,
+    "n_chars": 603065
+  },
+  "chatyuan_large_v2.cc100-ja": {
+    "vocab_size": 32128,
+    "n_bytes": 1774770,
+    "n_tokens": 575118,
+    "n_chars": 603065
+  },
+  "chinese_llama.cc100-ja": {
+    "vocab_size": 49953,
+    "n_bytes": 1774770,
+    "n_tokens": 614177,
+    "n_chars": 603065
+  },
+  "chinese_llama2.cc100-ja": {
+    "vocab_size": 55296,
+    "n_bytes": 1774770,
+    "n_tokens": 624362,
+    "n_chars": 603065
+  },
+  "code_davinci_002.cc100-ja": {
+    "vocab_size": 50281,
+    "n_bytes": 1774770,
+    "n_tokens": 844362,
+    "n_chars": 603065
+  },
+  "crystal_coder.cc100-ja": {
+    "vocab_size": 32022,
+    "n_bytes": 1774770,
+    "n_tokens": 718461,
+    "n_chars": 603065
+  },
+  "dbrx_instruct.cc100-ja": {
+    "vocab_size": 100280,
+    "n_bytes": 1774770,
+    "n_tokens": 630348,
+    "n_chars": 603065
+  },
+  "deepseek_coder_33b_instruct.cc100-ja": {
+    "vocab_size": 32022,
+    "n_bytes": 1774770,
+    "n_tokens": 1018060,
+    "n_chars": 603065
+  },
+  "deepseek_llm_7b_base.cc100-ja": {
+    "vocab_size": 100015,
+    "n_bytes": 1774770,
+    "n_tokens": 761467,
+    "n_chars": 603065
+  },
+  "falcon_180b.cc100-ja": {
+    "vocab_size": 65024,
+    "n_bytes": 1774770,
+    "n_tokens": 842458,
+    "n_chars": 603065
+  },
+  "falcon_7b.cc100-ja": {
+    "vocab_size": 65024,
+    "n_bytes": 1774770,
+    "n_tokens": 842458,
+    "n_chars": 603065
+  },
+  "fastchat_t5_3b.cc100-ja": {
+    "vocab_size": 32110,
+    "n_bytes": 1774770,
+    "n_tokens": 53915,
+    "n_chars": 603065
+  },
+  "flan_t5_base.cc100-ja": {
+    "vocab_size": 32100,
+    "n_bytes": 1774770,
+    "n_tokens": 51999,
+    "n_chars": 603065
+  },
+  "gemma_7b.cc100-ja": {
+    "vocab_size": 256000,
+    "n_bytes": 1774770,
+    "n_tokens": 317873,
+    "n_chars": 603065
+  },
+  "gpt2.cc100-ja": {
+    "vocab_size": 50257,
+    "n_bytes": 1774770,
+    "n_tokens": 844362,
+    "n_chars": 603065
+  },
+  "gpt2_chinese.cc100-ja": {
+    "vocab_size": 21128,
+    "n_bytes": 1774770,
+    "n_tokens": 503085,
+    "n_chars": 603065
+  },
+  "gpt_35_turbo.cc100-ja": {
+    "vocab_size": 100277,
+    "n_bytes": 1774770,
+    "n_tokens": 630348,
+    "n_chars": 603065
+  },
+  "gpt_4.cc100-ja": {
+    "vocab_size": 100277,
+    "n_bytes": 1774770,
+    "n_tokens": 630348,
+    "n_chars": 603065
+  },
+  "gpt_neox_japanese_2_7b.cc100-ja": {
+    "vocab_size": 32000,
+    "n_bytes": 1774770,
+    "n_tokens": 410803,
+    "n_chars": 603065
+  },
+  "gpt_nexo_20b.cc100-ja": {
+    "vocab_size": 50277,
+    "n_bytes": 1774770,
+    "n_tokens": 605168,
+    "n_chars": 603065
+  },
+  "grok_1.cc100-ja": {
+    "vocab_size": 131072,
+    "n_bytes": 1774770,
+    "n_tokens": 497590,
+    "n_chars": 603065
+  },
+  "internlm2_chat_7b.cc100-ja": {
+    "vocab_size": 92544,
+    "n_bytes": 1774770,
+    "n_tokens": 595803,
+    "n_chars": 603065
+  },
+  "internlm2_math_7b.cc100-ja": {
+    "vocab_size": 92544,
+    "n_bytes": 1774770,
+    "n_tokens": 595803,
+    "n_chars": 603065
+  },
+  "internlm_chat_7b.cc100-ja": {
+    "vocab_size": 103168,
+    "n_bytes": 1774770,
+    "n_tokens": 448212,
+    "n_chars": 603065
+  },
+  "internlm_xcomposer_7b.cc100-ja": {
+    "vocab_size": 103168,
+    "n_bytes": 1774770,
+    "n_tokens": 448212,
+    "n_chars": 603065
+  },
+  "jamba_v0_1.cc100-ja": {
+    "vocab_size": 65536,
+    "n_bytes": 1774770,
+    "n_tokens": 683256,
+    "n_chars": 603065
+  },
+  "kplug.cc100-ja": {
+    "vocab_size": 10261,
+    "n_bytes": 1774770,
+    "n_tokens": 338023,
+    "n_chars": 603065
+  },
+  "llama.cc100-ja": {
+    "vocab_size": 32000,
+    "n_bytes": 1774770,
+    "n_tokens": 728461,
+    "n_chars": 603065
+  },
+  "llama2.cc100-ja": {
+    "vocab_size": 32001,
+    "n_bytes": 1774770,
+    "n_tokens": 728461,
+    "n_chars": 603065
+  },
+  "llama3.cc100-ja": {
+    "vocab_size": 128256,
+    "n_bytes": 1774770,
+    "n_tokens": 414715,
+    "n_chars": 603065
+  },
+  "mistral_7b.cc100-ja": {
+    "vocab_size": 32000,
+    "n_bytes": 1774770,
+    "n_tokens": 685134,
+    "n_chars": 603065
+  },
+  "mixtral_8_7b.cc100-ja": {
+    "vocab_size": 32000,
+    "n_bytes": 1774770,
+    "n_tokens": 685134,
+    "n_chars": 603065
+  },
+  "mobilebert_uncased.cc100-ja": {
+    "vocab_size": 30522,
+    "n_bytes": 1774770,
+    "n_tokens": 580634,
+    "n_chars": 603065
+  },
+  "moss.cc100-ja": {
+    "vocab_size": 106072,
+    "n_bytes": 1774770,
+    "n_tokens": 600011,
+    "n_chars": 603065
+  },
+  "mt5_large.cc100-ja": {
+    "vocab_size": 250100,
+    "n_bytes": 1774770,
+    "n_tokens": 300542,
+    "n_chars": 603065
+  },
+  "olmo_7b.cc100-ja": {
+    "vocab_size": 50280,
+    "n_bytes": 1774770,
+    "n_tokens": 605168,
+    "n_chars": 603065
+  },
+  "orion_14b_chat.cc100-ja": {
+    "vocab_size": 84608,
+    "n_bytes": 1774770,
+    "n_tokens": 324956,
+    "n_chars": 603065
+  },
+  "phi_1.cc100-ja": {
+    "vocab_size": 50295,
+    "n_bytes": 1774770,
+    "n_tokens": 844362,
+    "n_chars": 603065
+  },
+  "phi_2.cc100-ja": {
+    "vocab_size": 50295,
+    "n_bytes": 1774770,
+    "n_tokens": 844362,
+    "n_chars": 603065
+  },
+  "phi_3_mini.cc100-ja": {
+    "vocab_size": 32011,
+    "n_bytes": 1774770,
+    "n_tokens": 728461,
+    "n_chars": 603065
+  },
+  "pko_t5_large.cc100-ja": {
+    "vocab_size": 50358,
+    "n_bytes": 1774770,
+    "n_tokens": 1766950,
+    "n_chars": 603065
+  },
+  "prompt_clue.cc100-ja": {
+    "vocab_size": 32128,
+    "n_bytes": 1774770,
+    "n_tokens": 575118,
+    "n_chars": 603065
+  },
+  "qwen1_5_14b_chat.cc100-ja": {
+    "vocab_size": 151646,
+    "n_bytes": 1774770,
+    "n_tokens": 377144,
+    "n_chars": 603065
+  },
+  "qwen_1_8b_chat.cc100-ja": {
+    "vocab_size": 151851,
+    "n_bytes": 1774770,
+    "n_tokens": 377144,
+    "n_chars": 603065
+  },
+  "qwen_72b_chat.cc100-ja": {
+    "vocab_size": 151851,
+    "n_bytes": 1774770,
+    "n_tokens": 377144,
+    "n_chars": 603065
+  },
+  "qwen_7b_chat.cc100-ja": {
+    "vocab_size": 151851,
+    "n_bytes": 1774770,
+    "n_tokens": 377144,
+    "n_chars": 603065
+  },
+  "roberta_chinese_clue.cc100-ja": {
+    "vocab_size": 8021,
+    "n_bytes": 1774770,
+    "n_tokens": 339411,
+    "n_chars": 603065
+  },
+  "skywork_13b_base.cc100-ja": {
+    "vocab_size": 65519,
+    "n_bytes": 1774770,
+    "n_tokens": 603613,
+    "n_chars": 603065
+  },
+  "skywork_13b_math.cc100-ja": {
+    "vocab_size": 65519,
+    "n_bytes": 1774770,
+    "n_tokens": 603613,
+    "n_chars": 603065
+  },
+  "solar_10_7b.cc100-ja": {
+    "vocab_size": 32000,
+    "n_bytes": 1774770,
+    "n_tokens": 685134,
+    "n_chars": 603065
+  },
+  "starchat_alpha.cc100-ja": {
+    "vocab_size": 49156,
+    "n_bytes": 1774770,
+    "n_tokens": 546876,
+    "n_chars": 603065
+  },
+  "switch_c_2048.cc100-ja": {
+    "vocab_size": 32100,
+    "n_bytes": 1774770,
+    "n_tokens": 51947,
+    "n_chars": 603065
+  },
+  "t5_base.cc100-ja": {
+    "vocab_size": 32100,
+    "n_bytes": 1774770,
+    "n_tokens": 51947,
+    "n_chars": 603065
+  },
+  "t5_large.cc100-ja": {
+    "vocab_size": 32100,
+    "n_bytes": 1774770,
+    "n_tokens": 51947,
+    "n_chars": 603065
+  },
+  "t5_small.cc100-ja": {
+    "vocab_size": 32100,
+    "n_bytes": 1774770,
+    "n_tokens": 51947,
+    "n_chars": 603065
+  },
+  "text_davinci_003.cc100-ja": {
+    "vocab_size": 50281,
+    "n_bytes": 1774770,
+    "n_tokens": 844362,
+    "n_chars": 603065
+  },
+  "tigerbot_13b_chat_v2.cc100-ja": {
+    "vocab_size": 60515,
+    "n_bytes": 1774770,
+    "n_tokens": 567792,
+    "n_chars": 603065
+  },
+  "tigerbot_70b_chat_v4_4k.cc100-ja": {
+    "vocab_size": 65110,
+    "n_bytes": 1774770,
+    "n_tokens": 406571,
+    "n_chars": 603065
+  },
+  "wizardcoder_15b_v1.cc100-ja": {
+    "vocab_size": 49153,
+    "n_bytes": 1774770,
+    "n_tokens": 546876,
+    "n_chars": 603065
+  },
+  "wizardcoder_python_7b_v1.cc100-ja": {
+    "vocab_size": 32001,
+    "n_bytes": 1774770,
+    "n_tokens": 728461,
+    "n_chars": 603065
+  },
+  "wizardlm_7b_v1.cc100-ja": {
+    "vocab_size": 32001,
+    "n_bytes": 1774770,
+    "n_tokens": 728461,
+    "n_chars": 603065
+  },
+  "wizardmath_70b_v1.cc100-ja": {
+    "vocab_size": 32002,
+    "n_bytes": 1774770,
+    "n_tokens": 728461,
+    "n_chars": 603065
+  },
+  "xlm_roberta.cc100-ja": {
+    "vocab_size": 250002,
+    "n_bytes": 1774770,
+    "n_tokens": 344820,
+    "n_chars": 603065
+  },
+  "yi_34b.cc100-ja": {
+    "vocab_size": 64000,
+    "n_bytes": 1774770,
+    "n_tokens": 740791,
+    "n_chars": 603065
+  },
+  "yi_6b.cc100-ja": {
+    "vocab_size": 64000,
+    "n_bytes": 1774770,
+    "n_tokens": 740791,
+    "n_chars": 603065
+  },
+  "yi_vl34b.cc100-ja": {
+    "vocab_size": 64000,
+    "n_bytes": 1774770,
+    "n_tokens": 749927,
+    "n_chars": 603065
+  },
+  "zephyr_7b_beta.cc100-ja": {
+    "vocab_size": 32000,
+    "n_bytes": 1774770,
+    "n_tokens": 685134,
+    "n_chars": 603065
+  },
+  "llama_3_chinese_8b.cc100-ar": {
+    "vocab_size": 128256,
+    "n_bytes": 2813283,
+    "n_tokens": 625514,
+    "n_chars": 1560987
+  },
+  "llama_3_chinese_8b.cc100-de": {
+    "vocab_size": 128256,
+    "n_bytes": 1814876,
+    "n_tokens": 509766,
+    "n_chars": 1784021
+  },
+  "llama_3_chinese_8b.cc100-en": {
+    "vocab_size": 128256,
+    "n_bytes": 1124813,
+    "n_tokens": 264944,
+    "n_chars": 1121360
+  },
+  "llama_3_chinese_8b.cc100-es": {
+    "vocab_size": 128256,
+    "n_bytes": 1664455,
+    "n_tokens": 443289,
+    "n_chars": 1630297
+  },
+  "aya_101.cc100-fa": {
+    "vocab_size": 250100,
+    "n_bytes": 2054052,
+    "n_tokens": 429922,
+    "n_chars": 1145876
+  },
+  "baichuan.cc100-fa": {
+    "vocab_size": 64000,
+    "n_bytes": 2054052,
+    "n_tokens": 1142057,
+    "n_chars": 1145876
+  },
+  "baichuan2.cc100-fa": {
+    "vocab_size": 125696,
+    "n_bytes": 2054052,
+    "n_tokens": 1052077,
+    "n_chars": 1145876
+  },
+  "bert_base_cased.cc100-fa": {
+    "vocab_size": 28996,
+    "n_bytes": 2054052,
+    "n_tokens": 903078,
+    "n_chars": 1145876
+  },
+  "bert_base_chinese.cc100-fa": {
+    "vocab_size": 21128,
+    "n_bytes": 2054052,
+    "n_tokens": 396414,
+    "n_chars": 1145876
+  },
+  "bert_base_uncased.cc100-fa": {
+    "vocab_size": 30522,
+    "n_bytes": 2054052,
+    "n_tokens": 910783,
+    "n_chars": 1145876
+  },
+  "bloom.cc100-fa": {
+    "vocab_size": 250680,
+    "n_bytes": 2054052,
+    "n_tokens": 434406,
+    "n_chars": 1145876
+  },
+  "byt5_small.cc100-fa": {
+    "vocab_size": 384,
+    "n_bytes": 2054052,
+    "n_tokens": 2064052,
+    "n_chars": 1145876
+  },
+  "character_glm_6b.cc100-fa": {
+    "vocab_size": 64789,
+    "n_bytes": 2054052,
+    "n_tokens": 1165051,
+    "n_chars": 1145876
+  },
+  "chatglm2_6b.cc100-fa": {
+    "vocab_size": 64787,
+    "n_bytes": 2054052,
+    "n_tokens": 1165051,
+    "n_chars": 1145876
+  },
+  "chatglm3_6b.cc100-fa": {
+    "vocab_size": 64796,
+    "n_bytes": 2054052,
+    "n_tokens": 1165051,
+    "n_chars": 1145876
+  },
+  "chatglm_6b.cc100-fa": {
+    "vocab_size": 150344,
+    "n_bytes": 2054052,
+    "n_tokens": 910808,
+    "n_chars": 1145876
+  },
+  "chatyuan_large_v2.cc100-fa": {
+    "vocab_size": 32128,
+    "n_bytes": 2054052,
+    "n_tokens": 740377,
+    "n_chars": 1145876
+  },
+  "chinese_llama.cc100-fa": {
+    "vocab_size": 49953,
+    "n_bytes": 2054052,
+    "n_tokens": 1150750,
+    "n_chars": 1145876
+  },
+  "chinese_llama2.cc100-fa": {
+    "vocab_size": 55296,
+    "n_bytes": 2054052,
+    "n_tokens": 1155078,
+    "n_chars": 1145876
+  },
+  "code_davinci_002.cc100-fa": {
+    "vocab_size": 50281,
+    "n_bytes": 2054052,
+    "n_tokens": 1292300,
+    "n_chars": 1145876
+  },
+  "crystal_coder.cc100-fa": {
+    "vocab_size": 32022,
+    "n_bytes": 2054052,
+    "n_tokens": 1145076,
+    "n_chars": 1145876
+  },
+  "dbrx_instruct.cc100-fa": {
+    "vocab_size": 100280,
+    "n_bytes": 2054052,
+    "n_tokens": 818067,
+    "n_chars": 1145876
+  },
+  "deepseek_coder_33b_instruct.cc100-fa": {
+    "vocab_size": 32022,
+    "n_bytes": 2054052,
+    "n_tokens": 1326109,
+    "n_chars": 1145876
+  },
+  "deepseek_llm_7b_base.cc100-fa": {
+    "vocab_size": 100015,
+    "n_bytes": 2054052,
+    "n_tokens": 973451,
+    "n_chars": 1145876
+  },
+  "falcon_180b.cc100-fa": {
+    "vocab_size": 65024,
+    "n_bytes": 2054052,
+    "n_tokens": 1246580,
+    "n_chars": 1145876
+  },
+  "falcon_7b.cc100-fa": {
+    "vocab_size": 65024,
+    "n_bytes": 2054052,
+    "n_tokens": 1246580,
+    "n_chars": 1145876
+  },
+  "fastchat_t5_3b.cc100-fa": {
+    "vocab_size": 32110,
+    "n_bytes": 2054052,
+    "n_tokens": 712443,
+    "n_chars": 1145876
+  },
+  "flan_t5_base.cc100-fa": {
+    "vocab_size": 32100,
+    "n_bytes": 2054052,
+    "n_tokens": 493779,
+    "n_chars": 1145876
+  },
+  "gemma_7b.cc100-fa": {
+    "vocab_size": 256000,
+    "n_bytes": 2054052,
+    "n_tokens": 373762,
+    "n_chars": 1145876
+  },
+  "gpt2.cc100-fa": {
+    "vocab_size": 50257,
+    "n_bytes": 2054052,
+    "n_tokens": 1292300,
+    "n_chars": 1145876
+  },
+  "gpt2_chinese.cc100-fa": {
+    "vocab_size": 21128,
+    "n_bytes": 2054052,
+    "n_tokens": 406174,
+    "n_chars": 1145876
+  },
+  "gpt_35_turbo.cc100-fa": {
+    "vocab_size": 100277,
+    "n_bytes": 2054052,
+    "n_tokens": 818067,
+    "n_chars": 1145876
+  },
+  "gpt_4.cc100-fa": {
+    "vocab_size": 100277,
+    "n_bytes": 2054052,
+    "n_tokens": 818067,
+    "n_chars": 1145876
+  },
+  "gpt_neox_japanese_2_7b.cc100-fa": {
+    "vocab_size": 32000,
+    "n_bytes": 2054052,
+    "n_tokens": 2036715,
+    "n_chars": 1145876
+  },
+  "gpt_nexo_20b.cc100-fa": {
+    "vocab_size": 50277,
+    "n_bytes": 2054052,
+    "n_tokens": 866434,
+    "n_chars": 1145876
+  },
+  "grok_1.cc100-fa": {
+    "vocab_size": 131072,
+    "n_bytes": 2054052,
+    "n_tokens": 1073281,
+    "n_chars": 1145876
+  },
+  "internlm2_chat_7b.cc100-fa": {
+    "vocab_size": 92544,
+    "n_bytes": 2054052,
+    "n_tokens": 1195032,
+    "n_chars": 1145876
+  },
+  "internlm2_math_7b.cc100-fa": {
+    "vocab_size": 92544,
+    "n_bytes": 2054052,
+    "n_tokens": 1195032,
+    "n_chars": 1145876
+  },
+  "internlm_chat_7b.cc100-fa": {
+    "vocab_size": 103168,
+    "n_bytes": 2054052,
+    "n_tokens": 640945,
+    "n_chars": 1145876
+  },
+  "internlm_xcomposer_7b.cc100-fa": {
+    "vocab_size": 103168,
+    "n_bytes": 2054052,
+    "n_tokens": 640945,
+    "n_chars": 1145876
+  },
+  "jamba_v0_1.cc100-fa": {
+    "vocab_size": 65536,
+    "n_bytes": 2054052,
+    "n_tokens": 732550,
+    "n_chars": 1145876
+  },
+  "kplug.cc100-fa": {
+    "vocab_size": 10261,
+    "n_bytes": 2054052,
+    "n_tokens": 274671,
+    "n_chars": 1145876
+  },
+  "llama.cc100-fa": {
+    "vocab_size": 32000,
+    "n_bytes": 2054052,
+    "n_tokens": 1155076,
+    "n_chars": 1145876
+  },
+  "llama2.cc100-fa": {
+    "vocab_size": 32001,
+    "n_bytes": 2054052,
+    "n_tokens": 1155076,
+    "n_chars": 1145876
+  },
+  "llama3.cc100-fa": {
+    "vocab_size": 128256,
+    "n_bytes": 2054052,
+    "n_tokens": 387448,
+    "n_chars": 1145876
+  },
+  "llama_3_chinese_8b.cc100-fa": {
+    "vocab_size": 128256,
+    "n_bytes": 2054052,
+    "n_tokens": 397448,
+    "n_chars": 1145876
+  },
+  "mistral_7b.cc100-fa": {
+    "vocab_size": 32000,
+    "n_bytes": 2054052,
+    "n_tokens": 1133278,
+    "n_chars": 1145876
+  },
+  "mixtral_8_7b.cc100-fa": {
+    "vocab_size": 32000,
+    "n_bytes": 2054052,
+    "n_tokens": 1133278,
+    "n_chars": 1145876
+  },
+  "mobilebert_uncased.cc100-fa": {
+    "vocab_size": 30522,
+    "n_bytes": 2054052,
+    "n_tokens": 910783,
+    "n_chars": 1145876
+  },
+  "moss.cc100-fa": {
+    "vocab_size": 106072,
+    "n_bytes": 2054052,
+    "n_tokens": 1285426,
+    "n_chars": 1145876
+  },
+  "mt5_large.cc100-fa": {
+    "vocab_size": 250100,
+    "n_bytes": 2054052,
+    "n_tokens": 429922,
+    "n_chars": 1145876
+  },
+  "olmo_7b.cc100-fa": {
+    "vocab_size": 50280,
+    "n_bytes": 2054052,
+    "n_tokens": 866434,
+    "n_chars": 1145876
+  },
+  "orion_14b_chat.cc100-fa": {
+    "vocab_size": 84608,
+    "n_bytes": 2054052,
+    "n_tokens": 1131108,
+    "n_chars": 1145876
+  },
+  "phi_1.cc100-fa": {
+    "vocab_size": 50295,
+    "n_bytes": 2054052,
+    "n_tokens": 1292300,
+    "n_chars": 1145876
+  },
+  "phi_2.cc100-fa": {
+    "vocab_size": 50295,
+    "n_bytes": 2054052,
+    "n_tokens": 1292300,
+    "n_chars": 1145876
+  },
+  "phi_3_mini.cc100-fa": {
+    "vocab_size": 32011,
+    "n_bytes": 2054052,
+    "n_tokens": 1155076,
+    "n_chars": 1145876
+  },
+  "pko_t5_large.cc100-fa": {
+    "vocab_size": 50358,
+    "n_bytes": 2054052,
+    "n_tokens": 2061040,
+    "n_chars": 1145876
+  },
+  "prompt_clue.cc100-fa": {
+    "vocab_size": 32128,
+    "n_bytes": 2054052,
+    "n_tokens": 740377,
+    "n_chars": 1145876
+  },
+  "qwen1_5_14b_chat.cc100-fa": {
+    "vocab_size": 151646,
+    "n_bytes": 2054052,
+    "n_tokens": 643421,
+    "n_chars": 1145876
+  },
+  "qwen_1_8b_chat.cc100-fa": {
+    "vocab_size": 151851,
+    "n_bytes": 2054052,
+    "n_tokens": 643421,
+    "n_chars": 1145876
+  },
+  "qwen_72b_chat.cc100-fa": {
+    "vocab_size": 151851,
+    "n_bytes": 2054052,
+    "n_tokens": 643421,
+    "n_chars": 1145876
+  },
+  "qwen_7b_chat.cc100-fa": {
+    "vocab_size": 151851,
+    "n_bytes": 2054052,
+    "n_tokens": 643421,
+    "n_chars": 1145876
+  },
+  "roberta_chinese_clue.cc100-fa": {
+    "vocab_size": 8021,
+    "n_bytes": 2054052,
+    "n_tokens": 407763,
+    "n_chars": 1145876
+  },
+  "skywork_13b_base.cc100-fa": {
+    "vocab_size": 65519,
+    "n_bytes": 2054052,
+    "n_tokens": 1155072,
+    "n_chars": 1145876
+  },
+  "skywork_13b_math.cc100-fa": {
+    "vocab_size": 65519,
+    "n_bytes": 2054052,
+    "n_tokens": 1155072,
+    "n_chars": 1145876
+  },
+  "solar_10_7b.cc100-fa": {
+    "vocab_size": 32000,
+    "n_bytes": 2054052,
+    "n_tokens": 1133278,
+    "n_chars": 1145876
+  },
+  "starchat_alpha.cc100-fa": {
+    "vocab_size": 49156,
+    "n_bytes": 2054052,
+    "n_tokens": 851630,
+    "n_chars": 1145876
+  },
+  "switch_c_2048.cc100-fa": {
+    "vocab_size": 32100,
+    "n_bytes": 2054052,
+    "n_tokens": 493767,
+    "n_chars": 1145876
+  },
+  "t5_base.cc100-fa": {
+    "vocab_size": 32100,
+    "n_bytes": 2054052,
+    "n_tokens": 493767,
+    "n_chars": 1145876
+  },
+  "t5_large.cc100-fa": {
+    "vocab_size": 32100,
+    "n_bytes": 2054052,
+    "n_tokens": 493767,
+    "n_chars": 1145876
+  },
+  "t5_small.cc100-fa": {
+    "vocab_size": 32100,
+    "n_bytes": 2054052,
+    "n_tokens": 493767,
+    "n_chars": 1145876
+  },
+  "text_davinci_003.cc100-fa": {
+    "vocab_size": 50281,
+    "n_bytes": 2054052,
+    "n_tokens": 1292300,
+    "n_chars": 1145876
+  },
+  "tigerbot_13b_chat_v2.cc100-fa": {
+    "vocab_size": 60515,
+    "n_bytes": 2054052,
+    "n_tokens": 1145046,
+    "n_chars": 1145876
+  },
+  "tigerbot_70b_chat_v4_4k.cc100-fa": {
+    "vocab_size": 65110,
+    "n_bytes": 2054052,
+    "n_tokens": 1145048,
+    "n_chars": 1145876
+  },
+  "wizardcoder_15b_v1.cc100-fa": {
+    "vocab_size": 49153,
+    "n_bytes": 2054052,
+    "n_tokens": 851630,
+    "n_chars": 1145876
+  },
+  "wizardcoder_python_7b_v1.cc100-fa": {
+    "vocab_size": 32001,
+    "n_bytes": 2054052,
+    "n_tokens": 1155076,
+    "n_chars": 1145876
+  },
+  "wizardlm_7b_v1.cc100-fa": {
+    "vocab_size": 32001,
+    "n_bytes": 2054052,
+    "n_tokens": 1155076,
+    "n_chars": 1145876
+  },
+  "wizardmath_70b_v1.cc100-fa": {
+    "vocab_size": 32002,
+    "n_bytes": 2054052,
+    "n_tokens": 1155076,
+    "n_chars": 1145876
+  },
+  "xlm_roberta.cc100-fa": {
+    "vocab_size": 250002,
+    "n_bytes": 2054052,
+    "n_tokens": 330926,
+    "n_chars": 1145876
+  },
+  "yi_34b.cc100-fa": {
+    "vocab_size": 64000,
+    "n_bytes": 2054052,
+    "n_tokens": 1337264,
+    "n_chars": 1145876
+  },
+  "yi_6b.cc100-fa": {
+    "vocab_size": 64000,
+    "n_bytes": 2054052,
+    "n_tokens": 1337264,
+    "n_chars": 1145876
+  },
+  "yi_vl34b.cc100-fa": {
+    "vocab_size": 64000,
+    "n_bytes": 2054052,
+    "n_tokens": 1346819,
+    "n_chars": 1145876
+  },
+  "zephyr_7b_beta.cc100-fa": {
+    "vocab_size": 32000,
+    "n_bytes": 2054052,
+    "n_tokens": 1133278,
+    "n_chars": 1145876
+  },
+  "llama_3_chinese_8b.cc100-fr": {
+    "vocab_size": 128256,
+    "n_bytes": 1540504,
+    "n_tokens": 422146,
+    "n_chars": 1484970
+  },
+  "llama_3_chinese_8b.cc100-ja": {
+    "vocab_size": 128256,
+    "n_bytes": 1774770,
+    "n_tokens": 424715,
+    "n_chars": 603065
+  },
+  "aya_101.cc100-ko": {
+    "vocab_size": 250100,
+    "n_bytes": 1524839,
+    "n_tokens": 434586,
+    "n_chars": 655190
+  },
+  "baichuan.cc100-ko": {
+    "vocab_size": 64000,
+    "n_bytes": 1524839,
+    "n_tokens": 639258,
+    "n_chars": 655190
+  },
+  "baichuan2.cc100-ko": {
+    "vocab_size": 125696,
+    "n_bytes": 1524839,
+    "n_tokens": 623358,
+    "n_chars": 655190
+  },
+  "bert_base_cased.cc100-ko": {
+    "vocab_size": 28996,
+    "n_bytes": 1524839,
+    "n_tokens": 222828,
+    "n_chars": 655190
+  },
+  "bert_base_chinese.cc100-ko": {
+    "vocab_size": 21128,
+    "n_bytes": 1524839,
+    "n_tokens": 219752,
+    "n_chars": 655190
+  },
+  "bert_base_uncased.cc100-ko": {
+    "vocab_size": 30522,
+    "n_bytes": 1524839,
+    "n_tokens": 904756,
+    "n_chars": 655190
+  },
+  "bloom.cc100-ko": {
+    "vocab_size": 250680,
+    "n_bytes": 1524839,
+    "n_tokens": 742111,
+    "n_chars": 655190
+  },
+  "byt5_small.cc100-ko": {
+    "vocab_size": 384,
+    "n_bytes": 1524839,
+    "n_tokens": 1534839,
+    "n_chars": 655190
+  },
+  "character_glm_6b.cc100-ko": {
+    "vocab_size": 64789,
+    "n_bytes": 1524839,
+    "n_tokens": 672160,
+    "n_chars": 655190
+  },
+  "chatglm2_6b.cc100-ko": {
+    "vocab_size": 64787,
+    "n_bytes": 1524839,
+    "n_tokens": 672156,
+    "n_chars": 655190
+  },
+  "chatglm3_6b.cc100-ko": {
+    "vocab_size": 64796,
+    "n_bytes": 1524839,
+    "n_tokens": 672160,
+    "n_chars": 655190
+  },
+  "chatglm_6b.cc100-ko": {
+    "vocab_size": 150344,
+    "n_bytes": 1524839,
+    "n_tokens": 939630,
+    "n_chars": 655190
+  },
+  "chatyuan_large_v2.cc100-ko": {
+    "vocab_size": 32128,
+    "n_bytes": 1524839,
+    "n_tokens": 354411,
+    "n_chars": 655190
+  },
+  "chinese_llama.cc100-ko": {
+    "vocab_size": 49953,
+    "n_bytes": 1524839,
+    "n_tokens": 913553,
+    "n_chars": 655190
+  },
+  "chinese_llama2.cc100-ko": {
+    "vocab_size": 55296,
+    "n_bytes": 1524839,
+    "n_tokens": 963427,
+    "n_chars": 655190
+  },
+  "code_davinci_002.cc100-ko": {
+    "vocab_size": 50281,
+    "n_bytes": 1524839,
+    "n_tokens": 1308993,
+    "n_chars": 655190
+  },
+  "crystal_coder.cc100-ko": {
+    "vocab_size": 32022,
+    "n_bytes": 1524839,
+    "n_tokens": 954428,
+    "n_chars": 655190
+  },
+  "dbrx_instruct.cc100-ko": {
+    "vocab_size": 100280,
+    "n_bytes": 1524839,
+    "n_tokens": 652277,
+    "n_chars": 655190
+  },
+  "deepseek_coder_33b_instruct.cc100-ko": {
+    "vocab_size": 32022,
+    "n_bytes": 1524839,
+    "n_tokens": 1454805,
+    "n_chars": 655190
+  },
+  "deepseek_llm_7b_base.cc100-ko": {
+    "vocab_size": 100015,
+    "n_bytes": 1524839,
+    "n_tokens": 1081983,
+    "n_chars": 655190
+  },
+  "falcon_180b.cc100-ko": {
+    "vocab_size": 65024,
+    "n_bytes": 1524839,
+    "n_tokens": 1330568,
+    "n_chars": 655190
+  },
+  "falcon_7b.cc100-ko": {
+    "vocab_size": 65024,
+    "n_bytes": 1524839,
+    "n_tokens": 1330568,
+    "n_chars": 655190
+  },
+  "fastchat_t5_3b.cc100-ko": {
+    "vocab_size": 32110,
+    "n_bytes": 1524839,
+    "n_tokens": 484953,
+    "n_chars": 655190
+  },
+  "flan_t5_base.cc100-ko": {
+    "vocab_size": 32100,
+    "n_bytes": 1524839,
+    "n_tokens": 344457,
+    "n_chars": 655190
+  },
+  "gemma_7b.cc100-ko": {
+    "vocab_size": 256000,
+    "n_bytes": 1524839,
+    "n_tokens": 464410,
+    "n_chars": 655190
+  },
+  "gpt2.cc100-ko": {
+    "vocab_size": 50257,
+    "n_bytes": 1524839,
+    "n_tokens": 1309029,
+    "n_chars": 655190
+  },
+  "gpt2_chinese.cc100-ko": {
+    "vocab_size": 21128,
+    "n_bytes": 1524839,
+    "n_tokens": 1055974,
+    "n_chars": 655190
+  },
+  "gpt_35_turbo.cc100-ko": {
+    "vocab_size": 100277,
+    "n_bytes": 1524839,
+    "n_tokens": 652277,
+    "n_chars": 655190
+  },
+  "gpt_4.cc100-ko": {
+    "vocab_size": 100277,
+    "n_bytes": 1524839,
+    "n_tokens": 652277,
+    "n_chars": 655190
+  },
+  "gpt_neox_japanese_2_7b.cc100-ko": {
+    "vocab_size": 32000,
+    "n_bytes": 1524839,
+    "n_tokens": 1512832,
+    "n_chars": 655190
+  },
+  "gpt_nexo_20b.cc100-ko": {
+    "vocab_size": 50277,
+    "n_bytes": 1524839,
+    "n_tokens": 973288,
+    "n_chars": 655190
+  },
+  "grok_1.cc100-ko": {
+    "vocab_size": 131072,
+    "n_bytes": 1524839,
+    "n_tokens": 1152005,
+    "n_chars": 655190
+  },
+  "internlm2_chat_7b.cc100-ko": {
+    "vocab_size": 92544,
+    "n_bytes": 1524839,
+    "n_tokens": 1008524,
+    "n_chars": 655190
+  },
+  "internlm2_math_7b.cc100-ko": {
+    "vocab_size": 92544,
+    "n_bytes": 1524839,
+    "n_tokens": 1008524,
+    "n_chars": 655190
+  },
+  "internlm_chat_7b.cc100-ko": {
+    "vocab_size": 103168,
+    "n_bytes": 1524839,
+    "n_tokens": 839609,
+    "n_chars": 655190
+  },
+  "internlm_xcomposer_7b.cc100-ko": {
+    "vocab_size": 103168,
+    "n_bytes": 1524839,
+    "n_tokens": 839609,
+    "n_chars": 655190
+  },
+  "jamba_v0_1.cc100-ko": {
+    "vocab_size": 65536,
+    "n_bytes": 1524839,
+    "n_tokens": 715688,
+    "n_chars": 655190
+  },
+  "kplug.cc100-ko": {
+    "vocab_size": 10261,
+    "n_bytes": 1524839,
+    "n_tokens": 222771,
+    "n_chars": 655190
+  },
+  "llama.cc100-ko": {
+    "vocab_size": 32000,
+    "n_bytes": 1524839,
+    "n_tokens": 964428,
+    "n_chars": 655190
+  },
+  "llama2.cc100-ko": {
+    "vocab_size": 32001,
+    "n_bytes": 1524839,
+    "n_tokens": 964428,
+    "n_chars": 655190
+  },
+  "llama3.cc100-ko": {
+    "vocab_size": 128256,
+    "n_bytes": 1524839,
+    "n_tokens": 412595,
+    "n_chars": 655190
+  },
+  "llama_3_chinese_8b.cc100-ko": {
+    "vocab_size": 128256,
+    "n_bytes": 1524839,
+    "n_tokens": 422595,
+    "n_chars": 655190
+  },
+  "mistral_7b.cc100-ko": {
+    "vocab_size": 32000,
+    "n_bytes": 1524839,
+    "n_tokens": 728766,
+    "n_chars": 655190
+  },
+  "mixtral_8_7b.cc100-ko": {
+    "vocab_size": 32000,
+    "n_bytes": 1524839,
+    "n_tokens": 728766,
+    "n_chars": 655190
+  },
+  "mobilebert_uncased.cc100-ko": {
+    "vocab_size": 30522,
+    "n_bytes": 1524839,
+    "n_tokens": 904756,
+    "n_chars": 655190
+  },
+  "moss.cc100-ko": {
+    "vocab_size": 106072,
+    "n_bytes": 1524839,
+    "n_tokens": 1305249,
+    "n_chars": 655190
+  },
+  "mt5_large.cc100-ko": {
+    "vocab_size": 250100,
+    "n_bytes": 1524839,
+    "n_tokens": 434586,
+    "n_chars": 655190
+  },
+  "olmo_7b.cc100-ko": {
+    "vocab_size": 50280,
+    "n_bytes": 1524839,
+    "n_tokens": 973288,
+    "n_chars": 655190
+  },
+  "orion_14b_chat.cc100-ko": {
+    "vocab_size": 84608,
+    "n_bytes": 1524839,
+    "n_tokens": 351149,
+    "n_chars": 655190
+  },
+  "phi_1.cc100-ko": {
+    "vocab_size": 50295,
+    "n_bytes": 1524839,
+    "n_tokens": 1308988,
+    "n_chars": 655190
+  },
+  "phi_2.cc100-ko": {
+    "vocab_size": 50295,
+    "n_bytes": 1524839,
+    "n_tokens": 1308988,
+    "n_chars": 655190
+  },
+  "phi_3_mini.cc100-ko": {
+    "vocab_size": 32011,
+    "n_bytes": 1524839,
+    "n_tokens": 964428,
+    "n_chars": 655190
+  },
+  "pko_t5_large.cc100-ko": {
+    "vocab_size": 50358,
+    "n_bytes": 1524839,
+    "n_tokens": 471643,
+    "n_chars": 655190
+  },
+  "prompt_clue.cc100-ko": {
+    "vocab_size": 32128,
+    "n_bytes": 1524839,
+    "n_tokens": 354411,
+    "n_chars": 655190
+  },
+  "qwen1_5_14b_chat.cc100-ko": {
+    "vocab_size": 151646,
+    "n_bytes": 1524839,
+    "n_tokens": 457492,
+    "n_chars": 655190
+  },
+  "qwen_1_8b_chat.cc100-ko": {
+    "vocab_size": 151851,
+    "n_bytes": 1524839,
+    "n_tokens": 457492,
+    "n_chars": 655190
+  },
+  "qwen_72b_chat.cc100-ko": {
+    "vocab_size": 151851,
+    "n_bytes": 1524839,
+    "n_tokens": 457492,
+    "n_chars": 655190
+  },
+  "qwen_7b_chat.cc100-ko": {
+    "vocab_size": 151851,
+    "n_bytes": 1524839,
+    "n_tokens": 457492,
+    "n_chars": 655190
+  },
+  "roberta_chinese_clue.cc100-ko": {
+    "vocab_size": 8021,
+    "n_bytes": 1524839,
+    "n_tokens": 226812,
+    "n_chars": 655190
+  },
+  "skywork_13b_base.cc100-ko": {
+    "vocab_size": 65519,
+    "n_bytes": 1524839,
+    "n_tokens": 962744,
+    "n_chars": 655190
+  },
+  "skywork_13b_math.cc100-ko": {
+    "vocab_size": 65519,
+    "n_bytes": 1524839,
+    "n_tokens": 962744,
+    "n_chars": 655190
+  },
+  "solar_10_7b.cc100-ko": {
+    "vocab_size": 32000,
+    "n_bytes": 1524839,
+    "n_tokens": 728766,
+    "n_chars": 655190
+  },
+  "starchat_alpha.cc100-ko": {
+    "vocab_size": 49156,
+    "n_bytes": 1524839,
+    "n_tokens": 580873,
+    "n_chars": 655190
+  },
+  "switch_c_2048.cc100-ko": {
+    "vocab_size": 32100,
+    "n_bytes": 1524839,
+    "n_tokens": 344457,
+    "n_chars": 655190
+  },
+  "t5_base.cc100-ko": {
+    "vocab_size": 32100,
+    "n_bytes": 1524839,
+    "n_tokens": 344457,
+    "n_chars": 655190
+  },
+  "t5_large.cc100-ko": {
+    "vocab_size": 32100,
+    "n_bytes": 1524839,
+    "n_tokens": 344457,
+    "n_chars": 655190
+  },
+  "t5_small.cc100-ko": {
+    "vocab_size": 32100,
+    "n_bytes": 1524839,
+    "n_tokens": 344457,
+    "n_chars": 655190
+  },
+  "text_davinci_003.cc100-ko": {
+    "vocab_size": 50281,
+    "n_bytes": 1524839,
+    "n_tokens": 1308993,
+    "n_chars": 655190
+  },
+  "tigerbot_13b_chat_v2.cc100-ko": {
+    "vocab_size": 60515,
+    "n_bytes": 1524839,
+    "n_tokens": 793053,
+    "n_chars": 655190
+  },
+  "tigerbot_70b_chat_v4_4k.cc100-ko": {
+    "vocab_size": 65110,
+    "n_bytes": 1524839,
+    "n_tokens": 484082,
+    "n_chars": 655190
+  },
+  "wizardcoder_15b_v1.cc100-ko": {
+    "vocab_size": 49153,
+    "n_bytes": 1524839,
+    "n_tokens": 580873,
+    "n_chars": 655190
+  },
+  "wizardcoder_python_7b_v1.cc100-ko": {
+    "vocab_size": 32001,
+    "n_bytes": 1524839,
+    "n_tokens": 964428,
+    "n_chars": 655190
+  },
+  "wizardlm_7b_v1.cc100-ko": {
+    "vocab_size": 32001,
+    "n_bytes": 1524839,
+    "n_tokens": 964428,
+    "n_chars": 655190
+  },
+  "wizardmath_70b_v1.cc100-ko": {
+    "vocab_size": 32002,
+    "n_bytes": 1524839,
+    "n_tokens": 964428,
+    "n_chars": 655190
+  },
+  "xlm_roberta.cc100-ko": {
+    "vocab_size": 250002,
+    "n_bytes": 1524839,
+    "n_tokens": 374571,
+    "n_chars": 655190
+  },
+  "yi_34b.cc100-ko": {
+    "vocab_size": 64000,
+    "n_bytes": 1524839,
+    "n_tokens": 1203134,
+    "n_chars": 655190
+  },
+  "yi_6b.cc100-ko": {
+    "vocab_size": 64000,
+    "n_bytes": 1524839,
+    "n_tokens": 1203134,
+    "n_chars": 655190
+  },
+  "yi_vl34b.cc100-ko": {
+    "vocab_size": 64000,
+    "n_bytes": 1524839,
+    "n_tokens": 1210021,
+    "n_chars": 655190
+  },
+  "zephyr_7b_beta.cc100-ko": {
+    "vocab_size": 32000,
+    "n_bytes": 1524839,
+    "n_tokens": 728766,
+    "n_chars": 655190
+  },
+  "llama_3_chinese_8b.cc100-zh-Hans": {
+    "vocab_size": 128256,
+    "n_bytes": 2633047,
+    "n_tokens": 757405,
+    "n_chars": 927311
   }
 }

utils/compression_util.py

@@ -20,7 +20,8 @@ from typing import List, Optional, Union, Literal
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 
 common_units = ["g_bytes/b_tokens", "b_tokens/g_bytes", "t_bytes/t_tokens", "t_tokens/t_bytes", "n_chars/n_tokens", ]
-common_corpuses = ["cc100-en", "cc100-zh-Hans", "cc100-es", "cc100-fr", "cc100-de", "cc100-ko" "cc100-fa", "cc100-ar"]
+common_corpuses = sorted(["cc100-en", "cc100-zh-Hans", "cc100-es", "cc100-fr", "cc100-de", "cc100-ko",
+                          "cc100-fa", "cc100-ar", "cc100-ja"])
 
 VALID_CODES_CC100 = [
     "am", "ar", "as", "az", "be", "bg", "bn", "bn_rom", "br", "bs", "ca", "cs", "cy", "da", "de",
@@ -198,10 +199,10 @@ def test():
 
 def main():
     if len(sys.argv) == 3:
-        tokenizers = [sys.argv[1]]
+        tokenizer_filter = [sys.argv[1]]
         corpuses = [sys.argv[2]]
     else:
-        tokenizers = all_tokenizers[:2]
+        tokenizer_filter = None
         corpuses = common_corpuses
     df = get_compression_leaderboard(corpuses)
     # print(df.to_markdown(index=False, tablefmt='fancy_grid'))

utils/lang_util_2.py

@@ -53,30 +53,6 @@ def is_all_en(text):
 
 
 
-# import opencc
-
-def is_russian():
-    """ 俄语 """
-    pass
-
-def is_french():
-    """ 法语 """
-
-def aa():
-    """
-    zh-Hans: Chinese (Simplified)
-    :return:
-    """
-    pass
-
-
-def bb():
-    """
-    zh-Hant: Chinese (Traditional)
-    :return:
-    """
-
-
 ranges = [
     {"from": ord(u"\u3300"), "to": ord(u"\u33ff")},  # compatibility ideographs
     {"from": ord(u"\ufe30"), "to": ord(u"\ufe4f")},  # compatibility ideographs

vocab/__init__.py

@@ -107,6 +107,7 @@ all_tokenizers = [
     ("llama3", "", "sentencepiece"),
     ("chinese_llama", "", "sentencepiece"),  #
     ("chinese_llama2", "", "sentencepiece"),  #
+    ("llama_3_chinese_8b", "sentencepiece"),
     # ("chinese_alpaca_lora_7b",  # 中文Alpaca模型在上述中文LLaMA模型的基础上进一步使用了指令数据进行精调。
     # ("belle_llama_ext_7b",
     # ("alpaca_7b",
@@ -179,7 +180,7 @@ all_tokenizers = [
     ("grok_1",),
     # ("claude",),
     ("gpt_nexo_20b", ),
-    ("gpt_neox_japanese_2.7b", ),
+    ("gpt_neox_japanese_2_7b", ),
 
 ]
 

vocab/gpt_neox_japanese_2_7b/README.md

@@ -0,0 +1,64 @@
+
+
+## vocab.txt
+
+```
+るのは
+よね
+写真,寫真,冩真,写眞,寫眞,冩眞
+マイ
+そん
+女性,𠨰性,⼥性,女𧢱,𠨰𧢱,⼥𧢱
+内容,內容,内㣑,内㝐,内彮,内𠕺,內㣑,內㝐,內彮,內𠕺
+```
+
+怎么还有不同写法？？
+
+
+
+
+## 文本归一化
+
+以下的normalization，在生成任务中并不好。
+
+```
+        self.content_repatter1 = re.compile(r"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)")
+        self.content_repatter2 = re.compile(r"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*")
+        self.content_repatter3 = re.compile(r"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}")
+        self.content_repatter4 = re.compile(
+            r"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"
+        )
+        self.content_repatter5 = re.compile(
+            r"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"
+        )
+        self.content_repatter6 = re.compile(
+            r"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"
+        )
+
+    def clean_text(self, content):
+        content = self.content_repatter1.sub("<URL>", content)
+        content = self.content_repatter2.sub("<EMAIL>", content)
+        content = self.content_repatter3.sub("<TEL>", content)
+        content = self.content_repatter4.sub("<DATE>", content)
+        content = self.content_repatter5.sub("<DATE>", content)
+        content = self.content_repatter6.sub("<PRICE>", content)
+        content = content.translate(self.content_trans1)
+        while "<BLOCK><BLOCK>" in content:
+            content = content.replace("<BLOCK><BLOCK>", "<BLOCK>")
+        return content
+
+    def tokenize(self, text, clean=False):
+        text = text.replace(" ", "<SP>")
+        text = text.replace("　", "<SP>")
+        text = text.replace("\r\n", "<BR>")
+        text = text.replace("\n", "<BR>")
+        text = text.replace("\r", "<BR>")
+        text = text.replace("\t", "<TAB>")
+        text = text.replace("—", "ー")
+        text = text.replace("−", "ー")
+        for k, v in self.emoji["emoji"].items():
+            if k in text:
+                text = text.replace(k, v)
+        if clean:
+            text = self.clean_text(text)
+```

vocab/{gpt_neox_japanese_2.7b → gpt_neox_japanese_2_7b}/__init__.py

@@ -1,3 +1,7 @@
+"""
+这里有个 emoji.json 是干嘛的？
+"""
+
 from transformers import AutoTokenizer
 
 tokenizer = AutoTokenizer.from_pretrained("abeja/gpt-neox-japanese-2.7b")

vocab/gpt_neox_japanese_2_7b/test.py

@@ -0,0 +1,19 @@
+
+
+from transformers import AutoTokenizer, GPTNeoXJapaneseTokenizer
+
+tokenizer = GPTNeoXJapaneseTokenizer.from_pretrained("tokenizer")
+
+# tokenizer = AutoTokenizer.from_pretrained("abeja/gpt-neox-japanese-2.7b")
+
+tokens = tokenizer.encode("人とAIが協調するためには http://baidu.com 🤣")
+
+for token in tokens:
+    print(token, tokenizer.decode([token]))
+
+
+tokens = tokenizer.tokenize("人とAIが協調するためには http://baidu.com 🤣", clean=True)
+print(tokens)
+# for token in tokens:
+#     print(token, tokenizer.decode([token]))
+

vocab/gpt_neox_japanese_2_7b/test_emoji.py

@@ -0,0 +1,12 @@
+
+"""
+
+## 疑问
+- \u200d  是啥？
+- 对emoji 划分12个类
+"""
+
+import json
+emoji = json.load(open("tokenizer/emoji.json", "r", encoding="utf-8"))
+
+print(emoji)

vocab/llama_3_chinese_8b/__init__.py

@@ -0,0 +1,5 @@
+
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("hfl/llama-3-chinese-8b", trust_remote_code=True)