ghost-x
/

ghost-7b-v0.9.0

Text Generation

text-generation-inference

Model card Files Files and versions

lamhieu commited on Jan 29, 2024

Commit

da5de4f

·

verified ·

1 Parent(s): fcdae7a

Update tokenizer configuration

Files changed (1) hide show

tokenizer_config.json +1 -2

tokenizer_config.json CHANGED Viewed

@@ -35,7 +35,7 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": true,
-  "max_length": 2048,
   "model_max_length": 1000000000000000019884624838656,
   "pad_to_multiple_of": null,
   "pad_token": "</s>",
@@ -46,7 +46,6 @@
   "stride": 0,
   "tokenizer_class": "LlamaTokenizer",
   "truncation_side": "left",
-  "truncation_strategy": "longest_first",
   "unk_token": "<unk>",
   "use_default_system_prompt": true
 }

   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": true,
+  "max_length": 8192,
   "model_max_length": 1000000000000000019884624838656,
   "pad_to_multiple_of": null,
   "pad_token": "</s>",
   "stride": 0,
   "tokenizer_class": "LlamaTokenizer",
   "truncation_side": "left",
   "unk_token": "<unk>",
   "use_default_system_prompt": true
 }