Shuu12121 commited on
Commit
cf85670
·
verified ·
1 Parent(s): d40278b

Upload ModernBERT model

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +4 -1
tokenizer_config.json CHANGED
@@ -66,8 +66,11 @@
66
  "extra_special_tokens": {},
67
  "mask_token": "[MASK]",
68
  "max_length": 256,
69
- "model_max_length": 1000000000000000019884624838656,
 
70
  "pad_token": "[PAD]",
 
 
71
  "sep_token": "[SEP]",
72
  "stride": 0,
73
  "tokenizer_class": "RobertaTokenizer",
 
66
  "extra_special_tokens": {},
67
  "mask_token": "[MASK]",
68
  "max_length": 256,
69
+ "model_max_length": 1024,
70
+ "pad_to_multiple_of": null,
71
  "pad_token": "[PAD]",
72
+ "pad_token_type_id": 0,
73
+ "padding_side": "right",
74
  "sep_token": "[SEP]",
75
  "stride": 0,
76
  "tokenizer_class": "RobertaTokenizer",