test tokenizer config
Browse files- configuration_gemma3mm.py +6 -2
- tokenizer_config.json +1 -1
configuration_gemma3mm.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
from typing import Optional
|
| 2 |
|
| 3 |
-
from transformers import Gemma3TextConfig
|
| 4 |
from transformers.configuration_utils import PretrainedConfig
|
| 5 |
from transformers.modeling_rope_utils import rope_config_validation
|
| 6 |
from transformers.utils import logging
|
|
@@ -203,4 +203,8 @@ class Gemma3MMConfig(PretrainedConfig):
|
|
| 203 |
self.audio_token_index = audio_token_index
|
| 204 |
self.initializer_range = initializer_range
|
| 205 |
|
| 206 |
-
super().__init__(**kwargs)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
from typing import Optional
|
| 2 |
|
| 3 |
+
from transformers import AutoConfig, Gemma3TextConfig
|
| 4 |
from transformers.configuration_utils import PretrainedConfig
|
| 5 |
from transformers.modeling_rope_utils import rope_config_validation
|
| 6 |
from transformers.utils import logging
|
|
|
|
| 203 |
self.audio_token_index = audio_token_index
|
| 204 |
self.initializer_range = initializer_range
|
| 205 |
|
| 206 |
+
super().__init__(**kwargs)
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
AutoConfig.register("gemma3mm", Gemma3MMConfig)
|
| 210 |
+
Gemma3MMConfig.register_for_auto_class()
|
tokenizer_config.json
CHANGED
|
@@ -51343,7 +51343,7 @@
|
|
| 51343 |
"image_token": "<image_soft_token>",
|
| 51344 |
"model_max_length": 1000000000000000019884624838656,
|
| 51345 |
"pad_token": "<pad>",
|
| 51346 |
-
"processor_class": "
|
| 51347 |
"sp_model_kwargs": null,
|
| 51348 |
"spaces_between_special_tokens": false,
|
| 51349 |
"tokenizer_class": "GemmaTokenizerFast",
|
|
|
|
| 51343 |
"image_token": "<image_soft_token>",
|
| 51344 |
"model_max_length": 1000000000000000019884624838656,
|
| 51345 |
"pad_token": "<pad>",
|
| 51346 |
+
"processor_class": "Gemma3MMProcessor",
|
| 51347 |
"sp_model_kwargs": null,
|
| 51348 |
"spaces_between_special_tokens": false,
|
| 51349 |
"tokenizer_class": "GemmaTokenizerFast",
|