Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes.  
							See raw diff
- config.json +70 -0
- configuration_deepseek.py +210 -0
- model-00001-of-000163.safetensors +3 -0
- model-00002-of-000163.safetensors +3 -0
- model-00003-of-000163.safetensors +3 -0
- model-00004-of-000163.safetensors +3 -0
- model-00005-of-000163.safetensors +3 -0
- model-00006-of-000163.safetensors +3 -0
- model-00007-of-000163.safetensors +3 -0
- model-00008-of-000163.safetensors +3 -0
- model-00009-of-000163.safetensors +3 -0
- model-00010-of-000163.safetensors +3 -0
- model-00011-of-000163.safetensors +3 -0
- model-00012-of-000163.safetensors +3 -0
- model-00013-of-000163.safetensors +3 -0
- model-00014-of-000163.safetensors +3 -0
- model-00015-of-000163.safetensors +3 -0
- model-00016-of-000163.safetensors +3 -0
- model-00017-of-000163.safetensors +3 -0
- model-00018-of-000163.safetensors +3 -0
- model-00019-of-000163.safetensors +3 -0
- model-00020-of-000163.safetensors +3 -0
- model-00021-of-000163.safetensors +3 -0
- model-00022-of-000163.safetensors +3 -0
- model-00023-of-000163.safetensors +3 -0
- model-00024-of-000163.safetensors +3 -0
- model-00025-of-000163.safetensors +3 -0
- model-00026-of-000163.safetensors +3 -0
- model-00027-of-000163.safetensors +3 -0
- model-00028-of-000163.safetensors +3 -0
- model-00029-of-000163.safetensors +3 -0
- model-00030-of-000163.safetensors +3 -0
- model-00031-of-000163.safetensors +3 -0
- model-00032-of-000163.safetensors +3 -0
- model-00033-of-000163.safetensors +3 -0
- model-00034-of-000163.safetensors +3 -0
- model-00035-of-000163.safetensors +3 -0
- model-00036-of-000163.safetensors +3 -0
- model-00037-of-000163.safetensors +3 -0
- model-00038-of-000163.safetensors +3 -0
- model-00039-of-000163.safetensors +3 -0
- model-00040-of-000163.safetensors +3 -0
- model-00041-of-000163.safetensors +3 -0
- model-00042-of-000163.safetensors +3 -0
- model-00043-of-000163.safetensors +3 -0
- model-00044-of-000163.safetensors +3 -0
- model-00045-of-000163.safetensors +3 -0
- model-00046-of-000163.safetensors +3 -0
- model-00047-of-000163.safetensors +3 -0
- model-00048-of-000163.safetensors +3 -0
    	
        config.json
    ADDED
    
    | @@ -0,0 +1,70 @@ | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            {
         | 
| 2 | 
            +
              "architectures": [
         | 
| 3 | 
            +
                "DeepseekV3ForCausalLM"
         | 
| 4 | 
            +
              ],
         | 
| 5 | 
            +
              "attention_bias": false,
         | 
| 6 | 
            +
              "attention_dropout": 0.0,
         | 
| 7 | 
            +
              "auto_map": {
         | 
| 8 | 
            +
                "AutoConfig": "configuration_deepseek.DeepseekV3Config",
         | 
| 9 | 
            +
                "AutoModel": "modeling_deepseek.DeepseekV3Model",
         | 
| 10 | 
            +
                "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"
         | 
| 11 | 
            +
              },
         | 
| 12 | 
            +
              "aux_loss_alpha": 0.001,
         | 
| 13 | 
            +
              "bos_token_id": 0,
         | 
| 14 | 
            +
              "eos_token_id": 1,
         | 
| 15 | 
            +
              "ep_size": 1,
         | 
| 16 | 
            +
              "first_k_dense_replace": 3,
         | 
| 17 | 
            +
              "hidden_act": "silu",
         | 
| 18 | 
            +
              "hidden_size": 7168,
         | 
| 19 | 
            +
              "initializer_range": 0.02,
         | 
| 20 | 
            +
              "intermediate_size": 18432,
         | 
| 21 | 
            +
              "kv_lora_rank": 512,
         | 
| 22 | 
            +
              "max_position_embeddings": 163840,
         | 
| 23 | 
            +
              "model_type": "deepseek_v3",
         | 
| 24 | 
            +
              "moe_intermediate_size": 2048,
         | 
| 25 | 
            +
              "moe_layer_freq": 1,
         | 
| 26 | 
            +
              "n_group": 8,
         | 
| 27 | 
            +
              "n_routed_experts": 256,
         | 
| 28 | 
            +
              "n_shared_experts": 1,
         | 
| 29 | 
            +
              "norm_topk_prob": true,
         | 
| 30 | 
            +
              "num_attention_heads": 128,
         | 
| 31 | 
            +
              "num_experts_per_tok": 8,
         | 
| 32 | 
            +
              "num_hidden_layers": 61,
         | 
| 33 | 
            +
              "num_key_value_heads": 128,
         | 
| 34 | 
            +
              "num_nextn_predict_layers": 1,
         | 
| 35 | 
            +
              "pretraining_tp": 1,
         | 
| 36 | 
            +
              "q_lora_rank": 1536,
         | 
| 37 | 
            +
              "qk_nope_head_dim": 128,
         | 
| 38 | 
            +
              "qk_rope_head_dim": 64,
         | 
| 39 | 
            +
              "quantization_config": {
         | 
| 40 | 
            +
                "activation_scheme": "dynamic",
         | 
| 41 | 
            +
                "fmt": "e4m3",
         | 
| 42 | 
            +
                "quant_method": "fp8",
         | 
| 43 | 
            +
                "weight_block_size": [
         | 
| 44 | 
            +
                  128,
         | 
| 45 | 
            +
                  128
         | 
| 46 | 
            +
                ]
         | 
| 47 | 
            +
              },
         | 
| 48 | 
            +
              "rms_norm_eps": 1e-06,
         | 
| 49 | 
            +
              "rope_scaling": {
         | 
| 50 | 
            +
                "beta_fast": 32,
         | 
| 51 | 
            +
                "beta_slow": 1,
         | 
| 52 | 
            +
                "factor": 40,
         | 
| 53 | 
            +
                "mscale": 1.0,
         | 
| 54 | 
            +
                "mscale_all_dim": 1.0,
         | 
| 55 | 
            +
                "original_max_position_embeddings": 4096,
         | 
| 56 | 
            +
                "type": "yarn"
         | 
| 57 | 
            +
              },
         | 
| 58 | 
            +
              "rope_theta": 10000,
         | 
| 59 | 
            +
              "routed_scaling_factor": 2.5,
         | 
| 60 | 
            +
              "scoring_func": "sigmoid",
         | 
| 61 | 
            +
              "seq_aux": true,
         | 
| 62 | 
            +
              "tie_word_embeddings": false,
         | 
| 63 | 
            +
              "topk_group": 4,
         | 
| 64 | 
            +
              "topk_method": "noaux_tc",
         | 
| 65 | 
            +
              "torch_dtype": "bfloat16",
         | 
| 66 | 
            +
              "transformers_version": "4.46.3",
         | 
| 67 | 
            +
              "use_cache": true,
         | 
| 68 | 
            +
              "v_head_dim": 128,
         | 
| 69 | 
            +
              "vocab_size": 129280
         | 
| 70 | 
            +
            }
         | 
    	
        configuration_deepseek.py
    ADDED
    
    | @@ -0,0 +1,210 @@ | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            from transformers.configuration_utils import PretrainedConfig
         | 
| 2 | 
            +
            from transformers.utils import logging
         | 
| 3 | 
            +
             | 
| 4 | 
            +
            logger = logging.get_logger(__name__)
         | 
| 5 | 
            +
             | 
| 6 | 
            +
            DEEPSEEK_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
         | 
| 7 | 
            +
            class DeepseekV3Config(PretrainedConfig):
         | 
| 8 | 
            +
                r"""
         | 
| 9 | 
            +
                This is the configuration class to store the configuration of a [`DeepseekV3Model`]. It is used to instantiate an DeepSeek
         | 
| 10 | 
            +
                model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
         | 
| 11 | 
            +
                defaults will yield a similar configuration to that of the DeepSeek-V3.
         | 
| 12 | 
            +
             | 
| 13 | 
            +
                Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
         | 
| 14 | 
            +
                documentation from [`PretrainedConfig`] for more information.
         | 
| 15 | 
            +
             | 
| 16 | 
            +
             | 
| 17 | 
            +
                Args:
         | 
| 18 | 
            +
                    vocab_size (`int`, *optional*, defaults to 129280):
         | 
| 19 | 
            +
                        Vocabulary size of the Deep model. Defines the number of different tokens that can be represented by the
         | 
| 20 | 
            +
                        `inputs_ids` passed when calling [`DeepseekV3Model`]
         | 
| 21 | 
            +
                    hidden_size (`int`, *optional*, defaults to 4096):
         | 
| 22 | 
            +
                        Dimension of the hidden representations.
         | 
| 23 | 
            +
                    intermediate_size (`int`, *optional*, defaults to 11008):
         | 
| 24 | 
            +
                        Dimension of the MLP representations.
         | 
| 25 | 
            +
                    moe_intermediate_size (`int`, *optional*, defaults to 1407):
         | 
| 26 | 
            +
                        Dimension of the MoE representations.
         | 
| 27 | 
            +
                    num_hidden_layers (`int`, *optional*, defaults to 32):
         | 
| 28 | 
            +
                        Number of hidden layers in the Transformer decoder.
         | 
| 29 | 
            +
                    num_nextn_predict_layers (`int`, *optional*, defaults to 1):
         | 
| 30 | 
            +
                        Number of nextn predict layers in the DeepSeekV3 Model.
         | 
| 31 | 
            +
                    num_attention_heads (`int`, *optional*, defaults to 32):
         | 
| 32 | 
            +
                        Number of attention heads for each attention layer in the Transformer decoder.
         | 
| 33 | 
            +
                    n_shared_experts (`int`, *optional*, defaults to None):
         | 
| 34 | 
            +
                        Number of shared experts, None means dense model.
         | 
| 35 | 
            +
                    n_routed_experts (`int`, *optional*, defaults to None):
         | 
| 36 | 
            +
                        Number of routed experts, None means dense model.
         | 
| 37 | 
            +
                    routed_scaling_factor (`float`, *optional*, defaults to 1.0):
         | 
| 38 | 
            +
                        Scaling factor or routed experts.
         | 
| 39 | 
            +
                    topk_method (`str`, *optional*, defaults to `gready`):
         | 
| 40 | 
            +
                        Topk method used in routed gate.
         | 
| 41 | 
            +
                    n_group (`int`, *optional*, defaults to None):
         | 
| 42 | 
            +
                        Number of groups for routed experts.
         | 
| 43 | 
            +
                    topk_group (`int`, *optional*, defaults to None):
         | 
| 44 | 
            +
                        Number of selected groups for each token(for each token, ensuring the selected experts is only within `topk_group` groups).
         | 
| 45 | 
            +
                    num_experts_per_tok (`int`, *optional*, defaults to None):
         | 
| 46 | 
            +
                        Number of selected experts, None means dense model.
         | 
| 47 | 
            +
                    moe_layer_freq (`int`, *optional*, defaults to 1):
         | 
| 48 | 
            +
                        The frequency of the MoE layer: one expert layer for every `moe_layer_freq - 1` dense layers.
         | 
| 49 | 
            +
                    first_k_dense_replace (`int`, *optional*, defaults to 0):
         | 
| 50 | 
            +
                        Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
         | 
| 51 | 
            +
                                                                        \--k dense layers--/
         | 
| 52 | 
            +
                    norm_topk_prob (`bool`, *optional*, defaults to False):
         | 
| 53 | 
            +
                        Whether to normalize the weights of the routed experts.
         | 
| 54 | 
            +
                    scoring_func (`str`, *optional*, defaults to 'softmax'):
         | 
| 55 | 
            +
                        Method of computing expert weights.
         | 
| 56 | 
            +
                    aux_loss_alpha (`float`, *optional*, defaults to 0.001):
         | 
| 57 | 
            +
                        Auxiliary loss weight coefficient.
         | 
| 58 | 
            +
                    seq_aux = (`bool`, *optional*, defaults to True):
         | 
| 59 | 
            +
                        Whether to compute the auxiliary loss for each individual sample.
         | 
| 60 | 
            +
                    num_key_value_heads (`int`, *optional*):
         | 
| 61 | 
            +
                        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
         | 
| 62 | 
            +
                        `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
         | 
| 63 | 
            +
                        `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
         | 
| 64 | 
            +
                        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
         | 
| 65 | 
            +
                        by meanpooling all the original heads within that group. For more details checkout [this
         | 
| 66 | 
            +
                        paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
         | 
| 67 | 
            +
                        `num_attention_heads`.
         | 
| 68 | 
            +
                    hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
         | 
| 69 | 
            +
                        The non-linear activation function (function or string) in the decoder.
         | 
| 70 | 
            +
                    max_position_embeddings (`int`, *optional*, defaults to 2048):
         | 
| 71 | 
            +
                        The maximum sequence length that this model might ever be used with.
         | 
| 72 | 
            +
                    initializer_range (`float`, *optional*, defaults to 0.02):
         | 
| 73 | 
            +
                        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
         | 
| 74 | 
            +
                    rms_norm_eps (`float`, *optional*, defaults to 1e-06):
         | 
| 75 | 
            +
                        The epsilon used by the rms normalization layers.
         | 
| 76 | 
            +
                    use_cache (`bool`, *optional*, defaults to `True`):
         | 
| 77 | 
            +
                        Whether or not the model should return the last key/values attentions (not used by all models). Only
         | 
| 78 | 
            +
                        relevant if `config.is_decoder=True`.
         | 
| 79 | 
            +
                    pad_token_id (`int`, *optional*):
         | 
| 80 | 
            +
                        Padding token id.
         | 
| 81 | 
            +
                    bos_token_id (`int`, *optional*, defaults to 1):
         | 
| 82 | 
            +
                        Beginning of stream token id.
         | 
| 83 | 
            +
                    eos_token_id (`int`, *optional*, defaults to 2):
         | 
| 84 | 
            +
                        End of stream token id.
         | 
| 85 | 
            +
                    pretraining_tp (`int`, *optional*, defaults to 1):
         | 
| 86 | 
            +
                        Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
         | 
| 87 | 
            +
                        document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
         | 
| 88 | 
            +
                        necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
         | 
| 89 | 
            +
                        issue](https://github.com/pytorch/pytorch/issues/76232).
         | 
| 90 | 
            +
                    tie_word_embeddings (`bool`, *optional*, defaults to `False`):
         | 
| 91 | 
            +
                        Whether to tie weight embeddings
         | 
| 92 | 
            +
                    rope_theta (`float`, *optional*, defaults to 10000.0):
         | 
| 93 | 
            +
                        The base period of the RoPE embeddings.
         | 
| 94 | 
            +
                    rope_scaling (`Dict`, *optional*):
         | 
| 95 | 
            +
                        Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
         | 
| 96 | 
            +
                        strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
         | 
| 97 | 
            +
                        `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
         | 
| 98 | 
            +
                        `max_position_embeddings` to the expected new maximum.
         | 
| 99 | 
            +
                    attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
         | 
| 100 | 
            +
                        Whether to use a bias in the query, key, value and output projection layers during self-attention.
         | 
| 101 | 
            +
                    attention_dropout (`float`, *optional*, defaults to 0.0):
         | 
| 102 | 
            +
                        The dropout ratio for the attention probabilities.
         | 
| 103 | 
            +
             | 
| 104 | 
            +
                ```python
         | 
| 105 | 
            +
                >>> from transformers import DeepseekV3Model, DeepseekV3Config
         | 
| 106 | 
            +
             | 
| 107 | 
            +
                >>> # Initializing a Deepseek-V3 style configuration
         | 
| 108 | 
            +
                >>> configuration = DeepseekV3Config()
         | 
| 109 | 
            +
             | 
| 110 | 
            +
                >>> # Accessing the model configuration
         | 
| 111 | 
            +
                >>> configuration = model.config
         | 
| 112 | 
            +
                ```"""
         | 
| 113 | 
            +
             | 
| 114 | 
            +
                model_type = "deepseek_v3"
         | 
| 115 | 
            +
                keys_to_ignore_at_inference = ["past_key_values"]
         | 
| 116 | 
            +
             | 
| 117 | 
            +
                def __init__(
         | 
| 118 | 
            +
                    self,
         | 
| 119 | 
            +
                    vocab_size=129280,
         | 
| 120 | 
            +
                    hidden_size=7168,
         | 
| 121 | 
            +
                    intermediate_size=18432,
         | 
| 122 | 
            +
                    moe_intermediate_size = 2048,
         | 
| 123 | 
            +
                    num_hidden_layers=61,
         | 
| 124 | 
            +
                    num_nextn_predict_layers=1,
         | 
| 125 | 
            +
                    num_attention_heads=128,
         | 
| 126 | 
            +
                    num_key_value_heads=128,
         | 
| 127 | 
            +
                    n_shared_experts = 1,
         | 
| 128 | 
            +
                    n_routed_experts = 256,
         | 
| 129 | 
            +
                    ep_size = 1,
         | 
| 130 | 
            +
                    routed_scaling_factor = 2.5,
         | 
| 131 | 
            +
                    kv_lora_rank = 512,
         | 
| 132 | 
            +
                    q_lora_rank = 1536,
         | 
| 133 | 
            +
                    qk_rope_head_dim = 64,
         | 
| 134 | 
            +
                    v_head_dim = 128,
         | 
| 135 | 
            +
                    qk_nope_head_dim = 128,
         | 
| 136 | 
            +
                    topk_method = 'noaux_tc',
         | 
| 137 | 
            +
                    n_group = 8,
         | 
| 138 | 
            +
                    topk_group = 4,
         | 
| 139 | 
            +
                    num_experts_per_tok = 8,
         | 
| 140 | 
            +
                    moe_layer_freq = 1,
         | 
| 141 | 
            +
                    first_k_dense_replace = 3,
         | 
| 142 | 
            +
                    norm_topk_prob = True,
         | 
| 143 | 
            +
                    scoring_func = 'sigmoid',
         | 
| 144 | 
            +
                    aux_loss_alpha = 0.001,
         | 
| 145 | 
            +
                    seq_aux = True,
         | 
| 146 | 
            +
                    hidden_act="silu",
         | 
| 147 | 
            +
                    max_position_embeddings=4096,
         | 
| 148 | 
            +
                    initializer_range=0.02,
         | 
| 149 | 
            +
                    rms_norm_eps=1e-6,
         | 
| 150 | 
            +
                    use_cache=True,
         | 
| 151 | 
            +
                    pad_token_id=None,
         | 
| 152 | 
            +
                    bos_token_id=0,
         | 
| 153 | 
            +
                    eos_token_id=1,
         | 
| 154 | 
            +
                    pretraining_tp=1,
         | 
| 155 | 
            +
                    tie_word_embeddings=False,
         | 
| 156 | 
            +
                    rope_theta=10000.0,
         | 
| 157 | 
            +
                    rope_scaling=None,
         | 
| 158 | 
            +
                    attention_bias=False,
         | 
| 159 | 
            +
                    attention_dropout=0.0,
         | 
| 160 | 
            +
                    **kwargs,
         | 
| 161 | 
            +
                ):
         | 
| 162 | 
            +
                    self.vocab_size = vocab_size
         | 
| 163 | 
            +
                    self.max_position_embeddings = max_position_embeddings
         | 
| 164 | 
            +
                    self.hidden_size = hidden_size
         | 
| 165 | 
            +
                    self.intermediate_size = intermediate_size
         | 
| 166 | 
            +
                    self.moe_intermediate_size = moe_intermediate_size
         | 
| 167 | 
            +
                    self.num_hidden_layers = num_hidden_layers
         | 
| 168 | 
            +
                    self.num_nextn_predict_layers = num_nextn_predict_layers
         | 
| 169 | 
            +
                    self.num_attention_heads = num_attention_heads
         | 
| 170 | 
            +
                    self.n_shared_experts = n_shared_experts
         | 
| 171 | 
            +
                    self.n_routed_experts = n_routed_experts
         | 
| 172 | 
            +
                    self.ep_size = ep_size
         | 
| 173 | 
            +
                    self.routed_scaling_factor = routed_scaling_factor
         | 
| 174 | 
            +
                    self.kv_lora_rank = kv_lora_rank
         | 
| 175 | 
            +
                    self.q_lora_rank = q_lora_rank
         | 
| 176 | 
            +
                    self.qk_rope_head_dim = qk_rope_head_dim
         | 
| 177 | 
            +
                    self.v_head_dim = v_head_dim
         | 
| 178 | 
            +
                    self.qk_nope_head_dim = qk_nope_head_dim
         | 
| 179 | 
            +
                    self.topk_method = topk_method
         | 
| 180 | 
            +
                    self.n_group = n_group
         | 
| 181 | 
            +
                    self.topk_group = topk_group
         | 
| 182 | 
            +
                    self.num_experts_per_tok = num_experts_per_tok
         | 
| 183 | 
            +
                    self.moe_layer_freq = moe_layer_freq
         | 
| 184 | 
            +
                    self.first_k_dense_replace = first_k_dense_replace
         | 
| 185 | 
            +
                    self.norm_topk_prob = norm_topk_prob
         | 
| 186 | 
            +
                    self.scoring_func = scoring_func
         | 
| 187 | 
            +
                    self.aux_loss_alpha = aux_loss_alpha
         | 
| 188 | 
            +
                    self.seq_aux = seq_aux
         | 
| 189 | 
            +
                    # for backward compatibility
         | 
| 190 | 
            +
                    if num_key_value_heads is None:
         | 
| 191 | 
            +
                        num_key_value_heads = num_attention_heads
         | 
| 192 | 
            +
             | 
| 193 | 
            +
                    self.num_key_value_heads = num_key_value_heads
         | 
| 194 | 
            +
                    self.hidden_act = hidden_act
         | 
| 195 | 
            +
                    self.initializer_range = initializer_range
         | 
| 196 | 
            +
                    self.rms_norm_eps = rms_norm_eps
         | 
| 197 | 
            +
                    self.pretraining_tp = pretraining_tp
         | 
| 198 | 
            +
                    self.use_cache = use_cache
         | 
| 199 | 
            +
                    self.rope_theta = rope_theta
         | 
| 200 | 
            +
                    self.rope_scaling = rope_scaling
         | 
| 201 | 
            +
                    self.attention_bias = attention_bias
         | 
| 202 | 
            +
                    self.attention_dropout = attention_dropout
         | 
| 203 | 
            +
             | 
| 204 | 
            +
                    super().__init__(
         | 
| 205 | 
            +
                        pad_token_id=pad_token_id,
         | 
| 206 | 
            +
                        bos_token_id=bos_token_id,
         | 
| 207 | 
            +
                        eos_token_id=eos_token_id,
         | 
| 208 | 
            +
                        tie_word_embeddings=tie_word_embeddings,
         | 
| 209 | 
            +
                        **kwargs,
         | 
| 210 | 
            +
                    )
         | 
    	
        model-00001-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:c2388e6b127ce6664e35c5e2529c3ce4bfc99f4f7fb6fa48e92b29ed5e4922af
         | 
| 3 | 
            +
            size 5234139343
         | 
    	
        model-00002-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:5f450c75da7eb897b74a092eee65df8bb115fce81cccd2bbaeb220bd97197875
         | 
| 3 | 
            +
            size 4302383966
         | 
    	
        model-00003-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:c79b3418357348d089e88f9de17ac52d7b7a1cfe5dfc7c1b2f1ac1451298b080
         | 
| 3 | 
            +
            size 4302384375
         | 
    	
        model-00004-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:ad5ca32dcb6c50a954bbeac833f767fcfe398b0d05aea63473907322c1a5bed5
         | 
| 3 | 
            +
            size 4302349996
         | 
    	
        model-00005-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:e06768f65df4c19a033253a1994fe7145be9ce28131398ad9f320e3d03e8ab73
         | 
| 3 | 
            +
            size 4302384154
         | 
    	
        model-00006-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:3eb84d165db019a61e5ca4754d3c6c6aa298b88af5a7617b15b0eca42f64e14f
         | 
| 3 | 
            +
            size 4372073602
         | 
    	
        model-00007-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:d6f299f7b410b9a7806927b5d2d413fae1f2c1dfa340bb0037d02d220cd8c080
         | 
| 3 | 
            +
            size 4306080097
         | 
    	
        model-00008-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:da197fe67d243284d133310255e436759ea1eff8c3f2a684da53496a76d11ce2
         | 
| 3 | 
            +
            size 4302384356
         | 
    	
        model-00009-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:dbd2a790cc3d94aa3d13a0301e23de8bb81e0168e100650dbcac36695533e1d3
         | 
| 3 | 
            +
            size 4302350190
         | 
    	
        model-00010-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:137f0f0407290d8d45d1a3f60c67aa10a2a5434e95d8d07eaf82df6f475d8cf8
         | 
| 3 | 
            +
            size 4302383960
         | 
    	
        model-00011-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:47b308ea2f7f7a4a812a28c0a5c75d5f648a08b0c118bd0863fd2f7320e6bc76
         | 
| 3 | 
            +
            size 4302384375
         | 
    	
        model-00012-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:4cda99d04536d50e067c3a2c8c9ead34d307b2002500b1a8b8c7ad4ac85a9061
         | 
| 3 | 
            +
            size 1321583941
         | 
    	
        model-00013-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:33adbb52d544cac90603081e871ff3260ee993c479fd96194716da54cfc57040
         | 
| 3 | 
            +
            size 4302317244
         | 
    	
        model-00014-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:05b19f1bd14644e5e6db24575322d3e74a611f26b71a8021345caf0d9b416fd9
         | 
| 3 | 
            +
            size 4302384328
         | 
    	
        model-00015-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:0484ca365f84ed446f992e3e43eeb4e182a05c8d554be385f1e1535d4d80f4fe
         | 
| 3 | 
            +
            size 4302350218
         | 
    	
        model-00016-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:1a70800c3373608f3b3f8dd4643c9574360896e3dd647acb39f4e794284e6ee6
         | 
| 3 | 
            +
            size 4302383932
         | 
    	
        model-00017-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:bd5abae258a54b0b4ded4f85c272b34a05f968b32dfc46412af8f5775b95ece9
         | 
| 3 | 
            +
            size 4302384377
         | 
    	
        model-00018-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:b5418a2987ad8d4a188592c014fbb107f7605266375c5b29d42b50d449a0a9b0
         | 
| 3 | 
            +
            size 4302350026
         | 
    	
        model-00019-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:69bbd687eafeb7e7d7ff111af3233a10e7c74c459b6d3b466a7d93f6878d4915
         | 
| 3 | 
            +
            size 4302384124
         | 
    	
        model-00020-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:2560e878221496c2326c0538d7898e28b30cec87c70c599b321c4292bac91fe1
         | 
| 3 | 
            +
            size 4302384377
         | 
    	
        model-00021-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:09e6825f850ba74ac9293d8179b3e7ebe3a69a3193fb0a129bf5dcaebf0b6201
         | 
| 3 | 
            +
            size 4302350413
         | 
    	
        model-00022-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:083a0025a915e3eb8077b24abf6d26cbbf60709f308e6eaec4a3121ab368788f
         | 
| 3 | 
            +
            size 4302384900
         | 
    	
        model-00023-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:a10ddf9357a7603dc603fa1bbc4e0e947637f33cb6f2bfc0725a721f93622342
         | 
| 3 | 
            +
            size 4302350808
         | 
    	
        model-00024-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:5f13210eca8ecf63123dce2ebcc9de5833b6d31b47f85970b6e6f7f00997ef78
         | 
| 3 | 
            +
            size 4302384504
         | 
    	
        model-00025-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:d511a1879fd6cdba6823f29af05242e3f46374eec1d957a5b4db01c5d83ebc8c
         | 
| 3 | 
            +
            size 4302384961
         | 
    	
        model-00026-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:0c50460d2ee189e76fc3eb111ef0ecd4d28dd05b5bca619bdd122827fca31e10
         | 
| 3 | 
            +
            size 4302350620
         | 
    	
        model-00027-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:260ad43d8099317e55c97d7931941419d04a45f2169967328dd721104f9ed63d
         | 
| 3 | 
            +
            size 4302384692
         | 
    	
        model-00028-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:1acbd0856de26f397bef297165e6a12d2fd419fdda8237891b0da5c43c27a2a9
         | 
| 3 | 
            +
            size 4302384963
         | 
    	
        model-00029-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:20d3ef472a46a6c7d340476b6ac105c7502ce71d1903022502cc5069c1971792
         | 
| 3 | 
            +
            size 4302350448
         | 
    	
        model-00030-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:ac1d12d32f3a95e111cfd09f0255ed6af2b9237187aef1dcf78230c1f637abdd
         | 
| 3 | 
            +
            size 4302384884
         | 
    	
        model-00031-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:eace4ef31c4ff04f22501b93eb476fd8e0289c749d0dd30c9c510ec520b98055
         | 
| 3 | 
            +
            size 4302350824
         | 
    	
        model-00032-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:ac7f5250660b596b6e3c430c9f0d3139b0e6b7aa1fc08069dc0e3966f08c92bf
         | 
| 3 | 
            +
            size 4302384488
         | 
    	
        model-00033-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:f71549c7a33eaacdff8a2406a82957562404548a7fe5f2130a3573b36dbddef0
         | 
| 3 | 
            +
            size 4302384963
         | 
    	
        model-00034-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:dc52ac9cd64be1135b3ed03df71d9c63af56b62738156b95af65b3400abe781f
         | 
| 3 | 
            +
            size 1747417474
         | 
    	
        model-00035-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:5f4677233a2a36db63ea5fb010dc3a5ef882285f3ceb2c428ba4c1cdae9ed585
         | 
| 3 | 
            +
            size 4302317817
         | 
    	
        model-00036-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:f0b62c567e3c9e0bcf51bcd994d0ed67d5df44c2cadcf407522e445be8ecf6b2
         | 
| 3 | 
            +
            size 4302384914
         | 
    	
        model-00037-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:9d7c8a1c35d1df030c95186eb6acaceebabbef95374de6c8a385c4b87e1895d2
         | 
| 3 | 
            +
            size 4302350794
         | 
    	
        model-00038-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:ea241680b9422782815a20eb1d7a99e37a65260b08a17557e946b9c20219649d
         | 
| 3 | 
            +
            size 4302384518
         | 
    	
        model-00039-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:6dfd35cf8163ebcdd73ee4cc6beacf011b1d00f82b842fc3d914f7230e52800b
         | 
| 3 | 
            +
            size 4302384963
         | 
    	
        model-00040-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:fc4c69c0e91ece15f1a1bc0bf17e1d36c514bbb138c1412a0ced41118d75a052
         | 
| 3 | 
            +
            size 4302350602
         | 
    	
        model-00041-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:333c52631a3de667f9c4a01dd644a9520832ca4a933be7c6ddfdfe6525573c3e
         | 
| 3 | 
            +
            size 4302384710
         | 
    	
        model-00042-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:450f75fd01ff57776f0679d9b3c819be08d7057e265d43dda2c65d4851efd89d
         | 
| 3 | 
            +
            size 4302384963
         | 
    	
        model-00043-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:aab6deabc7117de24a14a81aae0c5ff6e10be064913cc48b21d9a89fa511b45c
         | 
| 3 | 
            +
            size 4302350432
         | 
    	
        model-00044-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:c9d6582f530adcf2c8b864bd9344652fd0bb939360e7c30f6e47bc09986cc576
         | 
| 3 | 
            +
            size 4302384900
         | 
    	
        model-00045-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:938c7306ce56ae407005426e38a246a6ee5ebb9f941230da38d99493ea954831
         | 
| 3 | 
            +
            size 4302350808
         | 
    	
        model-00046-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:e13e11c8bacd5557df1e6f864e7e260283b6a9672f85afd1d97841b63db899b0
         | 
| 3 | 
            +
            size 4302384504
         | 
    	
        model-00047-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:3ae5dc07da78e72f3b8064b53d042081a64755e4dfd792ac13f1615d6f70a86a
         | 
| 3 | 
            +
            size 4302384961
         | 
    	
        model-00048-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:df85e837c8960a00205f57b27edb17278c18beda17ce1ed3acf482ed85076c6d
         | 
| 3 | 
            +
            size 4302350620
         | 
