Upload folder using huggingface_hub

checkpoint-200/config.json

@@ -0,0 +1,50 @@
+{
+  "_name_or_path": "/workspace/sigrid/kozistr-ko-sentence-embeddings/reranker-data/gte-korean-reranker-base-241210-c",
+  "architectures": [
+    "NewForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration.NewConfig",
+    "AutoModel": "Alibaba-NLP/new-impl--modeling.NewModel",
+    "AutoModelForMaskedLM": "Alibaba-NLP/new-impl--modeling.NewForMaskedLM",
+    "AutoModelForMultipleChoice": "Alibaba-NLP/new-impl--modeling.NewForMultipleChoice",
+    "AutoModelForQuestionAnswering": "Alibaba-NLP/new-impl--modeling.NewForQuestionAnswering",
+    "AutoModelForSequenceClassification": "modeling.NewForSequenceClassification",
+    "AutoModelForTokenClassification": "Alibaba-NLP/new-impl--modeling.NewForTokenClassification"
+  },
+  "classifier_dropout": 0.0,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "layer_norm_type": "layer_norm",
+  "logn_attention_clip1": false,
+  "logn_attention_scale": false,
+  "max_position_embeddings": 8192,
+  "model_type": "new",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pack_qkv": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "rope",
+  "rope_scaling": {
+    "factor": 8.0,
+    "type": "ntk"
+  },
+  "rope_theta": 20000,
+  "torch_dtype": "float16",
+  "transformers_version": "4.44.2",
+  "type_vocab_size": 1,
+  "unpad_inputs": false,
+  "use_memory_efficient_attention": false,
+  "vocab_size": 250048
+}

checkpoint-200/configuration.py

@@ -0,0 +1,145 @@
+# coding=utf-8
+# Copyright 2024 The GTE Team Authors and Alibaba Group.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" NEW model configuration"""
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class NewConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`NewModel`] or a [`TFNewModel`]. It is used to
+    instantiate a NEW model according to the specified arguments, defining the model architecture. Instantiating a
+    configuration with the defaults will yield a similar configuration to that of the NEW
+    [izhx/new-base-en](https://huggingface.co/izhx/new-base-en) architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 30522):
+            Vocabulary size of the NEW model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`NewModel`] or [`TFNewModel`].
+        hidden_size (`int`, *optional*, defaults to 768):
+            Dimensionality of the encoder layers and the pooler layer.
+        num_hidden_layers (`int`, *optional*, defaults to 12):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 12):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        intermediate_size (`int`, *optional*, defaults to 3072):
+            Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
+        hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"silu"` and `"gelu_new"` are supported.
+        hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
+            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+        attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
+            The dropout ratio for the attention probabilities.
+        max_position_embeddings (`int`, *optional*, defaults to 512):
+            The maximum sequence length that this model might ever be used with. Typically set this to something large
+            just in case (e.g., 512 or 1024 or 2048).
+        type_vocab_size (`int`, *optional*, defaults to 2):
+            The vocabulary size of the `token_type_ids` passed when calling [`NewModel`] or [`TFNewModel`].
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-12):
+            The epsilon used by the layer normalization layers.
+        position_embedding_type (`str`, *optional*, defaults to `"rope"`):
+            Type of position embedding. Choose one of `"absolute"`, `"rope"`.
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
+            strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
+            `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
+            `max_position_embeddings` to the expected new maximum. See the following thread for more information on how
+            these scaling strategies behave:
+            https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an
+            experimental feature, subject to breaking API changes in future versions.
+        classifier_dropout (`float`, *optional*):
+            The dropout ratio for the classification head.
+
+    Examples:
+
+    ```python
+    >>> from transformers import NewConfig, NewModel
+
+    >>> # Initializing a NEW izhx/new-base-en style configuration
+    >>> configuration = NewConfig()
+
+    >>> # Initializing a model (with random weights) from the izhx/new-base-en style configuration
+    >>> model = NewModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "new"
+
+    def __init__(
+        self,
+        vocab_size=30528,
+        hidden_size=768,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        intermediate_size=3072,
+        hidden_act="gelu",
+        hidden_dropout_prob=0.1,
+        attention_probs_dropout_prob=0.0,
+        max_position_embeddings=2048,
+        type_vocab_size=1,
+        initializer_range=0.02,
+        layer_norm_type='layer_norm',
+        layer_norm_eps=1e-12,
+        # pad_token_id=0,
+        position_embedding_type="rope",
+        rope_theta=10000.0,
+        rope_scaling=None,
+        classifier_dropout=None,
+        pack_qkv=True,
+        unpad_inputs=False,
+        use_memory_efficient_attention=False,
+        logn_attention_scale=False,
+        logn_attention_clip1=False,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.hidden_act = hidden_act
+        self.intermediate_size = intermediate_size
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.max_position_embeddings = max_position_embeddings
+        self.type_vocab_size = type_vocab_size
+        self.initializer_range = initializer_range
+        self.layer_norm_type = layer_norm_type
+        self.layer_norm_eps = layer_norm_eps
+        self.position_embedding_type = position_embedding_type
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.classifier_dropout = classifier_dropout
+
+        self.pack_qkv = pack_qkv
+        self.unpad_inputs = unpad_inputs
+        self.use_memory_efficient_attention = use_memory_efficient_attention
+        self.logn_attention_scale = logn_attention_scale
+        self.logn_attention_clip1 = logn_attention_clip1

checkpoint-200/global_step200/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a065a800810f8bf3a86a107772dca7e31bcdf93e42c41ff15e8bc84e57082939
+size 4283468624

checkpoint-200/latest

@@ -0,0 +1 @@
+global_step200

checkpoint-200/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efccdc583e449d5ad7b1ab1beda845d50c68d48c9acd72a1eef1ece00b1ac8b1
+size 611934706

checkpoint-200/modeling.py

@@ -0,0 +1,1418 @@
+# coding=utf-8
+# Copyright 2024 The GTE Team Authors and Alibaba Group.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch NEW model."""
+
+import math
+from dataclasses import dataclass
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.utils.checkpoint
+from torch import nn
+
+from transformers.activations import ACT2FN
+from transformers.modeling_outputs import (
+    BaseModelOutput,
+    BaseModelOutputWithPooling,
+    MaskedLMOutput,
+    MultipleChoiceModelOutput,
+    QuestionAnsweringModelOutput,
+    SequenceClassifierOutput,
+    ModelOutput,
+)
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import logging
+
+try:
+    import xformers.ops as xops
+except ImportError as e:
+    xops = None
+
+from .configuration import NewConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+# Adapted from https://github.com/HazyResearch/flash-attention/blob/main/flash_attn/bert_padding.py
+# Which was adapted from https://github.com/mlcommons/training_results_v1.1/blob/main/NVIDIA/benchmarks/bert/implementations/pytorch/padding.py
+class IndexFirstAxis(torch.autograd.Function):
+    @staticmethod
+    def forward(ctx, input, indices):
+        ctx.save_for_backward(indices)
+        assert input.ndim >= 2
+        ctx.first_axis_dim, other_shape = input.shape[0], input.shape[1:]
+        second_dim = other_shape.numel()
+        # TD [2022-03-04] For some reason torch.gather is a bit faster than indexing.
+        # return input[indices]
+        # return torch.gather(
+        #     rearrange(input, "b ... -> b (...)"), 0, repeat(indices, "z -> z d", d=second_dim)
+        # ).reshape(-1, *other_shape)
+        return torch.gather(
+            input.view(ctx.first_axis_dim, second_dim),
+            0,
+            indices.unsqueeze(-1).expand(indices.size(0), second_dim)
+        ).reshape(-1, *other_shape)
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        (indices,) = ctx.saved_tensors
+        assert grad_output.ndim >= 2
+        other_shape = grad_output.shape[1:]
+        # grad_output = rearrange(grad_output, "b ... -> b (...)")
+        grad_output = grad_output.view(grad_output.size(0), other_shape.numel())
+        grad_input = torch.zeros(
+            [ctx.first_axis_dim, grad_output.shape[1]],
+            device=grad_output.device,
+            dtype=grad_output.dtype,
+        )
+        # TD [2022-03-04] For some reason torch.scatter is a bit faster than indexing.
+        # grad_input[indices] = grad_output
+        # grad_input.scatter_(0, repeat(indices, "z -> z d", d=grad_output.shape[1]), grad_output)
+        grad_input.scatter_(
+            0, indices.unsqueeze(-1).expand(indices.size(0), grad_output.size(1)), grad_output
+        )
+        return grad_input.reshape(ctx.first_axis_dim, *other_shape), None
+
+
+index_first_axis = IndexFirstAxis.apply
+
+
+def unpad_input(hidden_states, attention_mask=None, indices=None):
+    """
+    Arguments:
+        hidden_states: (batch, seqlen, ...)
+        attention_mask: (batch, seqlen), bool / int, 1 means valid and 0 means not valid.
+        indices: (total_nnz), the indices of non-masked tokens from the flattened input sequence.
+    Return:
+        hidden_states: (total_nnz, ...), where total_nnz = number of tokens in selected in attention_mask.
+    """
+    if indices is None:
+        assert attention_mask is not None
+        indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+
+    # TD [2022-03-04] We don't want to index with a bool mask, because Pytorch will expand the
+    # bool mask, then call nonzero to get the indices, then index with those. The indices is @dim
+    # times larger than it needs to be, wasting memory. It's faster and more memory-efficient to
+    # index with integer indices. Moreover, torch's index is a bit slower than it needs to be,
+    # so we write custom forward and backward to make it a bit faster.
+    hidden_states = hidden_states.view(-1, *hidden_states.shape[2:])
+    return index_first_axis(hidden_states, indices)
+
+
+class IndexPutFirstAxis(torch.autograd.Function):
+    @staticmethod
+    def forward(
+        ctx,
+        values: torch.Tensor,
+        indices: torch.Tensor,
+        first_axis_dim
+    ) -> torch.Tensor:
+        ctx.save_for_backward(indices)
+        assert indices.ndim == 1
+        assert values.ndim >= 2
+        output = torch.zeros(
+            first_axis_dim, *values.shape[1:], device=values.device, dtype=values.dtype
+        )
+        output[indices] = values
+        return output
+
+    @staticmethod
+    def backward(ctx, grad_output: torch.Tensor) -> Tuple[torch.Tensor, None, None]:
+        indices, = ctx.saved_tensors
+        grad_values = grad_output[indices]
+        return grad_values, None, None
+
+
+index_put_first_axis = IndexPutFirstAxis.apply
+
+
+def pad_input(inputs: torch.Tensor, indices: torch.Tensor, batch: int, seqlen: int) -> torch.Tensor:
+    """Add padding to sequences.
+
+    Arguments:
+        inputs: (total_nnz, ...), where total_nnz = number of tokens in selected in attention_mask.
+        indices: (total_nnz), `indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()`
+        batch: int batch_size
+        seqlen: int max sequence length
+
+    Returns:
+        inputs: (batch, seqlen, ...)
+    """
+    output = index_put_first_axis(inputs, indices, batch * seqlen)
+    return output.view(batch, seqlen, *inputs.shape[1:])
+
+
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin):
+    """Applies Rotary Position Embedding to the query and key tensors.
+
+    Args:
+        q (`torch.Tensor`): The query tensor.
+        k (`torch.Tensor`): The key tensor.
+        cos (`torch.Tensor`): The cosine part of the rotary embedding.
+        sin (`torch.Tensor`): The sine part of the rotary embedding.
+    Returns:
+        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+    """
+    cos, sin = cos.to(q.dtype), sin.to(q.dtype)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+class RotaryEmbedding(torch.nn.Module):
+    def __init__(self, dim, max_position_embeddings=512, base=10000.0, device=None):
+        super().__init__()
+
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+        # Build here to make `torch.jit.trace` work.
+        self._set_cos_sin_cache(
+            seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()
+        )
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=torch.float32)
+
+        freqs = torch.einsum("i,j->ij", t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
+
+    def forward(self, x, seq_len=None):
+        # x: [bs, num_attention_heads, seq_len, head_size]
+        if seq_len > self.max_seq_len_cached:
+            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
+
+        return (
+            self.cos_cached[:seq_len, ...].to(dtype=x.dtype),
+            self.sin_cached[:seq_len, ...].to(dtype=x.dtype),
+        )
+
+
+class NTKScalingRotaryEmbedding(RotaryEmbedding):
+    """RotaryEmbedding extended with fixed and mixed NTK scaling. https://kexue.fm/archives/9706 """
+
+    def __init__(self, dim, max_position_embeddings=512, base=10000, device=None, scaling_factor=1.0, mixed_b=None):
+        self.scaling_factor = scaling_factor
+        self.mixed_b = mixed_b
+        super().__init__(dim, max_position_embeddings, base, device)
+        max_position_embeddings = max_position_embeddings * self.scaling_factor
+        self._set_cos_sin_cache(max_position_embeddings, self.inv_freq.device, torch.get_default_dtype())
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+
+        if seq_len > self.max_position_embeddings:
+            base = self.base * (self.scaling_factor if self.mixed_b is None else 1)
+            inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
+
+            if self.mixed_b is None:
+                inv_freq = inv_freq / self.scaling_factor ** (2 / self.dim)  # (6)
+            else:
+                a = torch.tensor(self.scaling_factor).log() / (self.dim / 2) ** self.mixed_b  # (13)
+                lambda_1_m = (a * torch.arange(1, self.dim // 2 + 1).float().to(device) ** self.mixed_b).exp()  # (12)
+                inv_freq = inv_freq / lambda_1_m  # (10)
+
+            self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=torch.float32)
+
+        freqs = torch.einsum("i,j->ij", t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        """
+        RMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+
+LAYER_NORM = {
+    'layer_norm': nn.LayerNorm,
+    'rms_norm': RMSNorm
+}
+
+
+class NewEmbeddings(nn.Module):
+    """
+    Embedding and Unpadding.
+    """
+
+    def __init__(self, config: NewConfig):
+        super().__init__()
+        self.padding_idx = config.pad_token_id
+        self.word_embeddings = nn.Embedding(
+            config.vocab_size, config.hidden_size, padding_idx=self.padding_idx
+        )
+
+        self.position_embedding_type = config.position_embedding_type
+        if self.position_embedding_type == 'absolute':
+            self.position_embeddings = nn.Embedding(
+                config.max_position_embeddings, config.hidden_size, padding_idx=self.padding_idx
+            )
+        elif self.position_embedding_type == 'rope':
+            self._init_rope(config)
+        else:
+            raise ValueError
+
+        self.type_vocab_size = config.type_vocab_size
+        if self.type_vocab_size > 0:
+            self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)
+
+        # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
+        # any TensorFlow checkpoint file
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        # position_ids is contiguous in memory and excluded when serialized
+        self.register_buffer(
+            "position_ids", torch.arange(config.max_position_embeddings), persistent=False
+        )
+
+    def _init_rope(self, config):
+        kwargs = dict(
+            dim=int(config.hidden_size / config.num_attention_heads),
+            max_position_embeddings=config.max_position_embeddings,
+            base=config.rope_theta
+        )
+        if config.rope_scaling is None:
+            self.rotary_emb = RotaryEmbedding(**kwargs)
+        else:
+            kwargs.update(scaling_factor=config.rope_scaling["factor"])
+            scaling_type = config.rope_scaling["type"]
+            if scaling_type == 'ntk':
+                kwargs.update(mixed_b=config.rope_scaling.get('mixed_b', None))
+                self.rotary_emb = NTKScalingRotaryEmbedding(**kwargs)
+            # elif scaling_type == "linear":
+            #     self.rotary_emb = LinearScalingRotaryEmbedding(**kwargs)
+            # elif scaling_type == "dynamic":
+            #     self.rotary_emb = DynamicNTKScalingRotaryEmbedding(**kwargs)
+            else:
+                raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
+
+    def forward(
+        self,
+        unpad_inputs: bool,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        length: Optional[List[int]] = None,
+        token_type_ids: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+    ) -> Tuple[torch.Tensor, torch.Tensor, Optional[Tuple], Optional[List[int]]]:
+        """
+        """
+        if inputs_embeds is None:
+            device, input_shape = input_ids.device, input_ids.shape
+        else:
+            device, input_shape = inputs_embeds.device, inputs_embeds.shape[:2]
+        batch_size, seq_length = input_shape
+
+        # Set attention_mask if it's None
+        if attention_mask is None:
+            attention_mask = torch.ones(input_shape, device=device)
+            if length is not None:
+                for i, l in enumerate(length):
+                    attention_mask[i, l:] = 0
+
+        # Set attention_mask_bool for unpadding
+        if unpad_inputs:
+            attention_mask_bool = attention_mask.bool()
+            if length is None:
+                length = attention_mask.sum(-1).tolist()
+
+        # Get word embeddings
+        if inputs_embeds is None:
+            if unpad_inputs:
+                input_ids = input_ids[attention_mask_bool].unsqueeze(0)
+            inputs_embeds = self.word_embeddings(input_ids)
+        else:
+            if unpad_inputs:
+                inputs_embeds = inputs_embeds[attention_mask_bool].unsqueeze(0)
+        embeddings = inputs_embeds
+
+        # Set and unpad position_ids
+        if position_ids is None:
+            if seq_length > self.position_ids.size(0):
+                self.register_buffer(
+                    "position_ids", torch.arange(seq_length, device=embeddings.device), persistent=False
+                )
+            if unpad_inputs:
+                # [1, cumsum_seq_len]
+                position_ids = torch.cat([self.position_ids[:l] for l in length]).unsqueeze(0)
+            else:
+                # [bs, seq_len]
+                position_ids = self.position_ids[:seq_length].expand(batch_size, -1)
+        elif unpad_inputs:
+            position_ids = position_ids[attention_mask_bool].unsqueeze(0)  # [1, cumsum_seq_len]
+
+        # Compute rotary embedding
+        if self.position_embedding_type == 'rope':
+            rope_cos, rope_sin = self.rotary_emb(inputs_embeds, seq_len=seq_length)
+            rope_cos = rope_cos[position_ids].unsqueeze(2)  # [bs, seq_len, 1, dim]
+            rope_sin = rope_sin[position_ids].unsqueeze(2)  # [bs, seq_len, 1, dim]
+            rope_embeds = rope_cos, rope_sin
+        else:
+            rope_embeds = None
+
+        if self.type_vocab_size > 0:
+            if token_type_ids is None:
+                token_type_ids = position_ids.mul(0)
+            else:
+                if self.type_vocab_size < 2:
+                    token_type_ids.mul_(0)
+                if unpad_inputs:
+                    token_type_ids = token_type_ids[attention_mask_bool].unsqueeze(0)
+
+            token_type_embeddings = self.token_type_embeddings(token_type_ids)
+            embeddings = embeddings + token_type_embeddings
+
+        # BERT position
+        if self.position_embedding_type == "absolute":
+            position_embeddings = self.position_embeddings(position_ids)
+            embeddings = embeddings + position_embeddings
+
+        embeddings = self.LayerNorm(embeddings)
+        embeddings = self.dropout(embeddings)
+
+        return embeddings, attention_mask, rope_embeds, length
+
+
+class NewAttention(nn.Module):
+    def __init__(self, config: NewConfig, pack_qkv=None, use_memory_efficient_attention=None):
+        super().__init__()
+        self.config = config
+        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
+            raise ValueError(
+                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
+                f"heads ({config.num_attention_heads})"
+            )
+
+        self.hidden_size = config.hidden_size
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+
+        if pack_qkv is None:
+            pack_qkv = config.pack_qkv
+        self.pack_qkv = pack_qkv
+
+        if self.pack_qkv:
+            self.qkv_proj = nn.Linear(config.hidden_size, self.all_head_size * 3, bias=True)
+        else:
+            self.q_proj = nn.Linear(config.hidden_size, self.all_head_size, bias=True)
+            self.k_proj = nn.Linear(config.hidden_size, self.all_head_size, bias=True)
+            self.v_proj = nn.Linear(config.hidden_size, self.all_head_size, bias=True)
+
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
+        self.o_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
+
+        if use_memory_efficient_attention is None:
+            use_memory_efficient_attention = self.config.use_memory_efficient_attention
+        self.use_memory_efficient_attention = use_memory_efficient_attention
+        self.memory_efficient_attention = None if xops is None else xops.memory_efficient_attention
+        if self.use_memory_efficient_attention:
+            assert self.memory_efficient_attention is not None, 'please install xformers'
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_bias: torch.FloatTensor,
+        rope_embeds: Optional[Tuple[torch.FloatTensor, torch.FloatTensor]] = None,
+        padding_inputs: Optional[Tuple] = None,  # indices, batch, seqlen
+        attention_scale: Optional[torch.FloatTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        output_attentions: Optional[bool] = False,
+        qkv_inputs: Optional[Tuple] = None,  # For RetroMAE
+    ) -> Tuple[torch.Tensor, ...]:
+        shape_hd = (self.num_attention_heads, self.attention_head_size)
+        # qkv
+        if self.pack_qkv and qkv_inputs is None:
+            qkv_pack = self.qkv_proj(hidden_states).split(self.all_head_size, dim=-1)
+        else:
+            if qkv_inputs is None:
+                qkv_inputs = (hidden_states, hidden_states, hidden_states)
+            qkv_pack = [
+                getattr(self, n + '_proj')(s) for s, n in zip(qkv_inputs, 'qkv')
+            ]
+        query_states, key_states, value_states = [t.view(t.shape[:-1] + shape_hd) for t in qkv_pack]
+
+        if self.config.position_embedding_type == 'rope':
+            query_states, key_states = apply_rotary_pos_emb(query_states, key_states, *rope_embeds)
+
+        dtype = query_states.dtype
+
+        if self.config.logn_attention_scale and attention_scale is not None:
+            # https://kexue.fm/archives/8823
+            query_states = query_states * attention_scale.to(dtype)
+
+        if padding_inputs is not None:
+            query_states = pad_input(query_states.squeeze(), *padding_inputs)
+            key_states = pad_input(key_states.squeeze(), *padding_inputs)
+            value_states = pad_input(value_states.squeeze(), *padding_inputs)
+
+        if self.use_memory_efficient_attention:
+            assert self.memory_efficient_attention is not None, "xformers is not loaded"
+            assert output_attentions is False, "memory_efficient_attention do not output attentions"
+            assert head_mask is None, "Not support yet"
+            attention_probs = None
+            if torch.is_tensor(attention_bias):
+                attention_bias = attention_bias.to(dtype)
+            context_layer = self.memory_efficient_attention(
+                query_states,
+                key_states,
+                value_states,
+                attn_bias=attention_bias,
+                p=self.dropout.p
+            )
+        else:
+            if output_attentions and isinstance(self, NewSdpaAttention):
+                raise RuntimeError("SDPA do not output attentions")
+            context_layer, attention_probs = self._attention(
+                query_states, key_states, value_states, attention_bias, head_mask
+            )
+
+        if padding_inputs is not None:
+            context_layer = unpad_input(context_layer, indices=padding_inputs[0])
+
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        context_layer = context_layer.view(new_context_layer_shape)
+
+        # output proj
+        attn_output = self.o_proj(context_layer)
+
+        # add attentions if we output them
+        outputs = (attn_output, attention_probs) if output_attentions else (attn_output,)
+        return outputs
+
+    def _attention(self, query_states, key_states, value_states, attention_bias, head_mask):
+        """
+        Args:
+            q/k/v: (B, L, n_head, head_dim),
+        Returns:
+            attn_output: (B L, n_head, head_dim)
+        """
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2))
+
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+        if attention_bias is not None:
+            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
+            attention_scores = attention_scores + attention_bias
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = nn.functional.softmax(attention_scores, dim=-1)
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        if self.dropout.p > 0:
+            attention_probs = self.dropout(attention_probs)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs = attention_probs * head_mask
+
+        context_layer = torch.matmul(attention_probs, value_states)
+
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        return context_layer, attention_probs
+
+
+class NewSdpaAttention(NewAttention):
+    """
+    New attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
+    `NewAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
+    SDPA API.
+    """
+    def __init__(self, config: NewConfig, **kwargs):
+        super().__init__(config, **kwargs)
+        # torch.backends.cuda.enable_mem_efficient_sdp(False)
+        # logger.warning(
+        #     "Disable memory efficient attention kernel for `NewSdpaAttention`, you can set "
+        #     "`use_memory_efficient_attention=True` if it expected to use."
+        # )
+
+    def _attention(self, query_states, key_states, value_states, attention_bias, head_mask):
+        attn_output = torch.nn.functional.scaled_dot_product_attention(
+            query_states.transpose(1, 2),
+            key_states.transpose(1, 2),
+            value_states.transpose(1, 2),
+            attn_mask=attention_bias,
+            dropout_p=self.dropout.p if self.training else 0.0,
+        )
+        attn_output = attn_output.permute(0, 2, 1, 3).contiguous()
+        return attn_output, None
+
+
+NEW_ATTENTION_CLASSES = {
+    "eager": NewAttention,
+    # "flash_attention_2": ,  # TODO
+    "sdpa": NewSdpaAttention,
+}
+
+
+class NewGatedMLP(nn.Module):
+    """
+    GLU Variants Improve Transformer.
+    """
+
+    def __init__(self, config: NewConfig):
+        super().__init__()
+        self.intermediate_size = config.intermediate_size
+        self.up_gate_proj = nn.Linear(config.hidden_size, self.intermediate_size * 2, bias=False)
+        self.down_proj = nn.Linear(self.intermediate_size, config.hidden_size, bias=True)
+        self.act_fn = ACT2FN[config.hidden_act]
+        if config.hidden_dropout_prob > 0:
+            self.hidden_dropout = nn.Dropout(config.hidden_dropout_prob)
+        else:
+            self.hidden_dropout = None
+
+    def forward(self, hidden_states):
+        up_gate = self.up_gate_proj(hidden_states)
+        up_states, gate = torch.split(up_gate, self.intermediate_size, dim=-1)
+        gate = self.act_fn(gate)
+        gated_states = gate * up_states
+        if self.hidden_dropout is not None:
+            gated_states = self.hidden_dropout(gated_states)
+        down_states = self.down_proj(gated_states)
+        return down_states
+
+
+class NewLayer(nn.Module):
+    def __init__(
+        self,
+        config: NewConfig,
+        pack_qkv=None,
+        use_memory_efficient_attention=None,
+        attn_implementation=None
+    ):
+        super().__init__()
+        if attn_implementation is None:
+            attn_implementation = config._attn_implementation
+        if use_memory_efficient_attention is None:
+            use_memory_efficient_attention = config.use_memory_efficient_attention
+        if use_memory_efficient_attention:
+            if attn_implementation != 'eager':
+                logger.warning_once(f"Override {attn_implementation=} to 'eager' as {use_memory_efficient_attention=}")
+                attn_implementation = 'eager'  # Since it will be SDPA by default for torch>=2.1.1
+        self.attention = NEW_ATTENTION_CLASSES[attn_implementation](
+            config, pack_qkv=pack_qkv, use_memory_efficient_attention=use_memory_efficient_attention
+        )
+        self.mlp = NewGatedMLP(config)
+
+        ln_class = LAYER_NORM[config.layer_norm_type]
+        self.attn_ln = ln_class(config.hidden_size, eps=config.layer_norm_eps)
+        self.mlp_ln = ln_class(config.hidden_size, eps=config.layer_norm_eps)
+
+        if config.hidden_dropout_prob > 0:
+            self.hidden_dropout = nn.Dropout(config.hidden_dropout_prob)
+        else:
+            self.hidden_dropout = None
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_bias: torch.FloatTensor,
+        rope_embeds: Optional[Tuple[torch.FloatTensor, torch.FloatTensor]] = None,
+        padding_inputs: Optional[Tuple] = None,  # indices, batch, seqlen
+        attention_scale: Optional[torch.FloatTensor] = None,
+        subset_indices: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        output_attentions: Optional[bool] = False,
+        qkv_inputs: Optional[Tuple] = None,  # For RetroMAE
+    ) -> Tuple[torch.Tensor, ...]:
+        # Multi head self attention
+        residual = hidden_states if qkv_inputs is None else qkv_inputs[0]
+        attention_outputs = self.attention(
+            hidden_states,
+            attention_bias,
+            rope_embeds,
+            padding_inputs,
+            attention_scale,
+            head_mask,
+            output_attentions=output_attentions,
+            qkv_inputs=qkv_inputs,
+        )
+        hidden_states = attention_outputs[0]
+        if self.hidden_dropout is not None:
+            hidden_states = self.hidden_dropout(hidden_states)
+        hidden_states = residual + hidden_states
+
+        # In pretraining, after the attention of last layer, we only need the masked tokens.
+        if subset_indices is not None:
+            hidden_states = hidden_states[subset_indices]
+
+        hidden_states = self.attn_ln(hidden_states)
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.mlp(hidden_states)
+        if self.hidden_dropout is not None:
+            hidden_states = self.hidden_dropout(hidden_states)
+        hidden_states = residual + hidden_states
+        hidden_states = self.mlp_ln(hidden_states)
+
+        # add self attentions if we output attention weights
+        outputs = (hidden_states,) + attention_outputs[1:]
+        return outputs
+
+
+class NewEncoder(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.layer = nn.ModuleList([NewLayer(config) for _ in range(config.num_hidden_layers)])
+        self.gradient_checkpointing = False
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_bias: Optional[torch.FloatTensor] = None,
+        rope_embeds: Optional[Tuple[torch.FloatTensor, torch.FloatTensor]] = None,
+        padding_inputs: Optional[Tuple] = None,  # indices, batch, seqlen
+        attention_scale: Optional[torch.FloatTensor] = None,
+        subset_indices: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        output_attentions: Optional[bool] = False,
+        output_hidden_states: Optional[bool] = False,
+        return_dict: Optional[bool] = True,
+    ) -> Union[Tuple[torch.Tensor], BaseModelOutput]:
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attentions = () if output_attentions else None
+
+        for i, layer_module in enumerate(self.layer):
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            if i >= len(self.layer) - 1:
+                layer_subset_indices = subset_indices
+            else:
+                layer_subset_indices = None
+
+            layer_head_mask = head_mask[i] if head_mask is not None else None
+
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    layer_module.__call__,
+                    hidden_states,
+                    attention_bias,
+                    rope_embeds,
+                    padding_inputs,
+                    attention_scale,
+                    layer_subset_indices,
+                    layer_head_mask,
+                )
+            else:
+                layer_outputs = layer_module(
+                    hidden_states,
+                    attention_bias,
+                    rope_embeds,
+                    padding_inputs,
+                    attention_scale,
+                    layer_subset_indices,
+                    layer_head_mask,
+                    output_attentions,
+                )
+
+            hidden_states = layer_outputs[0]
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (layer_outputs[1],)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [
+                    hidden_states,
+                    all_hidden_states,
+                    all_self_attentions,
+                ]
+                if v is not None
+            )
+        return BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+        )
+
+
+# Copied from transformers.models.bert.modeling_bert.BertPooler with Bert->New
+class NewPooler(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.activation = nn.Tanh()
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        # We "pool" the model by simply taking the hidden state corresponding
+        # to the first token.
+        first_token_tensor = hidden_states[:, 0]
+        pooled_output = self.dense(first_token_tensor)
+        pooled_output = self.activation(pooled_output)
+        return pooled_output
+
+
+class NewPreTrainedModel(PreTrainedModel):
+    """
+    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+    models.
+    """
+
+    config_class = NewConfig
+    base_model_prefix = "new"
+    supports_gradient_checkpointing = True
+    _supports_sdpa = True
+
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        if isinstance(module, nn.Linear):
+            # Slightly different from the TF version which uses truncated_normal for initialization
+            # cf https://github.com/pytorch/pytorch/pull/5617
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+
+
+class NewModel(NewPreTrainedModel):
+    """
+    The bare New Model transformer outputting raw hidden-states without any specific head on top.
+    """
+
+    def __init__(self, config: NewConfig, add_pooling_layer=False):
+        super().__init__(config)
+        self.config = config
+
+        self.embeddings = NewEmbeddings(config)
+        self.encoder = NewEncoder(config)
+
+        self.pooler = NewPooler(config) if add_pooling_layer else None
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embeddings.word_embeddings
+
+    def set_input_embeddings(self, value):
+        self.embeddings.word_embeddings = value
+
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        length: Optional[List[int]] = None,
+        subset_indices: Optional[torch.LongTensor] = None,
+        token_type_ids: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        head_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        unpad_inputs: Optional[bool] = None,
+    ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPooling]:
+        r"""
+        length  (`list` of length `batch_size`, *optional*):
+            If is `None`, return padded `last_hidden_state`.
+        subset_indices  ():
+            pass
+        unpad_inputs  (`bool`, *optional*):
+            pass
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        unpad_inputs = unpad_inputs if unpad_inputs is not None else self.config.unpad_inputs
+        output_padded = length is None
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
+            input_shape = input_ids.size()
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        # TODO: not used
+        # # Prepare head mask if needed
+        # # 1.0 in head_mask indicate we keep the head
+        # # attention_probs has shape bsz x n_heads x N x N
+        # # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
+        # # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
+        # head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
+
+        # Get embeddings, may unpad them
+        (embedding_output, attention_mask, rope_embeds, length) = self.embeddings(
+            unpad_inputs,
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            length=length,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds
+        )
+
+        batch_size, seq_length = input_shape
+        if unpad_inputs and self.config.use_memory_efficient_attention:
+            attention_bias = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens(length)
+        else:
+            # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+            # ourselves in which case we just need to make it broadcastable to all heads.
+            attention_bias = self.get_extended_attention_mask(attention_mask, input_shape)
+            if self.config.use_memory_efficient_attention:
+                # Invalid shape for attention bias: torch.Size([48, 1, 1, 512]) (expected (48, 12, 512, 512))
+                attention_bias = attention_bias.expand(-1, self.config.num_attention_heads, seq_length, -1)
+
+        padding_inputs = None
+        if unpad_inputs and (output_padded or not self.config.use_memory_efficient_attention):
+            indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+            if not self.config.use_memory_efficient_attention:
+                padding_inputs = (indices, *input_shape)
+
+        attention_scale = None
+        if self.config.logn_attention_scale:
+            logger.warning_once("TODO: logn_attention_scale")
+        #     # attention scale log_512(input_len)
+        #     attention_scale = attention_mask.sum(1).log() / torch.tensor(self.config.max_position_embeddings).log()
+        #     # inference-time logn scale need clip 1
+        #     if self.config.logn_attention_clip1:
+        #         attention_scale.clip_(1)
+        #     attention_scale = attention_scale[:, None, None, None]
+        # else:
+        #     attention_scale = None
+
+        encoder_outputs = self.encoder(
+            embedding_output,
+            attention_bias=attention_bias,
+            rope_embeds=rope_embeds,
+            padding_inputs=padding_inputs,
+            attention_scale=attention_scale,
+            subset_indices=subset_indices,
+            head_mask=head_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        sequence_output = encoder_outputs[0]
+        if unpad_inputs and output_padded:
+            sequence_output = pad_input(
+                sequence_output.squeeze(), indices, batch_size, seq_length
+            )
+
+        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
+
+        if not return_dict:
+            return (sequence_output, pooled_output) + encoder_outputs[1:]
+
+        return BaseModelOutputWithPooling(
+            last_hidden_state=sequence_output,
+            pooler_output=pooled_output,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )
+
+
+class NewLMPredictionHead(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.transform_act_fn = ACT2FN[config.hidden_act]
+        self.norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+        # The output weights are the same as the input embeddings, but there is
+        # an output-only bias for each token.
+        self.decoder = nn.Linear(config.hidden_size, config.vocab_size)
+
+    def forward(self, hidden_states):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.transform_act_fn(hidden_states)
+        hidden_states = self.norm(hidden_states)
+        hidden_states = self.decoder(hidden_states)
+        return hidden_states
+
+
+class NewForMaskedLM(NewPreTrainedModel):
+    _tied_weights_keys = ["lm_head.decoder.bias", "lm_head.decoder.weight"]
+
+    def __init__(self, config: NewConfig):
+        super().__init__(config)
+        self.new = NewModel(config, add_pooling_layer=False)
+        self.lm_head = NewLMPredictionHead(config)
+        self.loss_fct = nn.CrossEntropyLoss()
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_output_embeddings(self):
+        return self.lm_head.decoder
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head.decoder = new_embeddings
+
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        token_type_ids: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        head_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        labels: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        unpad_inputs: Optional[bool] = None,
+    ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+        """
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if labels is None or not self.new.config.unpad_inputs:
+            length = None
+            subset_indices = None
+        else:
+            length = attention_mask.sum(-1).tolist()
+            labels = labels[attention_mask.bool()].unsqueeze(0)
+            subset_indices = labels > -100
+
+        outputs = self.new(
+            input_ids,
+            attention_mask=attention_mask,
+            length=length,
+            subset_indices=subset_indices,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            unpad_inputs=unpad_inputs,
+        )
+
+        sequence_output = outputs[0]
+        prediction_scores = self.lm_head(sequence_output)
+
+        masked_lm_loss = None
+        if labels is not None:
+            if subset_indices is None:
+                mask = attention_mask.bool()
+                prediction_scores = prediction_scores[mask]
+                labels = labels[mask]
+            else:
+                labels = labels[subset_indices]
+            masked_lm_loss = self.loss_fct(prediction_scores, labels)
+
+        if not return_dict:
+            output = (prediction_scores,) + outputs[2:]
+            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output
+
+        return MaskedLMOutput(
+            loss=masked_lm_loss,
+            logits=prediction_scores,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+class NewForSequenceClassification(NewPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.config = config
+
+        self.new = NewModel(config, add_pooling_layer=True)
+        classifier_dropout = (
+            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
+        )
+        self.dropout = nn.Dropout(classifier_dropout)
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        token_type_ids: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        head_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        labels: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        unpad_inputs: Optional[bool] = None,
+    ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        outputs = self.new(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            unpad_inputs=unpad_inputs,
+        )
+
+        pooled_output = outputs[1]
+
+        pooled_output = self.dropout(pooled_output)
+        logits = self.classifier(pooled_output)
+
+        loss = None
+        if labels is not None:
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = nn.MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = nn.CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = nn.BCEWithLogitsLoss()
+                loss = loss_fct(logits, labels)
+
+        if not return_dict:
+            output = (logits,) + outputs[2:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+class NewForMultipleChoice(NewPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.new = NewModel(config, add_pooling_layer=True)
+        classifier_dropout = (
+            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
+        )
+        self.dropout = nn.Dropout(classifier_dropout)
+        self.classifier = nn.Linear(config.hidden_size, 1)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        token_type_ids: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        head_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        labels: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        unpad_inputs: Optional[bool] = None,
+    ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+            `input_ids` above)
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]
+
+        input_ids = input_ids.view(-1, input_ids.size(-1)) if input_ids is not None else None
+        attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None
+        token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None
+        position_ids = position_ids.view(-1, position_ids.size(-1)) if position_ids is not None else None
+        inputs_embeds = (
+            inputs_embeds.view(-1, inputs_embeds.size(-2), inputs_embeds.size(-1))
+            if inputs_embeds is not None
+            else None
+        )
+
+        outputs = self.new(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            unpad_inputs=unpad_inputs,
+        )
+
+        pooled_output = outputs[1]
+
+        pooled_output = self.dropout(pooled_output)
+        logits = self.classifier(pooled_output)
+        reshaped_logits = logits.view(-1, num_choices)
+
+        loss = None
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(reshaped_logits, labels)
+
+        if not return_dict:
+            output = (reshaped_logits,) + outputs[2:]
+            return ((loss,) + output) if loss is not None else output
+
+        return MultipleChoiceModelOutput(
+            loss=loss,
+            logits=reshaped_logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@dataclass
+class NewTokenClassifierOutput(ModelOutput):
+    loss: Optional[torch.FloatTensor] = None
+    logits: torch.FloatTensor = None
+    last_hidden_state: torch.FloatTensor = None
+    hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
+
+
+class NewForTokenClassification(NewPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+
+        self.new = NewModel(config, add_pooling_layer=False)
+        classifier_dropout = (
+            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
+        )
+        self.dropout = nn.Dropout(classifier_dropout)
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        token_type_ids: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        head_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        labels: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        unpad_inputs: Optional[bool] = None,
+    ) -> Union[Tuple[torch.Tensor], NewTokenClassifierOutput]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        outputs = self.new(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            unpad_inputs=unpad_inputs,
+        )
+
+        sequence_output = outputs[0]
+
+        sequence_output = self.dropout(sequence_output)
+        logits = self.classifier(sequence_output)
+
+        loss = None
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+
+        if not return_dict:
+            output = (logits,) + outputs[2:]
+            return ((loss,) + output) if loss is not None else output
+
+        return NewTokenClassifierOutput(
+            loss=loss,
+            logits=logits,
+            last_hidden_state=sequence_output,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+class NewForQuestionAnswering(NewPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+
+        self.new = NewModel(config, add_pooling_layer=False)
+        self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        token_type_ids: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        head_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        start_positions: Optional[torch.Tensor] = None,
+        end_positions: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        unpad_inputs: Optional[bool] = None,
+    ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]:
+        r"""
+        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for position (index) of the start of the labelled span for computing the token classification loss.
+            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+            are not taken into account for computing the loss.
+        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for position (index) of the end of the labelled span for computing the token classification loss.
+            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+            are not taken into account for computing the loss.
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        outputs = self.new(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            unpad_inputs=unpad_inputs,
+        )
+
+        sequence_output = outputs[0]
+
+        logits = self.qa_outputs(sequence_output)
+        start_logits, end_logits = logits.split(1, dim=-1)
+        start_logits = start_logits.squeeze(-1).contiguous()
+        end_logits = end_logits.squeeze(-1).contiguous()
+
+        total_loss = None
+        if start_positions is not None and end_positions is not None:
+            # If we are on multi-GPU, split add a dimension
+            if len(start_positions.size()) > 1:
+                start_positions = start_positions.squeeze(-1)
+            if len(end_positions.size()) > 1:
+                end_positions = end_positions.squeeze(-1)
+            # sometimes the start/end positions are outside our model inputs, we ignore these terms
+            ignored_index = start_logits.size(1)
+            start_positions = start_positions.clamp(0, ignored_index)
+            end_positions = end_positions.clamp(0, ignored_index)
+
+            loss_fct = nn.CrossEntropyLoss(ignore_index=ignored_index)
+            start_loss = loss_fct(start_logits, start_positions)
+            end_loss = loss_fct(end_logits, end_positions)
+            total_loss = (start_loss + end_loss) / 2
+
+        if not return_dict:
+            output = (start_logits, end_logits) + outputs[2:]
+            return ((total_loss,) + output) if total_loss is not None else output
+
+        return QuestionAnsweringModelOutput(
+            loss=total_loss,
+            start_logits=start_logits,
+            end_logits=end_logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )

checkpoint-200/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:491b54b4d910a39ffed7cc8b9a1a7522a2d46cb24fde3b6e4ea9e59e2b769678
+size 15984

checkpoint-200/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb57c59b1177680843a1b2932345a1bfdb11f77e64fae2e5464422d422b59fae
+size 15984

checkpoint-200/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13de24762cc88e04c2704b85d4e642bcf9031ef34e7817506fbb495c62e467f8
+size 15984

checkpoint-200/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:888d6550bcdbac3947838d60366b2f5873011d3b7f300f87f38058d7845e6a2e
+size 15920

checkpoint-200/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09dfc7f8fde7376ae2317747d57da83ba9ac4b7494b06444f753f2fbaddee07a
+size 15920

checkpoint-200/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ada59ce425718dd1feb844fb76799ec8199471c2f84b4055764e543aa59b6d03
+size 15920

checkpoint-200/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22e7d80046ff31c27340393abfc26100bdcecfb1edb86eeaf7c87625008ca77b
+size 15984

checkpoint-200/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e197de07e3bd0af6fddcdbccbafb25a8b6175280fbc1e2b113d3bc50508a9fe
+size 15920

checkpoint-200/special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-200/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b11d4b2e33aa47529535e2aa477b7bae26b5490e6bc956fa46422dfefd4b9366
+size 17082833

checkpoint-200/tokenizer_config.json

@@ -0,0 +1,61 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "max_length": 512,
+  "model_max_length": 32768,
+  "pad_to_multiple_of": null,
+  "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "</s>",
+  "stride": 0,
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<unk>"
+}

checkpoint-200/trainer_state.json

@@ -0,0 +1,1433 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 19.96879875195008,
+  "eval_steps": 500,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0998439937597504,
+      "grad_norm": 0.4140388733942774,
+      "learning_rate": 0.0,
+      "loss": 1.9149,
+      "step": 1
+    },
+    {
+      "epoch": 0.1996879875195008,
+      "grad_norm": 4.125320422125423,
+      "learning_rate": 0.0001,
+      "loss": 2.0842,
+      "step": 2
+    },
+    {
+      "epoch": 0.2995319812792512,
+      "grad_norm": 4.114561571796027,
+      "learning_rate": 0.0001,
+      "loss": 2.0845,
+      "step": 3
+    },
+    {
+      "epoch": 0.3993759750390016,
+      "grad_norm": 7.814135082802521,
+      "learning_rate": 9.94949494949495e-05,
+      "loss": 2.2529,
+      "step": 4
+    },
+    {
+      "epoch": 0.49921996879875197,
+      "grad_norm": 4.959578007928057,
+      "learning_rate": 9.8989898989899e-05,
+      "loss": 2.1802,
+      "step": 5
+    },
+    {
+      "epoch": 0.5990639625585024,
+      "grad_norm": 4.020980884960991,
+      "learning_rate": 9.848484848484849e-05,
+      "loss": 2.0996,
+      "step": 6
+    },
+    {
+      "epoch": 0.6989079563182528,
+      "grad_norm": 2.913148956691379,
+      "learning_rate": 9.797979797979798e-05,
+      "loss": 2.0739,
+      "step": 7
+    },
+    {
+      "epoch": 0.7987519500780031,
+      "grad_norm": 1.5904378694728827,
+      "learning_rate": 9.747474747474747e-05,
+      "loss": 2.0438,
+      "step": 8
+    },
+    {
+      "epoch": 0.8985959438377535,
+      "grad_norm": 1.1120971271683253,
+      "learning_rate": 9.696969696969698e-05,
+      "loss": 2.023,
+      "step": 9
+    },
+    {
+      "epoch": 0.9984399375975039,
+      "grad_norm": 1.4857926676190432,
+      "learning_rate": 9.646464646464647e-05,
+      "loss": 2.0256,
+      "step": 10
+    },
+    {
+      "epoch": 1.0982839313572543,
+      "grad_norm": 1.7514837650567026,
+      "learning_rate": 9.595959595959596e-05,
+      "loss": 2.0297,
+      "step": 11
+    },
+    {
+      "epoch": 1.1981279251170047,
+      "grad_norm": 1.3684063083659384,
+      "learning_rate": 9.545454545454546e-05,
+      "loss": 1.9984,
+      "step": 12
+    },
+    {
+      "epoch": 1.2979719188767551,
+      "grad_norm": 0.8391319984906789,
+      "learning_rate": 9.494949494949495e-05,
+      "loss": 1.9938,
+      "step": 13
+    },
+    {
+      "epoch": 1.3978159126365055,
+      "grad_norm": 0.7680152250335479,
+      "learning_rate": 9.444444444444444e-05,
+      "loss": 1.9984,
+      "step": 14
+    },
+    {
+      "epoch": 1.497659906396256,
+      "grad_norm": 1.1427900590537006,
+      "learning_rate": 9.393939393939395e-05,
+      "loss": 2.0059,
+      "step": 15
+    },
+    {
+      "epoch": 1.5975039001560063,
+      "grad_norm": 1.2852588832884364,
+      "learning_rate": 9.343434343434344e-05,
+      "loss": 2.0057,
+      "step": 16
+    },
+    {
+      "epoch": 1.6973478939157567,
+      "grad_norm": 0.8509981577726656,
+      "learning_rate": 9.292929292929293e-05,
+      "loss": 1.97,
+      "step": 17
+    },
+    {
+      "epoch": 1.797191887675507,
+      "grad_norm": 0.4374249257660765,
+      "learning_rate": 9.242424242424242e-05,
+      "loss": 1.9852,
+      "step": 18
+    },
+    {
+      "epoch": 1.8970358814352575,
+      "grad_norm": 1.006945747433108,
+      "learning_rate": 9.191919191919192e-05,
+      "loss": 1.9736,
+      "step": 19
+    },
+    {
+      "epoch": 1.9968798751950079,
+      "grad_norm": 1.1714326150671521,
+      "learning_rate": 9.141414141414141e-05,
+      "loss": 1.9866,
+      "step": 20
+    },
+    {
+      "epoch": 2.0967238689547583,
+      "grad_norm": 0.6697915843016325,
+      "learning_rate": 9.090909090909092e-05,
+      "loss": 1.9669,
+      "step": 21
+    },
+    {
+      "epoch": 2.1965678627145087,
+      "grad_norm": 0.43542954442572934,
+      "learning_rate": 9.040404040404041e-05,
+      "loss": 1.9596,
+      "step": 22
+    },
+    {
+      "epoch": 2.296411856474259,
+      "grad_norm": 0.8895989581186896,
+      "learning_rate": 8.98989898989899e-05,
+      "loss": 1.9777,
+      "step": 23
+    },
+    {
+      "epoch": 2.3962558502340094,
+      "grad_norm": 0.748473401890919,
+      "learning_rate": 8.93939393939394e-05,
+      "loss": 1.9828,
+      "step": 24
+    },
+    {
+      "epoch": 2.49609984399376,
+      "grad_norm": 0.4762840239068188,
+      "learning_rate": 8.888888888888889e-05,
+      "loss": 1.9863,
+      "step": 25
+    },
+    {
+      "epoch": 2.5959438377535102,
+      "grad_norm": 0.4634914120924797,
+      "learning_rate": 8.83838383838384e-05,
+      "loss": 1.9728,
+      "step": 26
+    },
+    {
+      "epoch": 2.6957878315132606,
+      "grad_norm": 0.576721007312459,
+      "learning_rate": 8.787878787878789e-05,
+      "loss": 1.9813,
+      "step": 27
+    },
+    {
+      "epoch": 2.795631825273011,
+      "grad_norm": 0.4717088615288276,
+      "learning_rate": 8.737373737373738e-05,
+      "loss": 1.9709,
+      "step": 28
+    },
+    {
+      "epoch": 2.8954758190327614,
+      "grad_norm": 0.5243076095653101,
+      "learning_rate": 8.686868686868688e-05,
+      "loss": 1.9889,
+      "step": 29
+    },
+    {
+      "epoch": 2.995319812792512,
+      "grad_norm": 0.35563844256479116,
+      "learning_rate": 8.636363636363637e-05,
+      "loss": 1.969,
+      "step": 30
+    },
+    {
+      "epoch": 3.095163806552262,
+      "grad_norm": 0.5040313506272054,
+      "learning_rate": 8.585858585858586e-05,
+      "loss": 1.9701,
+      "step": 31
+    },
+    {
+      "epoch": 3.1950078003120126,
+      "grad_norm": 0.5293887294443628,
+      "learning_rate": 8.535353535353535e-05,
+      "loss": 1.9774,
+      "step": 32
+    },
+    {
+      "epoch": 3.294851794071763,
+      "grad_norm": 0.33336016676106733,
+      "learning_rate": 8.484848484848486e-05,
+      "loss": 1.9702,
+      "step": 33
+    },
+    {
+      "epoch": 3.3946957878315134,
+      "grad_norm": 0.5156182664373749,
+      "learning_rate": 8.434343434343435e-05,
+      "loss": 1.9552,
+      "step": 34
+    },
+    {
+      "epoch": 3.4945397815912638,
+      "grad_norm": 0.410792592829029,
+      "learning_rate": 8.383838383838384e-05,
+      "loss": 1.9642,
+      "step": 35
+    },
+    {
+      "epoch": 3.594383775351014,
+      "grad_norm": 0.40267682408922495,
+      "learning_rate": 8.333333333333334e-05,
+      "loss": 1.9688,
+      "step": 36
+    },
+    {
+      "epoch": 3.6942277691107646,
+      "grad_norm": 0.3869359148412346,
+      "learning_rate": 8.282828282828283e-05,
+      "loss": 1.9733,
+      "step": 37
+    },
+    {
+      "epoch": 3.794071762870515,
+      "grad_norm": 0.37728712869432585,
+      "learning_rate": 8.232323232323233e-05,
+      "loss": 1.9648,
+      "step": 38
+    },
+    {
+      "epoch": 3.8939157566302653,
+      "grad_norm": 0.3922418131207954,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 1.9689,
+      "step": 39
+    },
+    {
+      "epoch": 3.9937597503900157,
+      "grad_norm": 0.26353046722639645,
+      "learning_rate": 8.131313131313132e-05,
+      "loss": 1.9727,
+      "step": 40
+    },
+    {
+      "epoch": 4.093603744149766,
+      "grad_norm": 0.3911091474488452,
+      "learning_rate": 8.080808080808081e-05,
+      "loss": 1.9631,
+      "step": 41
+    },
+    {
+      "epoch": 4.1934477379095165,
+      "grad_norm": 0.33402240826623614,
+      "learning_rate": 8.03030303030303e-05,
+      "loss": 1.9665,
+      "step": 42
+    },
+    {
+      "epoch": 4.2932917316692665,
+      "grad_norm": 0.34654808232868395,
+      "learning_rate": 7.97979797979798e-05,
+      "loss": 1.9646,
+      "step": 43
+    },
+    {
+      "epoch": 4.393135725429017,
+      "grad_norm": 0.3031078864703629,
+      "learning_rate": 7.92929292929293e-05,
+      "loss": 1.9693,
+      "step": 44
+    },
+    {
+      "epoch": 4.492979719188767,
+      "grad_norm": 0.35342072957234116,
+      "learning_rate": 7.878787878787879e-05,
+      "loss": 1.9688,
+      "step": 45
+    },
+    {
+      "epoch": 4.592823712948518,
+      "grad_norm": 0.3918161921811716,
+      "learning_rate": 7.828282828282829e-05,
+      "loss": 1.9609,
+      "step": 46
+    },
+    {
+      "epoch": 4.692667706708268,
+      "grad_norm": 0.24995683506017796,
+      "learning_rate": 7.777777777777778e-05,
+      "loss": 1.9515,
+      "step": 47
+    },
+    {
+      "epoch": 4.792511700468019,
+      "grad_norm": 0.3308078104166398,
+      "learning_rate": 7.727272727272727e-05,
+      "loss": 1.9607,
+      "step": 48
+    },
+    {
+      "epoch": 4.892355694227769,
+      "grad_norm": 0.3130926472973521,
+      "learning_rate": 7.676767676767676e-05,
+      "loss": 1.9699,
+      "step": 49
+    },
+    {
+      "epoch": 4.99219968798752,
+      "grad_norm": 0.30892356920484393,
+      "learning_rate": 7.626262626262627e-05,
+      "loss": 1.9645,
+      "step": 50
+    },
+    {
+      "epoch": 5.09204368174727,
+      "grad_norm": 0.2804202715276883,
+      "learning_rate": 7.575757575757576e-05,
+      "loss": 1.9569,
+      "step": 51
+    },
+    {
+      "epoch": 5.1918876755070205,
+      "grad_norm": 0.2789049399636327,
+      "learning_rate": 7.525252525252525e-05,
+      "loss": 1.9585,
+      "step": 52
+    },
+    {
+      "epoch": 5.29173166926677,
+      "grad_norm": 0.2906929505804403,
+      "learning_rate": 7.474747474747475e-05,
+      "loss": 1.9565,
+      "step": 53
+    },
+    {
+      "epoch": 5.391575663026521,
+      "grad_norm": 0.2033727950080347,
+      "learning_rate": 7.424242424242424e-05,
+      "loss": 1.9755,
+      "step": 54
+    },
+    {
+      "epoch": 5.491419656786271,
+      "grad_norm": 0.31364461369416163,
+      "learning_rate": 7.373737373737373e-05,
+      "loss": 1.9647,
+      "step": 55
+    },
+    {
+      "epoch": 5.591263650546022,
+      "grad_norm": 0.2531087381531638,
+      "learning_rate": 7.323232323232324e-05,
+      "loss": 1.9578,
+      "step": 56
+    },
+    {
+      "epoch": 5.691107644305772,
+      "grad_norm": 0.23764498764830225,
+      "learning_rate": 7.272727272727273e-05,
+      "loss": 1.9617,
+      "step": 57
+    },
+    {
+      "epoch": 5.790951638065523,
+      "grad_norm": 0.24888591334854687,
+      "learning_rate": 7.222222222222222e-05,
+      "loss": 1.963,
+      "step": 58
+    },
+    {
+      "epoch": 5.890795631825273,
+      "grad_norm": 0.2647075657405339,
+      "learning_rate": 7.171717171717171e-05,
+      "loss": 1.9685,
+      "step": 59
+    },
+    {
+      "epoch": 5.990639625585024,
+      "grad_norm": 0.27820470985704615,
+      "learning_rate": 7.121212121212121e-05,
+      "loss": 1.9654,
+      "step": 60
+    },
+    {
+      "epoch": 6.090483619344774,
+      "grad_norm": 0.20068946885468097,
+      "learning_rate": 7.07070707070707e-05,
+      "loss": 1.9667,
+      "step": 61
+    },
+    {
+      "epoch": 6.190327613104524,
+      "grad_norm": 0.25026234630326394,
+      "learning_rate": 7.020202020202021e-05,
+      "loss": 1.9542,
+      "step": 62
+    },
+    {
+      "epoch": 6.290171606864274,
+      "grad_norm": 0.22856925269883635,
+      "learning_rate": 6.96969696969697e-05,
+      "loss": 1.9573,
+      "step": 63
+    },
+    {
+      "epoch": 6.390015600624025,
+      "grad_norm": 0.2392183076591563,
+      "learning_rate": 6.91919191919192e-05,
+      "loss": 1.9647,
+      "step": 64
+    },
+    {
+      "epoch": 6.489859594383775,
+      "grad_norm": 0.20384525102843132,
+      "learning_rate": 6.86868686868687e-05,
+      "loss": 1.9628,
+      "step": 65
+    },
+    {
+      "epoch": 6.589703588143526,
+      "grad_norm": 0.23941897200051984,
+      "learning_rate": 6.818181818181818e-05,
+      "loss": 1.9667,
+      "step": 66
+    },
+    {
+      "epoch": 6.689547581903276,
+      "grad_norm": 0.20375278551444306,
+      "learning_rate": 6.767676767676769e-05,
+      "loss": 1.9572,
+      "step": 67
+    },
+    {
+      "epoch": 6.789391575663027,
+      "grad_norm": 0.20727005267599333,
+      "learning_rate": 6.717171717171718e-05,
+      "loss": 1.9581,
+      "step": 68
+    },
+    {
+      "epoch": 6.889235569422777,
+      "grad_norm": 0.22300809533132504,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 1.9693,
+      "step": 69
+    },
+    {
+      "epoch": 6.9890795631825275,
+      "grad_norm": 0.21742318730398613,
+      "learning_rate": 6.616161616161617e-05,
+      "loss": 1.9656,
+      "step": 70
+    },
+    {
+      "epoch": 7.0889235569422775,
+      "grad_norm": 0.20343822391223,
+      "learning_rate": 6.565656565656566e-05,
+      "loss": 1.9656,
+      "step": 71
+    },
+    {
+      "epoch": 7.188767550702028,
+      "grad_norm": 0.2364200066637671,
+      "learning_rate": 6.515151515151516e-05,
+      "loss": 1.95,
+      "step": 72
+    },
+    {
+      "epoch": 7.288611544461778,
+      "grad_norm": 0.18261048615751524,
+      "learning_rate": 6.464646464646466e-05,
+      "loss": 1.9538,
+      "step": 73
+    },
+    {
+      "epoch": 7.388455538221529,
+      "grad_norm": 0.24533487813163474,
+      "learning_rate": 6.414141414141415e-05,
+      "loss": 1.952,
+      "step": 74
+    },
+    {
+      "epoch": 7.488299531981279,
+      "grad_norm": 0.2539612930496735,
+      "learning_rate": 6.363636363636364e-05,
+      "loss": 1.9598,
+      "step": 75
+    },
+    {
+      "epoch": 7.58814352574103,
+      "grad_norm": 0.2991457603613546,
+      "learning_rate": 6.313131313131313e-05,
+      "loss": 1.9733,
+      "step": 76
+    },
+    {
+      "epoch": 7.68798751950078,
+      "grad_norm": 0.2209105824741669,
+      "learning_rate": 6.262626262626264e-05,
+      "loss": 1.9531,
+      "step": 77
+    },
+    {
+      "epoch": 7.787831513260531,
+      "grad_norm": 0.31698310918964695,
+      "learning_rate": 6.212121212121213e-05,
+      "loss": 1.964,
+      "step": 78
+    },
+    {
+      "epoch": 7.887675507020281,
+      "grad_norm": 0.17584276182725114,
+      "learning_rate": 6.161616161616162e-05,
+      "loss": 1.9644,
+      "step": 79
+    },
+    {
+      "epoch": 7.9875195007800315,
+      "grad_norm": 0.29996919622824225,
+      "learning_rate": 6.111111111111112e-05,
+      "loss": 1.9555,
+      "step": 80
+    },
+    {
+      "epoch": 8.087363494539781,
+      "grad_norm": 0.2406502202676367,
+      "learning_rate": 6.060606060606061e-05,
+      "loss": 1.9554,
+      "step": 81
+    },
+    {
+      "epoch": 8.187207488299531,
+      "grad_norm": 0.32705142732170855,
+      "learning_rate": 6.01010101010101e-05,
+      "loss": 1.9517,
+      "step": 82
+    },
+    {
+      "epoch": 8.287051482059283,
+      "grad_norm": 0.27249925952338305,
+      "learning_rate": 5.959595959595959e-05,
+      "loss": 1.9474,
+      "step": 83
+    },
+    {
+      "epoch": 8.386895475819033,
+      "grad_norm": 0.29448831027669287,
+      "learning_rate": 5.90909090909091e-05,
+      "loss": 1.9459,
+      "step": 84
+    },
+    {
+      "epoch": 8.486739469578783,
+      "grad_norm": 0.29998154037028857,
+      "learning_rate": 5.858585858585859e-05,
+      "loss": 1.9606,
+      "step": 85
+    },
+    {
+      "epoch": 8.586583463338533,
+      "grad_norm": 0.23153724936859055,
+      "learning_rate": 5.808080808080808e-05,
+      "loss": 1.9598,
+      "step": 86
+    },
+    {
+      "epoch": 8.686427457098285,
+      "grad_norm": 0.22081595887056477,
+      "learning_rate": 5.757575757575758e-05,
+      "loss": 1.9586,
+      "step": 87
+    },
+    {
+      "epoch": 8.786271450858035,
+      "grad_norm": 0.19177670537863922,
+      "learning_rate": 5.707070707070707e-05,
+      "loss": 1.9715,
+      "step": 88
+    },
+    {
+      "epoch": 8.886115444617785,
+      "grad_norm": 0.25725928107907137,
+      "learning_rate": 5.6565656565656563e-05,
+      "loss": 1.9602,
+      "step": 89
+    },
+    {
+      "epoch": 8.985959438377535,
+      "grad_norm": 0.26044371305524344,
+      "learning_rate": 5.606060606060606e-05,
+      "loss": 1.9607,
+      "step": 90
+    },
+    {
+      "epoch": 9.085803432137286,
+      "grad_norm": 0.23728151561491595,
+      "learning_rate": 5.555555555555556e-05,
+      "loss": 1.9588,
+      "step": 91
+    },
+    {
+      "epoch": 9.185647425897036,
+      "grad_norm": 0.20354348868729488,
+      "learning_rate": 5.5050505050505056e-05,
+      "loss": 1.9492,
+      "step": 92
+    },
+    {
+      "epoch": 9.285491419656786,
+      "grad_norm": 0.18672087839741056,
+      "learning_rate": 5.4545454545454546e-05,
+      "loss": 1.9457,
+      "step": 93
+    },
+    {
+      "epoch": 9.385335413416536,
+      "grad_norm": 0.1939858201242329,
+      "learning_rate": 5.4040404040404044e-05,
+      "loss": 1.9453,
+      "step": 94
+    },
+    {
+      "epoch": 9.485179407176288,
+      "grad_norm": 0.19172060706771135,
+      "learning_rate": 5.353535353535354e-05,
+      "loss": 1.958,
+      "step": 95
+    },
+    {
+      "epoch": 9.585023400936038,
+      "grad_norm": 0.1837920882880991,
+      "learning_rate": 5.303030303030303e-05,
+      "loss": 1.9577,
+      "step": 96
+    },
+    {
+      "epoch": 9.684867394695788,
+      "grad_norm": 0.2162949878555464,
+      "learning_rate": 5.2525252525252536e-05,
+      "loss": 1.9622,
+      "step": 97
+    },
+    {
+      "epoch": 9.784711388455538,
+      "grad_norm": 0.19325381586186333,
+      "learning_rate": 5.2020202020202026e-05,
+      "loss": 1.9433,
+      "step": 98
+    },
+    {
+      "epoch": 9.88455538221529,
+      "grad_norm": 0.2018142831658023,
+      "learning_rate": 5.151515151515152e-05,
+      "loss": 1.9605,
+      "step": 99
+    },
+    {
+      "epoch": 9.98439937597504,
+      "grad_norm": 0.176671565601027,
+      "learning_rate": 5.101010101010101e-05,
+      "loss": 1.9578,
+      "step": 100
+    },
+    {
+      "epoch": 10.08424336973479,
+      "grad_norm": 0.2117788085352089,
+      "learning_rate": 5.050505050505051e-05,
+      "loss": 1.9478,
+      "step": 101
+    },
+    {
+      "epoch": 10.18408736349454,
+      "grad_norm": 0.1816135304249716,
+      "learning_rate": 5e-05,
+      "loss": 1.9423,
+      "step": 102
+    },
+    {
+      "epoch": 10.283931357254291,
+      "grad_norm": 0.2680310363226074,
+      "learning_rate": 4.94949494949495e-05,
+      "loss": 1.9519,
+      "step": 103
+    },
+    {
+      "epoch": 10.383775351014041,
+      "grad_norm": 0.17934299698555412,
+      "learning_rate": 4.898989898989899e-05,
+      "loss": 1.9625,
+      "step": 104
+    },
+    {
+      "epoch": 10.48361934477379,
+      "grad_norm": 0.19786074542682824,
+      "learning_rate": 4.848484848484849e-05,
+      "loss": 1.95,
+      "step": 105
+    },
+    {
+      "epoch": 10.58346333853354,
+      "grad_norm": 0.17490489580858018,
+      "learning_rate": 4.797979797979798e-05,
+      "loss": 1.9513,
+      "step": 106
+    },
+    {
+      "epoch": 10.683307332293293,
+      "grad_norm": 0.224513887757472,
+      "learning_rate": 4.7474747474747476e-05,
+      "loss": 1.9499,
+      "step": 107
+    },
+    {
+      "epoch": 10.783151326053042,
+      "grad_norm": 0.16993980203530532,
+      "learning_rate": 4.696969696969697e-05,
+      "loss": 1.944,
+      "step": 108
+    },
+    {
+      "epoch": 10.882995319812792,
+      "grad_norm": 0.18436224376063975,
+      "learning_rate": 4.6464646464646464e-05,
+      "loss": 1.9494,
+      "step": 109
+    },
+    {
+      "epoch": 10.982839313572542,
+      "grad_norm": 0.1858801208244774,
+      "learning_rate": 4.595959595959596e-05,
+      "loss": 1.9504,
+      "step": 110
+    },
+    {
+      "epoch": 11.082683307332294,
+      "grad_norm": 0.21397140437157122,
+      "learning_rate": 4.545454545454546e-05,
+      "loss": 1.9467,
+      "step": 111
+    },
+    {
+      "epoch": 11.182527301092044,
+      "grad_norm": 0.16934479906947233,
+      "learning_rate": 4.494949494949495e-05,
+      "loss": 1.9475,
+      "step": 112
+    },
+    {
+      "epoch": 11.282371294851794,
+      "grad_norm": 0.17710883760980722,
+      "learning_rate": 4.4444444444444447e-05,
+      "loss": 1.9418,
+      "step": 113
+    },
+    {
+      "epoch": 11.382215288611544,
+      "grad_norm": 0.2278025006688675,
+      "learning_rate": 4.3939393939393944e-05,
+      "loss": 1.9456,
+      "step": 114
+    },
+    {
+      "epoch": 11.482059282371296,
+      "grad_norm": 0.18727166458531316,
+      "learning_rate": 4.343434343434344e-05,
+      "loss": 1.9408,
+      "step": 115
+    },
+    {
+      "epoch": 11.581903276131046,
+      "grad_norm": 0.17348080665741175,
+      "learning_rate": 4.292929292929293e-05,
+      "loss": 1.9367,
+      "step": 116
+    },
+    {
+      "epoch": 11.681747269890796,
+      "grad_norm": 0.21559975863343248,
+      "learning_rate": 4.242424242424243e-05,
+      "loss": 1.9509,
+      "step": 117
+    },
+    {
+      "epoch": 11.781591263650546,
+      "grad_norm": 0.20515384184563593,
+      "learning_rate": 4.191919191919192e-05,
+      "loss": 1.9503,
+      "step": 118
+    },
+    {
+      "epoch": 11.881435257410295,
+      "grad_norm": 0.17579996751101729,
+      "learning_rate": 4.141414141414142e-05,
+      "loss": 1.9443,
+      "step": 119
+    },
+    {
+      "epoch": 11.981279251170047,
+      "grad_norm": 0.1870399234707776,
+      "learning_rate": 4.0909090909090915e-05,
+      "loss": 1.9507,
+      "step": 120
+    },
+    {
+      "epoch": 12.081123244929797,
+      "grad_norm": 0.2323975590399996,
+      "learning_rate": 4.0404040404040405e-05,
+      "loss": 1.9486,
+      "step": 121
+    },
+    {
+      "epoch": 12.180967238689547,
+      "grad_norm": 0.17332911391024705,
+      "learning_rate": 3.98989898989899e-05,
+      "loss": 1.9441,
+      "step": 122
+    },
+    {
+      "epoch": 12.280811232449299,
+      "grad_norm": 0.23886491083540215,
+      "learning_rate": 3.939393939393939e-05,
+      "loss": 1.9489,
+      "step": 123
+    },
+    {
+      "epoch": 12.380655226209049,
+      "grad_norm": 0.192192583869745,
+      "learning_rate": 3.888888888888889e-05,
+      "loss": 1.936,
+      "step": 124
+    },
+    {
+      "epoch": 12.480499219968799,
+      "grad_norm": 0.24070020033146947,
+      "learning_rate": 3.838383838383838e-05,
+      "loss": 1.9363,
+      "step": 125
+    },
+    {
+      "epoch": 12.580343213728549,
+      "grad_norm": 0.17061145664967614,
+      "learning_rate": 3.787878787878788e-05,
+      "loss": 1.947,
+      "step": 126
+    },
+    {
+      "epoch": 12.680187207488299,
+      "grad_norm": 0.20420044689274344,
+      "learning_rate": 3.7373737373737376e-05,
+      "loss": 1.9462,
+      "step": 127
+    },
+    {
+      "epoch": 12.78003120124805,
+      "grad_norm": 0.16640664781155742,
+      "learning_rate": 3.686868686868687e-05,
+      "loss": 1.9404,
+      "step": 128
+    },
+    {
+      "epoch": 12.8798751950078,
+      "grad_norm": 0.17534875646136103,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 1.9441,
+      "step": 129
+    },
+    {
+      "epoch": 12.97971918876755,
+      "grad_norm": 0.1881647742956635,
+      "learning_rate": 3.5858585858585855e-05,
+      "loss": 1.9452,
+      "step": 130
+    },
+    {
+      "epoch": 13.0795631825273,
+      "grad_norm": 0.21130090774448568,
+      "learning_rate": 3.535353535353535e-05,
+      "loss": 1.938,
+      "step": 131
+    },
+    {
+      "epoch": 13.179407176287052,
+      "grad_norm": 0.19012207225624486,
+      "learning_rate": 3.484848484848485e-05,
+      "loss": 1.93,
+      "step": 132
+    },
+    {
+      "epoch": 13.279251170046802,
+      "grad_norm": 0.19535583015453165,
+      "learning_rate": 3.434343434343435e-05,
+      "loss": 1.9423,
+      "step": 133
+    },
+    {
+      "epoch": 13.379095163806552,
+      "grad_norm": 0.1972934873185412,
+      "learning_rate": 3.3838383838383844e-05,
+      "loss": 1.9449,
+      "step": 134
+    },
+    {
+      "epoch": 13.478939157566302,
+      "grad_norm": 0.21172258190614646,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 1.9423,
+      "step": 135
+    },
+    {
+      "epoch": 13.578783151326054,
+      "grad_norm": 0.20243808248600392,
+      "learning_rate": 3.282828282828283e-05,
+      "loss": 1.9454,
+      "step": 136
+    },
+    {
+      "epoch": 13.678627145085803,
+      "grad_norm": 0.29468220957824104,
+      "learning_rate": 3.232323232323233e-05,
+      "loss": 1.9329,
+      "step": 137
+    },
+    {
+      "epoch": 13.778471138845553,
+      "grad_norm": 0.1852836649334086,
+      "learning_rate": 3.181818181818182e-05,
+      "loss": 1.9397,
+      "step": 138
+    },
+    {
+      "epoch": 13.878315132605305,
+      "grad_norm": 0.17635021846243693,
+      "learning_rate": 3.131313131313132e-05,
+      "loss": 1.9414,
+      "step": 139
+    },
+    {
+      "epoch": 13.978159126365055,
+      "grad_norm": 0.1837620268343685,
+      "learning_rate": 3.080808080808081e-05,
+      "loss": 1.9265,
+      "step": 140
+    },
+    {
+      "epoch": 14.078003120124805,
+      "grad_norm": 0.1851416429157977,
+      "learning_rate": 3.0303030303030306e-05,
+      "loss": 1.938,
+      "step": 141
+    },
+    {
+      "epoch": 14.177847113884555,
+      "grad_norm": 0.18177436704033564,
+      "learning_rate": 2.9797979797979796e-05,
+      "loss": 1.9338,
+      "step": 142
+    },
+    {
+      "epoch": 14.277691107644305,
+      "grad_norm": 0.20249599488147646,
+      "learning_rate": 2.9292929292929294e-05,
+      "loss": 1.943,
+      "step": 143
+    },
+    {
+      "epoch": 14.377535101404057,
+      "grad_norm": 0.1914943764672633,
+      "learning_rate": 2.878787878787879e-05,
+      "loss": 1.9381,
+      "step": 144
+    },
+    {
+      "epoch": 14.477379095163807,
+      "grad_norm": 0.18144339446500468,
+      "learning_rate": 2.8282828282828282e-05,
+      "loss": 1.9493,
+      "step": 145
+    },
+    {
+      "epoch": 14.577223088923557,
+      "grad_norm": 0.22871591479507436,
+      "learning_rate": 2.777777777777778e-05,
+      "loss": 1.9394,
+      "step": 146
+    },
+    {
+      "epoch": 14.677067082683307,
+      "grad_norm": 0.2409736531836878,
+      "learning_rate": 2.7272727272727273e-05,
+      "loss": 1.9363,
+      "step": 147
+    },
+    {
+      "epoch": 14.776911076443058,
+      "grad_norm": 0.21702411701682794,
+      "learning_rate": 2.676767676767677e-05,
+      "loss": 1.9324,
+      "step": 148
+    },
+    {
+      "epoch": 14.876755070202808,
+      "grad_norm": 0.186963824720383,
+      "learning_rate": 2.6262626262626268e-05,
+      "loss": 1.9254,
+      "step": 149
+    },
+    {
+      "epoch": 14.976599063962558,
+      "grad_norm": 0.20551876684974787,
+      "learning_rate": 2.575757575757576e-05,
+      "loss": 1.9385,
+      "step": 150
+    },
+    {
+      "epoch": 15.076443057722308,
+      "grad_norm": 0.17794734645273458,
+      "learning_rate": 2.5252525252525256e-05,
+      "loss": 1.935,
+      "step": 151
+    },
+    {
+      "epoch": 15.17628705148206,
+      "grad_norm": 0.19787955354426204,
+      "learning_rate": 2.474747474747475e-05,
+      "loss": 1.9286,
+      "step": 152
+    },
+    {
+      "epoch": 15.27613104524181,
+      "grad_norm": 0.21663975391838738,
+      "learning_rate": 2.4242424242424244e-05,
+      "loss": 1.9274,
+      "step": 153
+    },
+    {
+      "epoch": 15.37597503900156,
+      "grad_norm": 0.19056508068402894,
+      "learning_rate": 2.3737373737373738e-05,
+      "loss": 1.9328,
+      "step": 154
+    },
+    {
+      "epoch": 15.47581903276131,
+      "grad_norm": 0.20643529046597323,
+      "learning_rate": 2.3232323232323232e-05,
+      "loss": 1.9374,
+      "step": 155
+    },
+    {
+      "epoch": 15.575663026521061,
+      "grad_norm": 0.17428582721990332,
+      "learning_rate": 2.272727272727273e-05,
+      "loss": 1.9435,
+      "step": 156
+    },
+    {
+      "epoch": 15.675507020280811,
+      "grad_norm": 0.17915807350384474,
+      "learning_rate": 2.2222222222222223e-05,
+      "loss": 1.9342,
+      "step": 157
+    },
+    {
+      "epoch": 15.775351014040561,
+      "grad_norm": 0.17934386940217817,
+      "learning_rate": 2.171717171717172e-05,
+      "loss": 1.9252,
+      "step": 158
+    },
+    {
+      "epoch": 15.875195007800311,
+      "grad_norm": 0.16971494417624172,
+      "learning_rate": 2.1212121212121215e-05,
+      "loss": 1.9333,
+      "step": 159
+    },
+    {
+      "epoch": 15.975039001560063,
+      "grad_norm": 0.1710725442382166,
+      "learning_rate": 2.070707070707071e-05,
+      "loss": 1.9397,
+      "step": 160
+    },
+    {
+      "epoch": 16.07488299531981,
+      "grad_norm": 0.16048331708079347,
+      "learning_rate": 2.0202020202020203e-05,
+      "loss": 1.9354,
+      "step": 161
+    },
+    {
+      "epoch": 16.174726989079563,
+      "grad_norm": 0.2209212482793433,
+      "learning_rate": 1.9696969696969697e-05,
+      "loss": 1.9572,
+      "step": 162
+    },
+    {
+      "epoch": 16.274570982839315,
+      "grad_norm": 0.17292517371584637,
+      "learning_rate": 1.919191919191919e-05,
+      "loss": 1.9384,
+      "step": 163
+    },
+    {
+      "epoch": 16.374414976599063,
+      "grad_norm": 0.1756696399704993,
+      "learning_rate": 1.8686868686868688e-05,
+      "loss": 1.9287,
+      "step": 164
+    },
+    {
+      "epoch": 16.474258970358814,
+      "grad_norm": 0.193814973934712,
+      "learning_rate": 1.8181818181818182e-05,
+      "loss": 1.9285,
+      "step": 165
+    },
+    {
+      "epoch": 16.574102964118566,
+      "grad_norm": 0.21108116449806094,
+      "learning_rate": 1.7676767676767676e-05,
+      "loss": 1.9249,
+      "step": 166
+    },
+    {
+      "epoch": 16.673946957878314,
+      "grad_norm": 0.164152325154632,
+      "learning_rate": 1.7171717171717173e-05,
+      "loss": 1.9335,
+      "step": 167
+    },
+    {
+      "epoch": 16.773790951638066,
+      "grad_norm": 0.1934976757474289,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 1.9344,
+      "step": 168
+    },
+    {
+      "epoch": 16.873634945397814,
+      "grad_norm": 0.17861559674997443,
+      "learning_rate": 1.6161616161616165e-05,
+      "loss": 1.9315,
+      "step": 169
+    },
+    {
+      "epoch": 16.973478939157566,
+      "grad_norm": 0.16812713496720977,
+      "learning_rate": 1.565656565656566e-05,
+      "loss": 1.9278,
+      "step": 170
+    },
+    {
+      "epoch": 17.073322932917318,
+      "grad_norm": 0.19243202935397094,
+      "learning_rate": 1.5151515151515153e-05,
+      "loss": 1.939,
+      "step": 171
+    },
+    {
+      "epoch": 17.173166926677066,
+      "grad_norm": 0.16546322416204856,
+      "learning_rate": 1.4646464646464647e-05,
+      "loss": 1.9295,
+      "step": 172
+    },
+    {
+      "epoch": 17.273010920436818,
+      "grad_norm": 0.19615095413628908,
+      "learning_rate": 1.4141414141414141e-05,
+      "loss": 1.9357,
+      "step": 173
+    },
+    {
+      "epoch": 17.37285491419657,
+      "grad_norm": 0.16562858231287156,
+      "learning_rate": 1.3636363636363637e-05,
+      "loss": 1.9372,
+      "step": 174
+    },
+    {
+      "epoch": 17.472698907956318,
+      "grad_norm": 0.1755423564949021,
+      "learning_rate": 1.3131313131313134e-05,
+      "loss": 1.9208,
+      "step": 175
+    },
+    {
+      "epoch": 17.57254290171607,
+      "grad_norm": 0.16572591523274388,
+      "learning_rate": 1.2626262626262628e-05,
+      "loss": 1.9196,
+      "step": 176
+    },
+    {
+      "epoch": 17.672386895475817,
+      "grad_norm": 0.16066050812369387,
+      "learning_rate": 1.2121212121212122e-05,
+      "loss": 1.9379,
+      "step": 177
+    },
+    {
+      "epoch": 17.77223088923557,
+      "grad_norm": 0.18230307180057742,
+      "learning_rate": 1.1616161616161616e-05,
+      "loss": 1.9344,
+      "step": 178
+    },
+    {
+      "epoch": 17.87207488299532,
+      "grad_norm": 0.16147840026521357,
+      "learning_rate": 1.1111111111111112e-05,
+      "loss": 1.9249,
+      "step": 179
+    },
+    {
+      "epoch": 17.97191887675507,
+      "grad_norm": 0.17234298543336798,
+      "learning_rate": 1.0606060606060607e-05,
+      "loss": 1.9341,
+      "step": 180
+    },
+    {
+      "epoch": 18.07176287051482,
+      "grad_norm": 0.16952419332241464,
+      "learning_rate": 1.0101010101010101e-05,
+      "loss": 1.9382,
+      "step": 181
+    },
+    {
+      "epoch": 18.171606864274573,
+      "grad_norm": 0.17503197241676455,
+      "learning_rate": 9.595959595959595e-06,
+      "loss": 1.9277,
+      "step": 182
+    },
+    {
+      "epoch": 18.27145085803432,
+      "grad_norm": 0.16018657280969506,
+      "learning_rate": 9.090909090909091e-06,
+      "loss": 1.9259,
+      "step": 183
+    },
+    {
+      "epoch": 18.371294851794072,
+      "grad_norm": 0.16577134954028483,
+      "learning_rate": 8.585858585858587e-06,
+      "loss": 1.9391,
+      "step": 184
+    },
+    {
+      "epoch": 18.47113884555382,
+      "grad_norm": 0.1758462044127833,
+      "learning_rate": 8.080808080808082e-06,
+      "loss": 1.9316,
+      "step": 185
+    },
+    {
+      "epoch": 18.570982839313572,
+      "grad_norm": 0.16928715932805172,
+      "learning_rate": 7.5757575757575764e-06,
+      "loss": 1.9218,
+      "step": 186
+    },
+    {
+      "epoch": 18.670826833073324,
+      "grad_norm": 0.16185874983512785,
+      "learning_rate": 7.0707070707070704e-06,
+      "loss": 1.9244,
+      "step": 187
+    },
+    {
+      "epoch": 18.770670826833072,
+      "grad_norm": 0.16445906712178507,
+      "learning_rate": 6.565656565656567e-06,
+      "loss": 1.9425,
+      "step": 188
+    },
+    {
+      "epoch": 18.870514820592824,
+      "grad_norm": 0.16313460189322437,
+      "learning_rate": 6.060606060606061e-06,
+      "loss": 1.9336,
+      "step": 189
+    },
+    {
+      "epoch": 18.970358814352576,
+      "grad_norm": 0.15990081630753986,
+      "learning_rate": 5.555555555555556e-06,
+      "loss": 1.9178,
+      "step": 190
+    },
+    {
+      "epoch": 19.070202808112324,
+      "grad_norm": 0.16547636636850527,
+      "learning_rate": 5.050505050505051e-06,
+      "loss": 1.9281,
+      "step": 191
+    },
+    {
+      "epoch": 19.170046801872076,
+      "grad_norm": 0.1625270231867559,
+      "learning_rate": 4.5454545454545455e-06,
+      "loss": 1.9348,
+      "step": 192
+    },
+    {
+      "epoch": 19.269890795631824,
+      "grad_norm": 0.16385675767663568,
+      "learning_rate": 4.040404040404041e-06,
+      "loss": 1.9305,
+      "step": 193
+    },
+    {
+      "epoch": 19.369734789391575,
+      "grad_norm": 0.16718542619114216,
+      "learning_rate": 3.5353535353535352e-06,
+      "loss": 1.9376,
+      "step": 194
+    },
+    {
+      "epoch": 19.469578783151327,
+      "grad_norm": 0.16595125072244407,
+      "learning_rate": 3.0303030303030305e-06,
+      "loss": 1.9264,
+      "step": 195
+    },
+    {
+      "epoch": 19.569422776911075,
+      "grad_norm": 0.16912445317015737,
+      "learning_rate": 2.5252525252525253e-06,
+      "loss": 1.9252,
+      "step": 196
+    },
+    {
+      "epoch": 19.669266770670827,
+      "grad_norm": 0.15257787442711698,
+      "learning_rate": 2.0202020202020206e-06,
+      "loss": 1.9312,
+      "step": 197
+    },
+    {
+      "epoch": 19.76911076443058,
+      "grad_norm": 0.17270934725449602,
+      "learning_rate": 1.5151515151515152e-06,
+      "loss": 1.9241,
+      "step": 198
+    },
+    {
+      "epoch": 19.868954758190327,
+      "grad_norm": 0.16771403116909167,
+      "learning_rate": 1.0101010101010103e-06,
+      "loss": 1.9342,
+      "step": 199
+    },
+    {
+      "epoch": 19.96879875195008,
+      "grad_norm": 0.17132458008674775,
+      "learning_rate": 5.050505050505052e-07,
+      "loss": 1.9347,
+      "step": 200
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 200,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 20,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e73c8433f9bd77a861884878bc3440468454cf7de3a2ce588dd20f479c1091e1
+size 6840

config.json

@@ -1,17 +1,17 @@
 {
-  "_name_or_path": "gte-korean-reranker-base-241210-c",
+  "_name_or_path": "/workspace/sigrid/kozistr-ko-sentence-embeddings/reranker-data/gte-korean-reranker-base-241210-c",
   "architectures": [
     "NewForSequenceClassification"
   ],
   "attention_probs_dropout_prob": 0.0,
   "auto_map": {
     "AutoConfig": "configuration.NewConfig",
-    "AutoModel": "new-impl--modeling.NewModel",
-    "AutoModelForMaskedLM": "new-impl--modeling.NewForMaskedLM",
-    "AutoModelForMultipleChoice": "new-impl--modeling.NewForMultipleChoice",
-    "AutoModelForQuestionAnswering": "new-impl--modeling.NewForQuestionAnswering",
+    "AutoModel": "Alibaba-NLP/new-impl--modeling.NewModel",
+    "AutoModelForMaskedLM": "Alibaba-NLP/new-impl--modeling.NewForMaskedLM",
+    "AutoModelForMultipleChoice": "Alibaba-NLP/new-impl--modeling.NewForMultipleChoice",
+    "AutoModelForQuestionAnswering": "Alibaba-NLP/new-impl--modeling.NewForQuestionAnswering",
     "AutoModelForSequenceClassification": "modeling.NewForSequenceClassification",
-    "AutoModelForTokenClassification": "new-impl--modeling.NewForTokenClassification"
+    "AutoModelForTokenClassification": "Alibaba-NLP/new-impl--modeling.NewForTokenClassification"
   },
   "classifier_dropout": 0.0,
   "hidden_act": "gelu",

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b9dcb302bd0fdf62bcbd0a594656e39ef4e0343906372f7613d148b5bad281d
+oid sha256:efccdc583e449d5ad7b1ab1beda845d50c68d48c9acd72a1eef1ece00b1ac8b1
 size 611934706

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c211c72915ee94f355409ecc5f90293478e9296d3d5f570648edaa36903e639a
+oid sha256:e73c8433f9bd77a861884878bc3440468454cf7de3a2ce588dd20f479c1091e1
 size 6840