Upload 2 files

configuration.py

@@ -0,0 +1,11 @@
+from transformers import Wav2Vec2Config
+
+
+class Wav2Vec2ForEmotionClassificationConfig(Wav2Vec2Config):
+    model_type = "wav2vec2_for_emotion_classification"
+
+    def __init__(
+        self,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)

modeling.py

@@ -0,0 +1,135 @@
+from typing import Optional
+
+import torch
+import torch.nn as nn
+from transformers.activations import get_activation
+from transformers.modeling_outputs import SequenceClassifierOutput
+from transformers.models.wav2vec2.modeling_wav2vec2 import (
+    Wav2Vec2Model,
+    Wav2Vec2PreTrainedModel,
+)
+
+from configuration import Wav2Vec2ForEmotionClassificationConfig
+
+_HIDDEN_STATES_START_POSITION = 2
+
+
+class ClassificationHead(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.classifier_proj_size)
+        self.layer_norm = nn.LayerNorm(config.classifier_proj_size)
+        self.dropout = nn.Dropout(config.final_dropout)
+        self.out_proj = nn.Linear(config.classifier_proj_size, config.num_labels)
+        self.activation = get_activation(config.head_activation)
+
+    def forward(self, features, **kwargs):
+        x = features
+        x = self.dense(x)
+        x = self.layer_norm(x)
+        x = self.activation(x)
+        x = self.dropout(x)
+        x = self.out_proj(x)
+        return x
+
+
+class Wav2Vec2ForEmotionClassification(Wav2Vec2PreTrainedModel):
+    """Speech emotion classifier."""
+
+    config_class = Wav2Vec2ForEmotionClassificationConfig
+
+    def __init__(self, config, counts: Optional[dict[int, int]] = None):
+        super().__init__(config)
+
+        self.config = config
+        self.wav2vec2 = Wav2Vec2Model(config)
+        self.classifier = ClassificationHead(config)
+        num_layers = (
+            config.num_hidden_layers + 1
+        )  # transformer layers + input embeddings
+        if config.use_weighted_layer_sum:
+            self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers)
+        self.init_weights()
+
+        # counts が与えられている場合、クラスの重みを計算
+        if counts is not None:
+            print(f"Using class weights: {counts}")
+            counts_list = [counts[i] for i in range(config.num_labels)]
+            counts_tensor = torch.tensor(
+                counts_list, dtype=torch.float, device="cuda:0"
+            )
+            total_samples = counts_tensor.sum()
+            class_weights = total_samples / (config.num_labels * counts_tensor)
+            # 重みを正規化（任意）
+            class_weights = class_weights / class_weights.sum() * config.num_labels
+            self.class_weights = class_weights
+        else:
+            self.class_weights = None  # counts がない場合は None に設定
+
+    def forward(
+        self,
+        input_values: Optional[torch.Tensor],
+        attention_mask: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        labels: Optional[torch.Tensor] = None,
+    ):
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+        output_hidden_states = (
+            True if self.config.use_weighted_layer_sum else output_hidden_states
+        )
+        # print(f"output_hidden_states: {output_hidden_states}")
+
+        outputs = self.wav2vec2(
+            input_values,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        if self.config.use_weighted_layer_sum:
+            hidden_states = outputs[_HIDDEN_STATES_START_POSITION]
+            hidden_states = torch.stack(hidden_states, dim=1)
+            norm_weights = nn.functional.softmax(self.layer_weights, dim=-1)
+            hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(dim=1)
+        else:
+            hidden_states = outputs[0]
+
+        if attention_mask is None:
+            pooled_output = hidden_states.mean(dim=1)
+        else:
+            padding_mask = self._get_feature_vector_attention_mask(
+                hidden_states.shape[1], attention_mask
+            )
+            hidden_states[~padding_mask] = 0.0
+            pooled_output = hidden_states.sum(dim=1) / padding_mask.sum(dim=1).view(
+                -1, 1
+            )
+
+        logits = self.classifier(pooled_output)
+
+        loss = None
+        if labels is not None:
+            # CrossEntropyLoss に重みを適用（class_weights が None でも機能する）
+            loss_fct = nn.CrossEntropyLoss(weight=self.class_weights)
+            loss = loss_fct(logits.view(-1, self.config.num_labels), labels.view(-1))
+
+        return SequenceClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def freeze_base_model(self):
+        r"""Freeze base model."""
+        for param in self.wav2vec2.parameters():
+            param.requires_grad = False
+
+    def freeze_feature_encoder(self):
+        r"""Freeze feature extractor."""
+        self.wav2vec2.freeze_feature_encoder()