|
--- |
|
license: apache-2.0 |
|
tags: |
|
- generated_from_trainer |
|
datasets: |
|
- librispeech_asr |
|
metrics: |
|
- f1 |
|
base_model: facebook/wav2vec2-xls-r-300m |
|
model-index: |
|
- name: weights |
|
results: [] |
|
--- |
|
|
|
<!-- This model card has been generated automatically according to the information the Trainer had access to. You |
|
should probably proofread and complete it, then remove this comment. --> |
|
|
|
# wav2vec2-large-xlsr-53-gender-recognition-librispeech |
|
|
|
This model is a fine-tuned version of [facebook/wav2vec2-xls-r-300m](https://huggingface.co/facebook/wav2vec2-xls-r-300m) on Librispeech-clean-100 for gender recognition. |
|
It achieves the following results on the evaluation set: |
|
- Loss: 0.0061 |
|
- F1: 0.9993 |
|
|
|
### Compute your inferences |
|
|
|
```python |
|
import os |
|
import random |
|
from glob import glob |
|
from typing import List, Optional, Union, Dict |
|
|
|
import tqdm |
|
import torch |
|
import torchaudio |
|
import numpy as np |
|
import pandas as pd |
|
from torch import nn |
|
from torch.utils.data import DataLoader |
|
from torch.nn import functional as F |
|
from transformers import ( |
|
AutoFeatureExtractor, |
|
AutoModelForAudioClassification, |
|
Wav2Vec2Processor |
|
) |
|
|
|
class CustomDataset(torch.utils.data.Dataset): |
|
def __init__( |
|
self, |
|
dataset: List, |
|
basedir: Optional[str] = None, |
|
sampling_rate: int = 16000, |
|
max_audio_len: int = 5, |
|
): |
|
self.dataset = dataset |
|
self.basedir = basedir |
|
|
|
self.sampling_rate = sampling_rate |
|
self.max_audio_len = max_audio_len |
|
|
|
def __len__(self): |
|
""" |
|
Return the length of the dataset |
|
""" |
|
return len(self.dataset) |
|
|
|
def __getitem__(self, index): |
|
if self.basedir is None: |
|
filepath = self.dataset[index] |
|
else: |
|
filepath = os.path.join(self.basedir, self.dataset[index]) |
|
|
|
speech_array, sr = torchaudio.load(filepath) |
|
|
|
if speech_array.shape[0] > 1: |
|
speech_array = torch.mean(speech_array, dim=0, keepdim=True) |
|
|
|
if sr != self.sampling_rate: |
|
transform = torchaudio.transforms.Resample(sr, self.sampling_rate) |
|
speech_array = transform(speech_array) |
|
sr = self.sampling_rate |
|
|
|
len_audio = speech_array.shape[1] |
|
|
|
# Pad or truncate the audio to match the desired length |
|
if len_audio < self.max_audio_len * self.sampling_rate: |
|
# Pad the audio if it's shorter than the desired length |
|
padding = torch.zeros(1, self.max_audio_len * self.sampling_rate - len_audio) |
|
speech_array = torch.cat([speech_array, padding], dim=1) |
|
else: |
|
# Truncate the audio if it's longer than the desired length |
|
speech_array = speech_array[:, :self.max_audio_len * self.sampling_rate] |
|
|
|
speech_array = speech_array.squeeze().numpy() |
|
|
|
return {"input_values": speech_array, "attention_mask": None} |
|
|
|
|
|
class CollateFunc: |
|
def __init__( |
|
self, |
|
processor: Wav2Vec2Processor, |
|
padding: Union[bool, str] = True, |
|
pad_to_multiple_of: Optional[int] = None, |
|
return_attention_mask: bool = True, |
|
sampling_rate: int = 16000, |
|
max_length: Optional[int] = None, |
|
): |
|
self.sampling_rate = sampling_rate |
|
self.processor = processor |
|
self.padding = padding |
|
self.pad_to_multiple_of = pad_to_multiple_of |
|
self.return_attention_mask = return_attention_mask |
|
self.max_length = max_length |
|
|
|
def __call__(self, batch: List[Dict[str, np.ndarray]]): |
|
# Extract input_values from the batch |
|
input_values = [item["input_values"] for item in batch] |
|
|
|
batch = self.processor( |
|
input_values, |
|
sampling_rate=self.sampling_rate, |
|
return_tensors="pt", |
|
padding=self.padding, |
|
max_length=self.max_length, |
|
pad_to_multiple_of=self.pad_to_multiple_of, |
|
return_attention_mask=self.return_attention_mask |
|
) |
|
|
|
return { |
|
"input_values": batch.input_values, |
|
"attention_mask": batch.attention_mask if self.return_attention_mask else None |
|
} |
|
|
|
|
|
def predict(test_dataloader, model, device: torch.device): |
|
""" |
|
Predict the class of the audio |
|
""" |
|
model.to(device) |
|
model.eval() |
|
preds = [] |
|
|
|
with torch.no_grad(): |
|
for batch in tqdm.tqdm(test_dataloader): |
|
input_values, attention_mask = batch['input_values'].to(device), batch['attention_mask'].to(device) |
|
|
|
logits = model(input_values, attention_mask=attention_mask).logits |
|
scores = F.softmax(logits, dim=-1) |
|
|
|
pred = torch.argmax(scores, dim=1).cpu().detach().numpy() |
|
|
|
preds.extend(pred) |
|
|
|
return preds |
|
|
|
|
|
def get_gender(model_name_or_path: str, audio_paths: List[str], label2id: Dict, id2label: Dict, device: torch.device): |
|
num_labels = 2 |
|
|
|
feature_extractor = AutoFeatureExtractor.from_pretrained(model_name_or_path) |
|
model = AutoModelForAudioClassification.from_pretrained( |
|
pretrained_model_name_or_path=model_name_or_path, |
|
num_labels=num_labels, |
|
label2id=label2id, |
|
id2label=id2label, |
|
) |
|
|
|
test_dataset = CustomDataset(audio_paths, max_audio_len=5) # for 5-second audio |
|
|
|
data_collator = CollateFunc( |
|
processor=feature_extractor, |
|
padding=True, |
|
sampling_rate=16000, |
|
) |
|
|
|
test_dataloader = DataLoader( |
|
dataset=test_dataset, |
|
batch_size=16, |
|
collate_fn=data_collator, |
|
shuffle=False, |
|
num_workers=2 |
|
) |
|
|
|
preds = predict(test_dataloader=test_dataloader, model=model, device=device) |
|
|
|
return preds |
|
|
|
model_name_or_path = "alefiury/wav2vec2-large-xlsr-53-gender-recognition-librispeech" |
|
|
|
audio_paths = [] # Must be a list with absolute paths of the audios that will be used in inference |
|
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") |
|
|
|
label2id = { |
|
"female": 0, |
|
"male": 1 |
|
} |
|
|
|
id2label = { |
|
0: "female", |
|
1: "male" |
|
} |
|
|
|
num_labels = 2 |
|
|
|
preds = get_gender(model_name_or_path, audio_paths, label2id, id2label, device) |
|
``` |
|
|
|
|
|
## Training and evaluation data |
|
|
|
The Librispeech-clean-100 dataset was used to train the model, with 70% of the data used for training, 10% for validation, and 20% for testing. |
|
|
|
### Training hyperparameters |
|
|
|
The following hyperparameters were used during training: |
|
- learning_rate: 3e-05 |
|
- train_batch_size: 4 |
|
- eval_batch_size: 4 |
|
- seed: 42 |
|
- gradient_accumulation_steps: 4 |
|
- total_train_batch_size: 16 |
|
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08 |
|
- lr_scheduler_type: linear |
|
- lr_scheduler_warmup_ratio: 0.1 |
|
- num_epochs: 1 |
|
- mixed_precision_training: Native AMP |
|
|
|
### Training results |
|
|
|
| Training Loss | Epoch | Step | Validation Loss | F1 | |
|
|:-------------:|:-----:|:----:|:---------------:|:------:| |
|
| 0.002 | 1.0 | 1248 | 0.0061 | 0.9993 | |
|
|
|
|
|
### Framework versions |
|
|
|
- Transformers 4.28.0 |
|
- Pytorch 2.0.0+cu118 |
|
- Tokenizers 0.13.3 |
