<jupyter_start><jupyter_text>calculating mfccs<jupyter_code>base_folder = pathlib.Path("/content/drive/My Drive/validated_clips/validated_clip/")
out_folder =  pathlib.Path("/content/drive/My Drive/validated_clips/preprocessed/")
outfiles = os.listdir(out_folder)
already_done = set()

for file in outfiles:
  thing = re.compile('[a-z]')
  already_done.add(file[:thing.search(file).start()])

folders = os.listdir(base_folder)
todo = ["2.000000"]
for folder in folders:
  #if folder in already_done:
  #  print(f'skipping {folder}')
  # continue
  if folder not in todo:
    print(f'skipping{folder}')
    continue

  print(folder)
  inner_folder = base_folder / folder
  globby = str(inner_folder)+"/*.mp3"
  downloaded_files = glob.glob(globby)
  #print(downloaded_files)

  filepath_dict = {pathlib.Path(filepath).name : filepath for filepath in downloaded_files}
  print(filepath_dict)

  dev = pd.read_csv("/content/drive/My Drive/tsvs/dev.tsv", sep="\t")
  train = pd.read_csv("/content/drive/My Drive/tsvs/train.tsv", sep="\t")
  test = pd.read_csv("/content/drive/My Drive/tsvs/test.tsv", sep="\t")
  splits = {
      "dev": dev,
      "train": train, 
      "test": test
  }

  #only files in current folder
  for key in splits:
    print(key)
    print(splits[key].shape)
    splits[key]["path"] = splits[key]["path"].map(filepath_dict)
    splits[key] = splits[key][splits[key]['path'].notna()]
    print(splits[key].shape)

  #only ones with accents
  for key in splits:
    splits[key].dropna(axis=0, subset=["accent"], inplace=True)
  

  downloaded_audios_dict = {key : [pydub.AudioSegment.from_mp3(f) for f in splits[key]['path']] for key in splits}
  max_lengths = [np.max([len(sample) for sample in samples]) for samples in downloaded_audios_dict.values() if samples != []]
  if max_lengths == []:
    print("I cry")
    continue
  max_audio_length = np.max(max_lengths)
  padded_audio_dict = {key: [zero_pad_in_end(audio, max_audio_length) for audio in audios] for key, audios in downloaded_audios_dict.items()}

  labels_dict = {key: splits[key]['accent'].to_numpy() for key in splits}
  paths_dict ={key: [f for f in splits[key]['path']] for key in splits}
  mfccs_padded_dict = {key: np.array([extract_mfcc(audio) for audio in padded_audios]) for key, padded_audios in padded_audio_dict.items()}

  label_filename = folder+"label.pkl"
  paths_filename = folder+"paths.pkl"
  mfccs_padded_filename = folder+"mfccs_padded.pkl"

  label_filepath = out_folder / label_filename
  paths_filepath = out_folder / paths_filename
  mfccs_padded_filepath = out_folder / mfccs_padded_filename

  with open(label_filepath , "wb") as f:
    dill.dump(labels_dict, f)
  with open(paths_filepath , "wb") as f:
    dill.dump(paths_dict, f)
  with open(mfccs_padded_filepath , "wb") as f:
    dill.dump(mfccs_padded_dict, f)
#SAVE GENDER

wanted_accents=["england", "indian", "us", "scotland", "australia"]
preprocessed_folder = pathlib.Path('/content/drive/My Drive/validated_clips/preprocessed/')

for folder in folders:
  print(folder)
  inner_folder = base_folder / folder
  globby = str(inner_folder)+"/*.mp3"
  downloaded_files = glob.glob(globby)
  #print(downloaded_files)

  filepath_dict = {pathlib.Path(filepath).name : filepath for filepath in downloaded_files}
  #print(filepath_dict)

  dev = pd.read_csv("/content/drive/My Drive/tsvs/dev.tsv", sep="\t")
  train = pd.read_csv("/content/drive/My Drive/tsvs/train.tsv", sep="\t")
  test = pd.read_csv("/content/drive/My Drive/tsvs/test.tsv", sep="\t")
  splits = {
      "dev": dev,
      "train": train, 
      "test": test
  }

  #only files in current folder
  for key in splits:
    splits[key]["path"] = splits[key]["path"].map(filepath_dict)
    splits[key] = splits[key][splits[key]['path'].notna()]

  #only ones with accents
  for key in splits:
    splits[key].dropna(axis=0, subset=["accent"], inplace=True)
  
  #only ones in wanted accents
  for key in splits:
    splits[key].loc[splits[key]['accent'].isin(wanted_accents)]


  genders_dict = {key: splits[key]['gender'].to_numpy() for key in splits}
  genders_filename = folder+"genders_5accents.pkl"
  genders_filepath = preprocessed_folder / genders_filename

  with open(genders_filepath , "wb") as f:
    dill.dump(genders_dict, f)
#SAVE speaker

wanted_accents=["england", "indian", "us", "scotland", "australia"]
preprocessed_folder = pathlib.Path('/content/drive/My Drive/validated_clips/preprocessed/')

for folder in folders:
  print(folder)
  inner_folder = base_folder / folder
  globby = str(inner_folder)+"/*.mp3"
  downloaded_files = glob.glob(globby)
  #print(downloaded_files)

  filepath_dict = {pathlib.Path(filepath).name : filepath for filepath in downloaded_files}
  #print(filepath_dict)

  dev = pd.read_csv("/content/drive/My Drive/tsvs/dev.tsv", sep="\t")
  train = pd.read_csv("/content/drive/My Drive/tsvs/train.tsv", sep="\t")
  test = pd.read_csv("/content/drive/My Drive/tsvs/test.tsv", sep="\t")
  splits = {
      "dev": dev,
      "train": train, 
      "test": test
  }

  #only files in current folder
  for key in splits:
    splits[key]["path"] = splits[key]["path"].map(filepath_dict)
    splits[key] = splits[key][splits[key]['path'].notna()]

  #only ones with accents
  for key in splits:
    splits[key].dropna(axis=0, subset=["accent"], inplace=True)
  
  #only ones in wanted accents
  for key in splits:
    splits[key].loc[splits[key]['accent'].isin(wanted_accents)]


  speakers_dict = {key: splits[key]['client_id'].to_numpy() for key in splits}
  speakers_filename = folder+"speakers_5accents.pkl"
  speakers_filepath = preprocessed_folder / speakers_filename
  print(speakers_filepath)

  with open(speakers_filepath , "wb") as f:
    dill.dump(speakers_dict, f)
print(len(already_done))
print(len(folders))<jupyter_output>208<jupyter_text>fixing preprocessing<jupyter_code>preprocessed_folder = pathlib.Path('/content/drive/My Drive/validated_clips/preprocessed/')

mfccs_padded = {"dev":[], "train":[], "test":[]}
for i in range(1,209):
  number = '{:.6f}'.format(i)
  mfcc_filename = number+"mfccs_padded.pkl"
  mfcc_filepath = preprocessed_folder / mfcc_filename

  with open(mfcc_filepath, "rb") as f:
    mfcc_local = dill.load(f)
  mfccs_padded['dev'].extend(mfcc_local['dev'])
  mfccs_padded['train'].extend(mfcc_local['train'])
  mfccs_padded['test'].extend(mfcc_local['test'])
max_lengths = [np.max([len(sample) for sample in samples]) for samples in mfccs_padded.values() if samples != []]
print(max_lengths)
max_length = np.max(max_lengths)
print(max_length)
wanted_accents=["england", "indian", "us", "scotland", "australia"]

for folder_num in range(1,209):
  number = '{:.6f}'.format(folder_num)
  mfcc_filename = number+"mfccs_padded_real.pkl"
  label_filename = number+"label.pkl"
  paths_filename = number+"paths.pkl"
  mfcc_filepath = preprocessed_folder / mfcc_filename
  label_filepath = preprocessed_folder / label_filename
  paths_filepath = preprocessed_folder / paths_filename

  with open(mfcc_filepath, "rb") as f:
    mfcc_local = dill.load(f)
  with open(label_filepath, "rb") as f:
    label_local = dill.load(f)
  with open(paths_filepath , "rb") as f:
    path_local = dill.load(f)

  new_labels = {key: [label_local[key][i] for i in range(len(label_local[key])) if label_local[key][i] in wanted_accents] for key in label_local}
  new_mfccs = {key: [mfcc_local[key][i] for i in range(len(label_local[key])) if label_local[key][i] in wanted_accents] for key in label_local}
  new_paths = {key: [path_local[key][i] for i in range(len(label_local[key])) if label_local[key][i] in wanted_accents] for key in label_local}

  

  new_label_filename = number+"label_5accents.pkl"
  new_mfcc_filename = number+"mfcc_5accents.pkl"
  new_path_filename = number+"path_5accents.pkl"

  new_label_filepath = preprocessed_folder / new_label_filename
  new_mfccs_filepath = preprocessed_folder / new_mfcc_filename
  new_paths_filepath = preprocessed_folder / new_path_filename

  print(folder_num, [len(value) for value in new_mfccs.values()],'\n', new_labels)

  with open(new_label_filepath , "wb") as f:
    dill.dump(new_labels, f)
  with open(new_paths_filepath , "wb") as f:
    dill.dump(new_paths, f)
  with open(new_mfccs_filepath , "wb") as f:
    dill.dump(new_mfccs, f)
overall_padded= {key: np.stack( overall_padded[key], axis=0 ) for key in overall_padded.keys()}
print(stackyboi)

overall_filepath = preprocessed_folder / "overall_mfcc_padded.pkl"
with open(mfcc_filepath, "wb") as f:
    mfcc_local = dill.write(overall_padded, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14239067055393587, 'fraction_numerical': 0.003848396501457726, 'mean_word_length': 5.009810791871058, 'pattern_counts': {'":': 12, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '239525', 'license': 'MIT', 'n_tokens_mistral': 3297, 'n_tokens_neox': 3178, 'n_words': 621, 'path': 'mfcc saver.ipynb', 'repo_name': 'Faryab/accent_recognition', 'size': '8575'}