<commit_before>import argparse
import os
import librosa
from joblib import Parallel, delayed
import pickle
import json
import sys
sys.path.insert(0, '../')
import common
from configurations import config_spectro
import signal

"""
spectrograms.py: computes spectrograms.

Requires pre-precomputing an 'index_file',a .tsv where an index with id,path is defined for a dataset.

The results and parameters of this script are stored in common.DATA_PATH/spectrograms/
'index.tsv' stores the 'id,path_spectrogram,path_audio'.
'path_spectrgram' and 'path_audio' absolute reference from common.DATA_PATH.

Step 1/5 of the pipeline.

"""

# Choi et al.: 	'original_sr' : 16000, 'resample_sr' : 12000, 'hop' : 256, 'spectrogram_type' : 'mel', 'n_fft' : 512, 'n_mels' : 96.
# Dieleman et al.: 	'original_sr' : 16000, 'resample_sr' : 16000, 'hop' : 256, 'spectrogram_type' : 'mel', 'n_fft' : 512, 'n_mels' : 128.

path2id = dict()
config = dict()

def signal_handler(signum, frame):
    raise Exception("Timed out!")

def compute_spec(audio_file,spectro_file):
	# Get actual audio
	audio, sr = librosa.load(audio_file, sr=config['resample_sr'])
	# Compute spectrogram
	if config['spectrogram_type']=='cqt':
		spec = librosa.cqt(audio, sr=sr, hop_length=config['hop'], n_bins=config['cqt_bins'], real=False)
	elif config['spectrogram_type']=='mel':
		spec = librosa.feature.melspectrogram(y=audio, sr=sr, hop_length=config['hop'],n_fft=config['n_fft'],n_mels=config['n_mels'])
	elif config['spectrogram_type']=='stft':
		spec = librosa.stft(y=audio,n_fft=config['n_fft'])
	# Write results:
	with open(spectro_file, "w") as f:
		pickle.dump(spec, f, protocol=-1) # spec shape: MxN.

def do_process(id, audio_file, spectro_file):
	try:
		if config['compute_spectro']:
			if not os.path.exists(spectro_file[:spectro_file.rfind('/')+1]):
				os.makedirs(spectro_file[:spectro_file.rfind('/')+1])
			if not os.path.isfile(spectro_file): 
				signal.signal(signal.SIGALRM, signal_handler)
				signal.alarm(10)
				compute_spec(audio_file,spectro_file)
			fw = open(common.SPECTRO_PATH+config['spectro_folder']+"index.tsv","a")
			fw.write("%s\t%s\t%s\n" % (id,spectro_file[len(common.SPECTRO_PATH):],audio_file[len(common.AUDIO_PATH):]))
			fw.close()
			print 'Computed spec: %s' % audio_file
		else:
			if os.path.isfile(spectro_file):
				fw = open(common.SPECTRO_PATH+config['spectro_folder']+"index.tsv","a")
				fw.write("%s\t%s\t%s\n" % (id,spectro_file[len(common.SPECTRO_PATH):],audio_file[len(common.AUDIO_PATH):]))
				fw.close()
	except Exception as e:
		ferrors = open(common.SPECTRO_PATH+config['spectro_folder']+"errors.txt","a")
		ferrors.write(audio_file+"\n")
		ferrors.write(str(e))
		ferrors.close()
		print 'Error computing spec', audio_file
		print str(e)
<commit_msg>corrected errors create_spectro<commit_after>import argparse
import os
import librosa
from joblib import Parallel, delayed
import pickle
import json
import sys
sys.path.insert(0, '../')
import common
from configurations import config_spectro
import signal

"""
spectrograms.py: computes spectrograms.

Requires pre-precomputing an 'index_file',a .tsv where an index with id,path is defined for a dataset.

The results and parameters of this script are stored in common.DATA_PATH/spectrograms/
'index.tsv' stores the 'id,path_spectrogram,path_audio'.
'path_spectrgram' and 'path_audio' absolute reference from common.DATA_PATH.

Step 1/5 of the pipeline.

"""

# Choi et al.: 	'original_sr' : 16000, 'resample_sr' : 12000, 'hop' : 256, 'spectrogram_type' : 'mel', 'n_fft' : 512, 'n_mels' : 96.
# Dieleman et al.: 	'original_sr' : 16000, 'resample_sr' : 16000, 'hop' : 256, 'spectrogram_type' : 'mel', 'n_fft' : 512, 'n_mels' : 128.

path2id = dict()
config = dict()

def signal_handler(signum, frame):
    raise Exception("Timed out!")

def compute_spec(audio_file,spectro_file):
	# Get actual audio
	audio, sr = librosa.load(audio_file, sr=config['resample_sr'])
	# Compute spectrogram
	if config['spectrogram_type']=='cqt':
		spec = librosa.cqt(audio, sr=sr, hop_length=config['hop'], n_bins=config['cqt_bins'], real=False)
	elif config['spectrogram_type']=='mel':
		spec = librosa.feature.melspectrogram(y=audio, sr=sr, hop_length=config['hop'],n_fft=config['n_fft'],n_mels=config['n_mels'])
	elif config['spectrogram_type']=='stft':
		spec = librosa.stft(y=audio,n_fft=config['n_fft'])
	# Write results:
	with open(spectro_file, "w") as f:
		pickle.dump(spec, f, protocol=-1) # spec shape: MxN.

def do_process(id, audio_file, spectro_file):
	try:
		if config['compute_spectro']:
			#print spectro_file[:spectro_file.rfind('/')+1]
			try:
				if not os.path.exists(spectro_file[:spectro_file.rfind('/')+1]):
					os.makedirs(spectro_file[:spectro_file.rfind('/')+1])
			except:
				pass
			if not os.path.isfile(spectro_file): 
				signal.signal(signal.SIGALRM, signal_handler)
				signal.alarm(50)
				compute_spec(audio_file,spectro_file)
			fw = open(common.SPECTRO_PATH+config['spectro_folder']+"index.tsv","a")
			fw.write("%s\t%s\t%s\n" % (id,spectro_file[len(common.SPECTRO_PATH):],audio_file[len(common.AUDIO_PATH):]))
			fw.close()
			print 'Computed spec: %s' % audio_file
		else:
			if os.path.isfile(spectro_file):
				fw = open(common.SPECTRO_PATH+config['spectro_folder']+"index.tsv","a")
				fw.write("%s\t%s\t%s\n" % (id,spectro_file[len(common.SPECTRO_PATH):],audio_file[len(common.AUDIO_PATH):]))
				fw.close()
	except Exception as e:
		ferrors = open(common.SPECTRO_PATH+config['spectro_folder']+"errors.txt","a")
		ferrors.write(audio_file+"\n")
		ferrors.write(str(e))
	#	ferrors.close()
	#	print 'Error computing spec', audio_file
	#	print str(e)

{'fraction_non_alphanumeric': 0.15804394046775336, 'fraction_numerical': 0.016477675407512403, 'mean_word_length': 5.728247914183552, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2957905', 'n_tokens_mistral': 2360, 'n_tokens_neox': 2153, 'n_words': 417}