<jupyter_start><jupyter_text>create 1 GMM per TIMIT phone type, fitted only on those phone observations.<jupyter_code># general imports
import os
import pickle
import numpy as np
import copy
# import Scykit learn GMM library
from sklearn import mixture
# import custom functions
import sys
# path to libraries
# currently in ../scripts-lib/
tool_path = os.path.abspath('..' + os.sep + 'scripts-lib')

if tool_path not in sys.path:
    sys.path.append(tool_path)
import lib_phones as lph

# print the loaded functions
print dir(lph)[5:]
# load phone list
phone_path = os.path.abspath('../datasets/TIMIT-MFCCs/TIMIT_phone_list.txt')
phone_list = lph.load_phone_file(phone_path)
print len(phone_list), phone_list
#load mfccs into sklearn observations, each frame is an obs

train_TIMIT_dir = os.path.abspath('../datasets/TIMIT-MFCCs/dev')

#create individual obs list for each phone type
train_phone_obs_dict = {}
for phone in phone_list:
    train_phone_obs_dict[phone] = []

# complete obs 
train_obs = []
train_obs_labels = []

# walk the directories
for (path, dirs, files) in os.walk(train_TIMIT_dir):
    print "working in path : " + path

    for file in files:
        # skip the SA files
        #dev, only work on file si1573.mfcc.csv     "si1573" in file and
        if ".mfcc" in file  and "sa" not in file:
            #check if corresponding .phn file exists
            if not os.path.exists(path + "/" + file[:-8] + "phn"):
                print path + "/" + file[:-8] + "phn"
                print "corresponding .phn file does not exist!"
            else:
                
                print "working on: " + file
#                 print "from path : " + path

                # open the files
                mfcc_file = open(path + "/" + file)
                phn_file = open(path + "/" + file[:-8] + "phn")

                # extract phone times
                phone_times = []
                for phn_line in phn_file:
                    phone_times.append(phn_line.split())
                # transpose for easier use
                phone_times = map(list, zip(*phone_times))

                # skip mfcc_file header
                next(mfcc_file)

                # reset frame count
                frame_cnt = 0

                # for each line of mfcc_file
                for mfcc_line in mfcc_file:

                    # increment frame count
                    frame_cnt += 1 

                    # print "frame line #:", frame_cnt 

                    # frame start time in seconds
                    start_t = mfcc_line.split(";")[1]

                    # create frame (skiping first 2 values, frame_index and frame_time)
                    frame = map( float,  mfcc_line.split(";")[2:])
                    # print numpy.shape(frame)
                    # print frame

                    # find correspond phoneme and index in the list
                    phn_index = lph.find_phone_index(start_t, phone_times, phone_list)

                    # add to instances for corresponding 
                    train_phone_obs_dict[phone_list[phn_index]].append(frame)
                    # add to instances
                    train_obs.append(frame)
                    train_obs_labels.append(phone_list[phn_index])
# smallest phone observations
min([len(train_phone_obs_dict[phone]) for phone in phone_list])
# gmm parameters
num_components = 10
num_iter=2

# dictionary containing a gmm for each phone, trained directly on obervation on those phones
gmm_dict = dict()

# for each phone, build a GMM
for phone in phone_list:
    
    # check if enough observations exist
    if len(train_phone_obs_dict[phone]) < num_components:
        print "not enough obs for phone", phone
        pass
    else:
        print "training gmm for phone", phone
        g = mixture.GMM(n_components= num_components, n_iter=num_iter) 
        g.fit(train_phone_obs_dict[phone])
        gmm_dict[phone] = g
    
#save gmm in pickled form
import cPickle as pickle

# name and location to save in
pickle_name = "TIMIT_gmm_unadapted_dict" + ".pckl"
pickle_dir = os.path.abspath('..'+ os.sep + 'datasets' + os.sep + 'TIMIT Pickled Data')

if not os.path.isdir(pickle_dir):
    os.makedirs(pickle_dir)
    
pickle.dump( gmm_dict, open( pickle_dir + os.sep + pickle_name, "wb") )
print "saved unadapted gmm dictionary in ", pickle_dir + os.sep + pickle_name<jupyter_output>saved unadapted gmm dictionary in  C:\Users\FG\Desktop\PhD\Research\Reservoirs\datasets\TIMIT Pickled Data\TIMIT_gmm_unadapted_dict.pckl
{'fraction_non_alphanumeric': 0.08883977900552487, 'fraction_numerical': 0.004640883977900552, 'mean_word_length': 2.2076541459957477, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '326383', 'license': 'MIT', 'n_tokens_mistral': 1437, 'n_tokens_neox': 1361, 'n_words': 399, 'path': 'Reservoirs/Notebooks/non-adapted GMM.ipynb', 'repo_name': 'grezesf/Research', 'size': '4525'}