<jupyter_start><jupyter_text>IEMOCAP  01  extract transcript<jupyter_code>import csv
import os
from file_util import *
create_folder('../data/processed/IEMOCAP')
out_file = '../data/processed/IEMOCAP/processed_tran.csv'
os.system('rm ' + out_file)
def extract_trans( list_in_file, out_file ) :
    
    lines = []
    
    for in_file in list_in_file:
        cnt = 0
        
        with open(in_file, 'r') as f:
            lines = f.readlines()

        with open(out_file, 'a') as f:

            csv_writer = csv.writer( f )
            lines = sorted(lines)                  # sort based on first element
            
            for line in lines:

                name = line.split(':')[0].split(' ')[0].strip()
                
                # unwanted case 
                if name[:3] != 'Ses':             # noise transcription such as reply  M: sorry
                    continue
                elif name[-3:-1] == 'XX':        # we don't have matching pair in label
                    continue
                trans = line.split(':')[1].strip()
                
                cnt += 1
                csv_writer.writerow([name, trans])
# [schema] ID, transcriptions [csv]

list_files = []

for x in xrange(5):
    sess_name = 'Session' + str(x+1)

    path = '../data/raw/IEMOCAP_full_release/' + sess_name + '/dialog/transcriptions/'
    file_search(path, list_files)
    list_files = sorted(list_files)

    print sess_name + ", #sum files: " + str(len(list_files))

extract_trans(list_files, out_file)<jupyter_output><empty_output><jupyter_text>02 extract label<jupyter_code>import csv
import os
import sys
import numpy as np
from file_util import *
out_file = '../data/processed/IEMOCAP/label.csv'
os.system('rm ' + out_file)
list_category = [
                'ang',
                'hap',
                'sad',
                'neu',
                'fru',
                'exc',
                'fea',
                'sur',
                'dis',
                'oth',
                'xxx'
                ]

category = {}
for c_type in list_category:
    if category.has_key(c_type):
        ;
    else:
        category[c_type] = len(category)
def find_category(lines):
    is_target = True
    
    id = ''
    c_label = ''
    list_ret = []
    
    for line in lines:
        
        if is_target == True:
            
            try:
                id          = line.split('\t')[1].strip()  #  extract ID
                c_label  = line.split('\t')[2].strip()  #  extract category
                if not category.has_key(c_label):
                    print "ERROR nokey" + c_label
                    sys.exit()
                
                list_ret.append( [id, c_label] )
                is_target = False

            except:
                print "ERROR " + line
                sys.exit()
        
        else:
            if line == '\n':
                is_target = True
        
    return list_ret
def extract_labels( list_in_file, out_file ) :
    id = ''
    lines = []
    list_ret = []
    
    for in_file in list_in_file:
        
        with open(in_file, 'r') as f:
            lines = f.readlines()
            lines = lines[2:]                           # remove head
            list_ret = find_category(lines)
            
        list_ret = sorted(list_ret)                   # sort based on first element
    
        with open(out_file, 'a') as f:
            csv_writer = csv.writer( f )
            csv_writer.writerows( list_ret )                   
# [schema] ID, label [csv]

list_files = []
list_avoid_dir = ['Attribute', 'Categorical', 'Self-evaluation']

for x in xrange(5):
    sess_name = 'Session' + str(x+1)

    path = '../data/raw/IEMOCAP_full_release/' + sess_name + '/dialog/EmoEvaluation/'
    file_search(path, list_files, list_avoid_dir)
    list_files = sorted(list_files)

    print sess_name + ", #sum files: " + str(len(list_files))

extract_labels(list_files, out_file)<jupyter_output><empty_output><jupyter_text>02-1 convert label category- angry : 0        ang(1103)- happy : 1       exc(1041), hap(595)- sad : 2          sad(1084)   - neutral : 3     neu(1708)<jupyter_code>lines = []
with open('../data/processed/IEMOCAP/label.csv') as f :
    csv_reader = csv.reader(f)
    lines = [x for x in csv_reader]
with open('../data/processed/IEMOCAP/processed_label.txt', 'w') as f:
    
    with open('../data/processed/IEMOCAP/processed_ids.txt', 'w') as f2:
    
        for line in lines:
            if line[1] == 'ang':
                f.write('0\n')
                f2.write(line[0]+'\n')
            elif line[1] == 'hap':
                f.write('1\n')
                f2.write(line[0]+'\n')
            elif line[1] == 'exc':
                f.write('1\n')
                f2.write(line[0]+'\n')
            elif line[1] == 'sad':
                f.write('2\n')
                f2.write(line[0]+'\n')
            elif line[1] == 'neu':
                f.write('3\n')
                f2.write(line[0]+'\n')
            else :
                f.write('-1\n')
lines = []
with open('../data/processed/IEMOCAP/processed_label.txt') as f :
    lines = f.readlines()
lines = [x.strip() for x in lines]

print len([x for x in lines if x=='0'])
print len([x for x in lines if x=='1'])
print len([x for x in lines if x=='2'])
print len([x for x in lines if x=='3'])<jupyter_output>0
0
0
0<jupyter_text>03 verify<jupyter_code>import csv
in_file = '../data/processed/IEMOCAP/processed_label.txt'
label = []
with open( in_file, 'r') as f:
    label = f.readlines()
    
label_id = [ x.strip() for x in label ]
in_file = '../data/processed/IEMOCAP/label.csv'
label = []
with open( in_file, 'r') as f:
    csv_reader = csv.reader( f )
    label = [x for x in csv_reader]
    
label_id = [ x[0] for x in label]
in_file = '../data/processed/IEMOCAP/processed_tran.csv'
tran = []
with open( in_file, 'r') as f:
    csv_reader = csv.reader( f )
    tran = [x for x in csv_reader]
    
tran_id = [ x[0] for x in tran]
for l, t in zip(label_id, tran_id):
    if l != t:
        print 'ERROR'<jupyter_output><empty_output><jupyter_text>04 statistics<jupyter_code>import numpy as np
in_file = '../data/processed/IEMOCAP/label.csv'
label = []
with open( in_file, 'r') as f:
    csv_reader = csv.reader( f )
    label = [x for x in csv_reader]
    
label_cat = [ x[1] for x in label]
count = np.zeros( len(category), dtype=np.int )

for cat in label_cat:
    count[ category[cat] ] = count[ category[cat] ] + 1
for key in category.keys():
    print key + '\t:' + str(count[ category[key] ])<jupyter_output>oth	:0
ang	:0
exc	:0
fea	:0
xxx	:0
sad	:0
sur	:0
fru	:0
hap	:0
neu	:0
dis	:0
{'fraction_non_alphanumeric': 0.13543543543543543, 'fraction_numerical': 0.014114114114114115, 'mean_word_length': 1.8129222972972974, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '52324', 'license': 'MIT', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2232, 'n_words': 541, 'path': 'preprocessing/IEMOCAP_00_extract_label_transcription.ipynb', 'repo_name': 'ggzhang0071/multimodal-speech-emotion', 'size': '6660'}