<jupyter_start><jupyter_text>Index exploration This notebook explores the indicies computed from sound files in a pumilio database.  Required packages pandas numpy matplotlib pymilio  Import statements<jupyter_code>import pandas
from Pymilio import database
import numpy as np
import matplotlib.pylab as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Connect to database<jupyter_code>db = database.Pymilio_db_connection(user='pumilio',
                                    database='pumilio',
                                    read_default_file='~/.my.cnf.pumilio')
Sounds = db.fetch_as_pandas_df(table='Sounds', fields=['SoundID', 'SiteID', 'ColID']).set_index('SoundID')
Sites = db.fetch_as_pandas_df(table='Sites', fields=['SiteID', 'ID', 'SiteName']).set_index('SiteID')<jupyter_output><empty_output><jupyter_text>NDSI<jupyter_code>IndexNDSI = db.fetch_as_pandas_df(table='IndexNDSI', fields=['Sound', 'ndsi_left', 'ndsi_right', 'biophony_left', 'biophony_right', 'anthrophony_left', 'anthrophony_right']).set_index('Sound')
ndsi = IndexNDSI.join(Sounds).join(Sites, on='SiteID').drop('SiteID', axis=1)
ndsi_collection1 = ndsi.groupby('ColID').get_group(1)
ndsi_collection1_byID = ndsi_collection1.groupby('ID')
plt.figure(figsize=(15,10))
for name, group in ndsi_collection1_byID:
    plt.plot(group['ID'].as_matrix(), group['ndsi_left'].as_matrix(), 'k-')
    plt.scatter(group['ID'].as_matrix(), group['ndsi_left'].as_matrix(), color='black', marker='.')
plt.xlim(0, 31)
plt.ylim(-1, 1)
plt.xlabel('Sites')
plt.ylabel('NDSI')
plt.title('ndsi_left')
xticks = [i for i in range(1, len(ndsi_collection1_byID)+1)]
xticklabels = Sites.sort_values(by='ID')['SiteName'].iloc[1:31].as_matrix()
xticklabels = ["{0} - {1}".format(xticklabels[i-1], i) for i in xticks]
plt.xticks(xticks, xticklabels, rotation='vertical')
plt.grid()
plt.figure(figsize=(15,10))
for name, group in ndsi_collection1_byID:
    plt.plot(group['ID'].as_matrix(), group['ndsi_right'].as_matrix(), 'k-')
    plt.scatter(group['ID'].as_matrix(), group['ndsi_right'].as_matrix(), color='black', marker='.')
plt.xlim(0, 31)
plt.ylim(-1, 1)
plt.xlabel('Sites')
plt.ylabel('NDSI')
plt.title('ndsi_right')
xticks = [i for i in range(1, len(ndsi_collection1_byID)+1)]
xticklabels = Sites.sort_values(by='ID')['SiteName'].iloc[1:31].as_matrix()
xticklabels = ["{0} - {1}".format(xticklabels[i-1], i) for i in xticks]
plt.xticks(xticks, xticklabels, rotation='vertical')
plt.grid()
plt.figure(figsize=(15,10))
for name, group in ndsi_collection1_byID:
    x = (group['ID'].as_matrix()) - 0.1
    y = group['ndsi_left'].as_matrix()
    plt.plot(x, y, 'r-')
    plt.scatter(x, y, color='red', marker='.')
    x = (group['ID'].as_matrix()) + 0.1
    y = group['ndsi_right'].as_matrix()
    plt.plot(x, y, 'b-')
    plt.scatter(x, y, color='blue', marker='.')
plt.xlim(0, 31)
plt.ylim(-1, 1)
plt.xlabel('Sites')
plt.ylabel('NDSI')
plt.title('ndsi_left and ndsi_right')
xticks = [i for i in range(1, len(ndsi_collection1_byID)+1)]
xticklabels = Sites.sort_values(by='ID')['SiteName'].iloc[1:31].as_matrix()
xticklabels = ["{0} - {1}".format(xticklabels[i-1], i) for i in xticks]
plt.xticks(xticks, xticklabels, rotation='vertical')
plt.grid()
ordered_list = [23, 30, 22,  2, 17, 19, 11,  4,  9, 10, 25, 13, 24, 12, 14, 27,  1,
       28,  6, 18,  8, 15, 20,  7, 26, 16,  5,  3, 21, 29]

plt.figure(figsize=(15,10))
for name, group in ndsi_collection1_byID:
#for n in ordered_list:
 #   group = ndsi_collection1_byID.get_group(n)
    x = (group['ID'].as_matrix())
    y_left = group['biophony_left'].as_matrix()
    #plt.plot(x, y, 'r-')
    #plt.scatter(x, y, color='red', marker='.')
    y_right = group['biophony_right'].as_matrix()
    y = np.append(y_left, y_right)
    x = np.append(x, x)
    plt.plot(x, y, 'k-')
    plt.scatter(x, y, color='gray', marker='.')
    # average
    avg = y.mean()
    plt.plot(x[0], avg, 'ro')
plt.xlim(0, 31)
plt.ylim(0, 2.5)
plt.xlabel('Sites')
plt.ylabel('biophony')
plt.title('biophony_left and biophony_right')
#xticks = [i for i in range(1, len(ndsi_collection1_byID)+1)]
#xticklabels = Sites.sort_values(by='ID')['SiteName'].iloc[1:31].as_matrix()
#xticklabels = ["{0} - {1}".format(xticklabels[i-1], i) for i in xticks]
#plt.xticks(xticks, xticklabels, rotation='vertical')
#plt.grid()
#plt.savefig('/Users/Jake/Desktop/biophony_nosort.png', format='png')
plt.figure(figsize=(15,10))
for name, group in ndsi_collection1_byID:
    x = (group['ID'].as_matrix()) - 0.1
    y = group['anthrophony_left'].as_matrix()
    plt.plot(x, y, 'r-')
    plt.scatter(x, y, color='red', marker='.')
    x = (group['ID'].as_matrix()) + 0.1
    y = group['anthrophony_right'].as_matrix()
    plt.plot(x, y, 'b-')
    plt.scatter(x, y, color='blue', marker='.')
plt.xlim(0, 31)
#plt.ylim(0, 2.5)
plt.xlabel('Sites')
plt.ylabel('anthrophony')
plt.title('anthrophony_left and anthrophony_right')
xticks = [i for i in range(1, len(ndsi_collection1_byID)+1)]
xticklabels = Sites.sort_values(by='ID')['SiteName'].iloc[1:31].as_matrix()
xticklabels = ["{0} - {1}".format(xticklabels[i-1], i) for i in xticks]
plt.xticks(xticks, xticklabels, rotation='vertical')
plt.grid()<jupyter_output><empty_output><jupyter_text>ACI<jupyter_code>IndexACI = db.fetch_as_pandas_df(table='IndexACI', fields=['Sound', 'AciTotAll_left_bymin', 'AciTotAll_right_bymin']).set_index('Sound')
aci = IndexACI.join(Sounds).join(Sites, on='SiteID').drop('SiteID', axis=1)
aci_collection1 = aci.groupby('ColID').get_group(1)
aci_collection1_byID = aci_collection1.groupby('ID')
plt.figure(figsize=(15,10))
for name, group in aci_collection1_byID:
    x = (group['ID'].as_matrix()) - 0.1
    y = group['AciTotAll_left_bymin'].as_matrix()
    plt.plot(x, y, 'r-')
    plt.scatter(x, y, color='red', marker='.')
    x = (group['ID'].as_matrix()) + 0.1
    y = group['AciTotAll_right_bymin'].as_matrix()
    plt.plot(x, y, 'b-')
    plt.scatter(x, y, color='blue', marker='.')
plt.xlim(0, 31)
plt.xlabel('Sites')
plt.ylabel('ACI')
plt.title('AciTotAll_left_bymin and AciTotAll_right_bymin')
xticks = [i for i in range(1, len(aci_collection1_byID)+1)]
xticklabels = Sites.sort_values(by='ID')['SiteName'].iloc[1:31].as_matrix()
xticklabels = ["{0} - {1}".format(xticklabels[i-1], i) for i in xticks]
plt.xticks(xticks, xticklabels, rotation='vertical')
plt.grid()<jupyter_output><empty_output><jupyter_text>ADI<jupyter_code>IndexADI = db.fetch_as_pandas_df(table='IndexADI', fields=['Sound', 'adi_left', 'adi_right']).set_index('Sound')
adi = IndexADI.join(Sounds).join(Sites, on='SiteID').drop('SiteID', axis=1)
adi_collection1 = adi.groupby('ColID').get_group(1)
adi_collection1_byID = adi_collection1.groupby('ID')
plt.figure(figsize=(15,10))
for name, group in adi_collection1_byID:
    x = (group['ID'].as_matrix()) - 0.1
    y = group['adi_left'].as_matrix()
    plt.plot(x, y, 'r-')
    plt.scatter(x, y, color='red', marker='.')
    x = (group['ID'].as_matrix()) + 0.1
    y = group['adi_right'].as_matrix()
    plt.plot(x, y, 'b-')
    plt.scatter(x, y, color='blue', marker='.')
plt.xlim(0, 31)
#plt.ylim(-1, 1)
plt.xlabel('Sites')
plt.ylabel('ADI')
plt.title('adi_left and adi_right')
xticks = [i for i in range(1, len(adi_collection1_byID)+1)]
xticklabels = Sites.sort_values(by='ID')['SiteName'].iloc[1:31].as_matrix()
xticklabels = ["{0} - {1}".format(xticklabels[i-1], i) for i in xticks]
plt.xticks(xticks, xticklabels, rotation='vertical')
plt.grid()<jupyter_output><empty_output><jupyter_text>AEI<jupyter_code>IndexAEI = db.fetch_as_pandas_df(table='IndexAEI', fields=['Sound', 'aei_left', 'aei_right']).set_index('Sound')
aei = IndexAEI.join(Sounds).join(Sites, on='SiteID').drop('SiteID', axis=1)
aei_collection1 = aei.groupby('ColID').get_group(1)
aei_collection1_byID = aei_collection1.groupby('ID')
plt.figure(figsize=(15,10))
for name, group in aei_collection1_byID:
    x = (group['ID'].as_matrix()) - 0.1
    y = group['aei_left'].as_matrix()
    plt.plot(x, y, 'r-')
    plt.scatter(x, y, color='red', marker='.')
    x = (group['ID'].as_matrix()) + 0.1
    y = group['aei_right'].as_matrix()
    plt.plot(x, y, 'b-')
    plt.scatter(x, y, color='blue', marker='.')
plt.xlim(0, 31)
#plt.ylim(-1, 1)
plt.xlabel('Sites')
plt.ylabel('AEI')
plt.title('aei_left and aei_right')
xticks = [i for i in range(1, len(aei_collection1_byID)+1)]
xticklabels = Sites.sort_values(by='ID')['SiteName'].iloc[1:31].as_matrix()
xticklabels = ["{0} - {1}".format(xticklabels[i-1], i) for i in xticks]
plt.xticks(xticks, xticklabels, rotation='vertical')
plt.grid()<jupyter_output><empty_output><jupyter_text>BAI<jupyter_code>IndexBAI = db.fetch_as_pandas_df(table='IndexBAI', fields=['Sound', 'left_area', 'right_area']).set_index('Sound')
bai = IndexBAI.join(Sounds).join(Sites, on='SiteID').drop('SiteID', axis=1)
bai_collection1 = bai.groupby('ColID').get_group(1)
bai_collection1_byID = bai_collection1.groupby('ID')
plt.figure(figsize=(15,10))
for name, group in bai_collection1_byID:
    x = (group['ID'].as_matrix()) - 0.1
    y = group['left_area'].as_matrix()
    plt.plot(x, y, 'r-')
    plt.scatter(x, y, color='red', marker='.')
    x = (group['ID'].as_matrix()) + 0.1
    y = group['right_area'].as_matrix()
    plt.plot(x, y, 'b-')
    plt.scatter(x, y, color='blue', marker='.')
plt.xlim(0, 31)
#plt.ylim(-1, 1)
plt.xlabel('Sites')
plt.ylabel('BAI')
plt.title('left_area and right_area')
xticks = [i for i in range(1, len(bai_collection1_byID)+1)]
xticklabels = Sites.sort_values(by='ID')['SiteName'].iloc[1:31].as_matrix()
xticklabels = ["{0} - {1}".format(xticklabels[i-1], i) for i in xticks]
plt.xticks(xticks, xticklabels, rotation='vertical')
plt.grid()<jupyter_output><empty_output><jupyter_text>SSS (soundscapespec)<jupyter_code>IndexSSS = db.fetch_as_pandas_df(table='IndexSSS', fields=['Sound', 'frequency_power']).set_index('Sound')
sss = IndexSSS.join(Sounds).join(Sites, on='SiteID').drop('SiteID', axis=1)
sss_collection1 = sss.groupby('ColID').get_group(1)
sss_collection1_byID = sss_collection1.groupby('ID')
sss_group1 = sss_collection1_byID.get_group(1)
sss_group2 = sss_collection1_byID.get_group(2)
sss_group1
plt.figure(figsize=(15,30))
count = 1
for row in sss_group2.iterrows():
    plt.subplot(len(sss_group1), 1, count)
    heights = [ float(n) for n in eval(row[1]['frequency_power']) ]
    lefts = [ n for n in range(1, len(heights)+1) ]
    plt.bar(lefts, heights)
    plt.xlim(1, 24)
    plt.ylim(0, 1)
    plt.xticks( np.array(lefts)+0.5, [ str(n) for n in lefts ])
    count = count + 1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19933554817275748, 'fraction_numerical': 0.028856193640246795, 'mean_word_length': 5.77556270096463, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '572019', 'license': 'MIT', 'n_tokens_mistral': 4572, 'n_tokens_neox': 4381, 'n_words': 720, 'path': 'archive/Index exploration.ipynb', 'repo_name': 'jacobdein/alpine-soundscapes', 'size': '10535'}