<jupyter_start><jupyter_text>Temperatures Note: for this demo to work properly, you'll need to install [statsmodels](http://http://statsmodels.sourceforge.net/)  Import libraries<jupyter_code>import glob
import os
import numpy as np
import pandas as pd
import hypertools as hyp
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib as mpl
import statsmodels

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Read in data and parse text<jupyter_code>files = glob.glob('data/temperatures/*.txt')
results = pd.DataFrame()
locs = pd.DataFrame()
for i in files:
    file_name = os.path.splitext(os.path.basename(i))[0]
    n = 20
    templocs = pd.DataFrame({'City': [], 'Lat':[], 'Long':[]})
    with open(i, 'r') as f:
        for line in f:
            ### parse text: find monthly absolute values
            if str('Jan    Feb    Mar    Apr    May    Jun    Jul    Aug    Sep    Oct    Nov    Dec') in line:
                months = line.split()[1:]
                month_temps = f.next().split()[1:]
                month_temps = np.array(map(float, month_temps))
            ### parse text: find latitude and longitude
            if str('method for the location: ') in line:
                blank = f.next()
                locations = f.next().split()[1:]
                if str('S') in locations[1]:
                    latitude = -1 * float(locations[0])
                else:
                    latitude = float(locations[0])
                if str('W') in locations[3]:
                    longitude = -1 * float(locations[2])
                else:
                    longitude = float(locations[2])
                templocs = pd.DataFrame({'City': file_name, 'Lat': latitude, 'Long': longitude},
                                        index=[0])
    temp = pd.read_csv(i, skiprows= 200, delim_whitespace=True, header=None, usecols=range(3))
    temp.columns = ["Year", "Month",file_name+str('_anomaly')]
    temp = temp[temp["Year"] > 1849]
    # noinspection PyUnboundLocalVariable
    temp[file_name] = month_temps[temp['Month'].values - 1] + temp[file_name+str('_anomaly')]
    if results.empty:
        results = temp
    else:
        results = pd.merge(results, temp, how='left', on=['Year', 'Month'])
    locs = pd.concat([locs, templocs])<jupyter_output><empty_output><jupyter_text>Temperature dataframe<jupyter_code>results.head()<jupyter_output><empty_output><jupyter_text>Locations dataframe<jupyter_code>print(locs)<jupyter_output>City    Lat    Long
0      Bangkok  13.66   99.91
0       Bombay  18.48   72.68
0        Cairo  29.74   31.38
0    Cape_Town -32.95   18.19
0      Chicago  42.59  -87.27
0     Istanbul  40.99   29.82
0       London  52.24   -0.00
0  Los_Angeles  34.56 -118.70
0       Mexico  20.09  -98.96
0     Montreal  45.81  -72.69
0       Moscow  55.45   36.85
0     New_York  40.99  -74.56
0         Rome  42.59   13.09
0     Santiago -32.95  -69.89
0    Sao_Paulo -23.31  -46.31
0        Seoul  37.78  126.10
0     Shanghai  31.35  120.63
0      Somalia   2.41   45.80
0       Sydney -34.56  151.78
0        Tokyo  36.17  139.23<jupyter_text>Clean up NAs and convert to numpy array to pass into hyperplot<jupyter_code>results = results.dropna()
temps = results.as_matrix(locs['City'].as_matrix())
years = results.as_matrix(['Year'])
month = results.as_matrix(['Month'])<jupyter_output><empty_output><jupyter_text>Static 3D image: plot the high-dimensional data in a low dimensional space<jupyter_code>geo = hyp.plot(temps, group=years.flatten(), palette='RdBu_r', normalize='across')<jupyter_output><empty_output><jupyter_text>Static 2D image<jupyter_code>geo.plot(ndims=2)<jupyter_output><empty_output><jupyter_text>Color Bar<jupyter_code>fig = plt.figure(figsize=(8, 3))
ax1 = fig.add_axes([0.05, 0.80, 0.9, 0.15])
cmap = mpl.cm.RdBu_r
norm = mpl.colors.Normalize(vmin=1875, vmax=2013)
cb1 = mpl.colorbar.ColorbarBase(ax1, cmap=cmap,
                                norm=norm,
                                orientation='horizontal')
cb1.set_label('Years')
plt.show()<jupyter_output><empty_output><jupyter_text>Lowess predictions of PCA values by year<jupyter_code>sns.set_style('darkgrid')
sns.set_palette(palette='muted')
reduced_data = hyp.reduce(temps, reduce='PCA', ndims = 3)

results['PCA 1'] = reduced_data[:,0]
results['PCA 2'] = reduced_data[:,1]
results['PCA 3'] = reduced_data[:,2]

PCA1_df = results[['Year', 'PCA 1']]
PCA2_df = results[['Year', 'PCA 2']]
PCA3_df = results[['Year', 'PCA 3']]
sns.regplot(x='Year', y='PCA 1', data=PCA1_df.groupby(['Year']).mean().reset_index(), lowess=True)
sns.regplot(x='Year', y='PCA 2', data=PCA2_df.groupby(['Year']).mean().reset_index(), lowess=True)
sns.regplot(x='Year', y='PCA 3', data=PCA3_df.groupby(['Year']).mean().reset_index(), lowess=True)
plt.show()<jupyter_output><empty_output><jupyter_text>Lowess predictions of PCA values by  average yearly temperature<jupyter_code>temp_PCA = pd.DataFrame()
temp_PCA['ave_temp'] = results.groupby(['Year']).mean()[locs['City']].mean(axis=1)
temp_PCA = temp_PCA.join(PCA1_df.groupby(['Year']).mean(), how = 'outer')
temp_PCA = temp_PCA.join(PCA2_df.groupby(['Year']).mean(), how = 'outer')
temp_PCA = temp_PCA.join(PCA3_df.groupby(['Year']).mean(), how = 'outer')
sns.regplot(x='ave_temp', y='PCA 1', data = temp_PCA, lowess=True)
plt.show()
sns.regplot(x='ave_temp', y='PCA 2', data = temp_PCA, lowess=True)
plt.show()
sns.regplot(x='ave_temp', y='PCA 3', data = temp_PCA, lowess=True)
plt.show()<jupyter_output><empty_output><jupyter_text>Linear regression predictions of PCA values by month<jupyter_code>PCA1_df = results[['Month', 'Year', 'PCA 1']]
PCA2_df = results[['Month', 'Year', 'PCA 2']]
PCA3_df = results[['Month', 'Year', 'PCA 3']]
sns.regplot(x='Month', y='PCA 1', data=PCA1_df.groupby(['Month', 'Year']).mean().reset_index(), truncate = True, lowess=True)
sns.regplot(x='Month', y='PCA 2', data=PCA2_df.groupby(['Month', 'Year']).mean().reset_index(), truncate = True, lowess=True)
sns.regplot(x='Month', y='PCA 3', data=PCA3_df.groupby(['Month', 'Year']).mean().reset_index(), truncate = True, lowess=True)
plt.show()
temps_locs = pd.DataFrame()
temps_locs = results.groupby(['Year']).mean()[locs['City']].reset_index()
meltCov = pd.melt(temps_locs,id_vars=['Year'], var_name='Cities', value_name='Average Temperature')
sns.set(style="whitegrid")
sns.set_palette(palette='muted')
g = sns.lmplot(x = 'Year', y = 'Average Temperature', hue="Cities", data = meltCov, fit_reg=False, legend_out=True)
regplot = sns.regplot(x = 'Year', y = 'Average Temperature', data = meltCov, lowess = True, scatter=False, ax=g.axes[0, 0], color='k')
regplot.set(ylim=(0, 30), xlim=(1870,2040))
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16179976162097737, 'fraction_numerical': 0.04261025029797378, 'mean_word_length': 3.771144278606965, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '434495', 'license': 'MIT', 'n_tokens_mistral': 2662, 'n_tokens_neox': 2482, 'n_words': 501, 'path': 'temperatures.ipynb', 'repo_name': 'rsumner33/hypertools-paper-notebooks', 'size': '6712'}