<jupyter_start><jupyter_text>Histograms, Means, and Standard Deviations<jupyter_code>h = [63, 66, 71, 65, 70, 66, 67, 65, 67, 74, 64, 75, 68, 67, 70, 73, 66, 70, 72, 62, 68, 
     70, 62, 69, 66, 70, 70, 68, 69, 70, 71, 65, 64, 71, 64, 78, 69, 70, 65, 66, 72, 64]

d = {}
for i in h:
    d[i] = d.get(i, 0)+1

histlist = []
for i in d:
    histlist.append((i, d.get(i)))
    
sorted(histlist)
hist0 = [i for (i,j) in histlist]
hist1 = [j for (i,j) in histlist]
plt.bar(hist0, hist1, width=1)
plt.title("Faculty Heights Histogram")
plt.xlabel("Height")
plt.xticks(np.arange(78-62+2)+62)
plt.ylabel("Frequency")

fig = plt.gcf()
heights_mean = sum(h)/len(h)
heights_stdDev = sqrt(sum([(heights_mean-i)**2 for i in h])//len(h))

heights_mean_auto = np.mean(h)
heights_stdDev_auto = np.std(h)

print [heights_mean, heights_mean_auto] #these are equal
print [heights_stdDev, heights_stdDev_auto] #these are equal<jupyter_output>[477/7, 68.142857142857139]
[1/7*sqrt(1831/3), 3.5292750407024345]<jupyter_text>Correlation<jupyter_code>#not used any more - panda data frames are easier
stringData = []
with open('./stateData.csv','rb') as csvfile:
    stateData = csv.reader(csvfile, delimiter=' ', quotechar='|')
    for line in stateData:
        stringData.append(line)
data = []
for j in range(len(stringData)-1):
    data.append([i for i in stringData[j][0].split(',')])
pd.read_csv('./stateData.csv')
#This was easier than calculating it by hand, and more useful to me.

def findCorrelation(df, test1, test2):
    mean1 = df[test1].mean()
    mean2 = df[test2].mean()
    r=0
    elements = len(df[test1])
    for i in range(elements):
        r+=((df[test1][i])-mean1)*((df[test2][i])-mean2)
    rxy = r/elements/df[test1].std()/df[test2].std()
    return rxy

data = pd.read_csv('./stateData.csv')
findCorrelation(data,'University','Income')
#Create three column vector of (data-mean)/stdDev for 3 given categories in dataframe

row_vectors = []
for i in ['University','Income','Infant Mort']:
    l = data[i]
    l_vector = (l-l.mean())/l.std()
    row_vectors.append(l_vector)

col_vectors = np.transpose(row_vectors)
print col_vectors
row_vectors = np.transpose(col_vectors)
print row_vectors
correlation = sp.imread('./correlation.png')

fig = plt.figure(figsize=(15,15))
plt.grid(False)
plt.imshow(correlation)<jupyter_output><empty_output><jupyter_text>Linear Regression<jupyter_code>t = [53,54,58,66,69,70,71,73,81]
c = [19,26,21,33,31,36,36,38,45]

xi = sum(t)
yi = sum(c)
xsqr = sum([i**2 for i in t])
xiyi = sum([t[i]*c[i] for i in range(len(t))])
n  = len(t)

print xi
print yi
print xsqr
print xiyi
print n
a = [[xsqr, xi],[xi,n]]
a_inv = np.linalg.inv(a)
v = [[xiyi],[yi]]

[[a],[b]] = np.dot(a_inv,v) #find a and b

#find two points on line of best fit for plotting
y1 = a*t[0]+b
y2 = a*t[n-1]+b
plt.plot(t,c)
plt.plot([t[0], t[n-1]],[y1, y2]) #line of best fit 
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1820363387041481, 'fraction_numerical': 0.07199177236887212, 'mean_word_length': 4.7328094302554025, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '63520', 'license': 'MIT', 'n_tokens_mistral': 1397, 'n_tokens_neox': 1218, 'n_words': 287, 'path': 'Day5/bb9_qea_day5_beforeclass.ipynb', 'repo_name': 'laurengulland/QEA-work', 'size': '2917'}