<jupyter_start><jupyter_text>Maximum likelihood estimation<jupyter_code>from os import listdir
import numpy as np
import pandas as pd
from scipy.stats import norm
pathR='data/ChunkedData_R/'
pathN='data/ChunkedData_NR/'
filesR = listdir(pathR)
filesN = listdir(pathN)
filesR[0]
# Read one CSV file. Drop the date column.
def file_mean (filepath):
    mydata = pd.read_csv(filepath)
    # Drop the date column.
    mydata = mydata.drop('Date',axis=1) 
    # Transpose column of mean values into a row.
    mymean = mydata.mean(axis=0).to_frame().T
    return mymean
# Read directory of CSV files. Retain only one row per file = column averages.
def mean_per_file (directory):
    files = listdir(directory)
    means = pd.DataFrame()
    for fp in files:
        dfp = directory+fp
        m = file_mean(dfp)
        # Let Pandas number the rows sequentially.
        means = means.append(m,ignore_index=True)
    return means
meansR = mean_per_file(pathR)
meansN = mean_per_file(pathN)
meansAll = pd.concat((meansR,meansN),ignore_index=True)
def make_question_features(X):
    c = X.columns[:13] # Excel columns B-N
    z = X[c]
    return np.asarray(z)
def make_physiol_features(X):
    c = X.columns[249:] # Excel columns IQ-IZ
    z = X[c]
    return np.asarray(z)
X_questionFeatures_R = make_question_features(meansR)
X_questionFeatures_N = make_question_features(meansN)
X_physiolFeatures_R = make_physiol_features(meansR)
X_physiolFeatures_N = make_physiol_features(meansN)
# Mean vector should have 13 features
np.mean(X_questionFeatures_R,axis=0)
# Confirm the mean for the first feature
s = 0
num = len(X_questionFeatures_R)
for vec in X_questionFeatures_R:
    v = vec[0]
    s = s + v
print ("mean=",s/num)
# Get four mean vectors
mean_Q_R = np.mean(X_questionFeatures_R,axis=0)
mean_Q_N = np.mean(X_questionFeatures_N,axis=0)
mean_P_R = np.mean(X_physiolFeatures_R,axis=0)
mean_P_N = np.mean(X_physiolFeatures_N,axis=0)
print("Features=BN, Class=R")
print(mean_Q_R)
print("Features=BN, Class=N")
print(mean_Q_N)
print("Features=IQIZ, Class=R")
print(mean_P_R)
print("Features=IQIZ, Class=N")
print(mean_P_N)
# Get four covariance matrices
# Get four mean vectors
cov_Q_R = np.cov(X_questionFeatures_R,rowvar=False)
cov_Q_N = np.cov(X_questionFeatures_N,rowvar=False)
cov_P_R = np.cov(X_physiolFeatures_R,rowvar=False)
cov_P_N = np.cov(X_physiolFeatures_N,rowvar=False)
print("Features=BN, Class=R")
print(cov_Q_R)
print("Features=BN, Class=N")
print(cov_Q_N)
print("Features=IQIZ, Class=R")
print(cov_P_R)
print("Features=IQIZ, Class=N")
print(cov_P_N)
print(mean_Q_R.shape)
print(cov_Q_R.shape)
print(X_questionFeatures_R.shape)
print(X_questionFeatures_R.shape[0])
norm.pdf(X_questionFeatures_R[0],mean_Q_R,cov_Q_R)
D_Q_R = norm(mean_Q_R,cov_Q_R)
print(D_Q_R.pdf(X_questionFeatures_R.shape[0]))
def pdf_multivariate_gauss(x, mu, cov):
    part1 = 1 / ( ((2* np.pi)**(len(mu)/2)) * (np.linalg.det(cov)**(1/2)) )
    part2 = (-1/2) * ((x-mu).T.dot(np.linalg.inv(cov))).dot((x-mu))
    return float(part1 * np.exp(part2))
print(pdf_multivariate_gauss(X_questionFeatures_R.shape[0],mean_Q_R,cov_Q_R))<jupyter_output>nan
{'fraction_non_alphanumeric': 0.12751677852348994, 'fraction_numerical': 0.010226909555768616, 'mean_word_length': 5.688034188034188, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '122839', 'license': 'MIT', 'n_tokens_mistral': 1274, 'n_tokens_neox': 1225, 'n_words': 235, 'path': 'PR677/Opioid09_MLE.ipynb', 'repo_name': 'ShepherdCode/ShepherdML', 'size': '3129'}