<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
import statsmodels.api as sm<jupyter_output><empty_output><jupyter_text>Read CSV<jupyter_code>df = pd.read_csv("allyear.csv")
df2 = df.corr().abs()
df3 = df2[df2['POP_SNAP'] > 0.7]
df3<jupyter_output><empty_output><jupyter_text>SNAP Cutoff<jupyter_code>drop1 = ['population','unemployedRate', 'povertyRate','medianIncome', 'avgmealmon', 'pplweekmon', 'drivealone', 'carpooled', 'publicTrans','meanHouseIncome', 'yeshealth','privatehealth']
df3.drop(drop1, axis=1, inplace=True)
df3<jupyter_output><empty_output><jupyter_text>Variable Cutoff<jupyter_code>upper = df3.where(np.triu(np.ones(df3.shape), k=1).astype(bool))

# Find features with correlation greater than 0.83
to_drop = [column for column in upper.columns if any(upper[column] > 0.83)]

# Drop features 
df3.drop(to_drop, axis=1, inplace=True)
df3
drop3 = ['nohealth', 'POP_SNAP']
df3.drop(drop3, inplace=True)
df3
opvars = ['homeless','walked','publichealth','POP_SNAP']
df = df[opvars]
df
df.to_csv('method1D.csv')
df.cov()
df.var()<jupyter_output><empty_output><jupyter_text>Multiple Linear Regression<jupyter_code>df = pd.read_csv("method1D.csv")
df.drop(columns=df.columns[0], axis=1, inplace=True)
df
x = df.drop(['POP_SNAP'],axis=1).values
y = df['POP_SNAP'].values
# splitting dataset into training and test set

x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.3, random_state=0)
# training model on training set
ml = LinearRegression()
ml.fit(x_train,y_train)
# predict test set results
y_pred=ml.predict(x_test)
print(y_pred)
#ml.predict([[263670,4.875000,0.111,3.43,62.840000,4174]])
# scatterplot of results
plt.figure(figsize=(8,5))
plt.scatter(y_test,y_pred)
plt.xlabel('Actual', fontsize=16)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.ylabel('Predicted', fontsize=16)
plt.title('Method 1D', fontsize=16)
pred_y_df = pd.DataFrame({'Actual Value':y_test,'Predicted value':y_pred,'Difference':y_test-y_pred})
pred_y_df[0:20]
# define our intput
X2 = sm.add_constant(x)

# create a OLS model
model = sm.OLS(y, X2)

# fit the data
est = model.fit()

print(est.summary())<jupyter_output>OLS Regression Results                            
==============================================================================
Dep. Variable:                      y   R-squared:                       0.783
Model:                            OLS   Adj. R-squared:                  0.776
Method:                 Least Squares   F-statistic:                     121.3
Date:                Sat, 31 Jul 2021   Prob (F-statistic):           2.39e-33
Time:                        14:39:10   Log-Likelihood:                -1160.8
No. Observations:                 105   AIC:                             2330.
Df Residuals:                     101   BIC:                             2340.
Df Model:                           3                                         
Covariance Type:            nonrobust                                         
==============================================================================
                 coef    std err          t      P>|[...]
{'fraction_non_alphanumeric': 0.1682356513817188, 'fraction_numerical': 0.040996052232007286, 'mean_word_length': 3.1643489254108723, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '238549', 'license': 'MIT', 'n_tokens_mistral': 1206, 'n_tokens_neox': 1107, 'n_words': 208, 'path': 'method1/metric1D.ipynb', 'repo_name': 'myracui5/foodinsecurity', 'size': '3293'}