<commit_before>import collections
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats

# read csv file into pandas
data = pd.read_csv('https://spark-public.s3.amazonaws.com/dataanalysis/loansData.csv')
data.dropna(inplace=True)

# count the number of instances of open credit lines
open_credit_freq = collections.Counter(data['Open.CREDIT.Lines'])

# plot the number of open credit lines as a hist
plt.figure()
plt.bar(open_credit_freq.keys(), open_credit_freq.values(), width=1)
plt.show()

# run a Chi-Squared Test on the data
chi, p = stats.chisquare(open_credit_freq.values())
print chi
print p<commit_msg>added freq redux selection on chi^2 data<commit_after>import collections
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats

# read csv file into pandas
data = pd.read_csv('https://spark-public.s3.amazonaws.com/dataanalysis/loansData.csv')
data.dropna(inplace=True)

# count the number of instances of open credit lines
open_credit_freq = collections.Counter(data['Open.CREDIT.Lines']

# plot the number of open credit lines as a hist
plt.figure()
plt.bar(open_credit_freq.keys(), open_credit_freq.values(), width=1)
plt.show()

# Run a Chi-Squared Test on the data
# Step 1: only include categories with freqs greater than or equal to 5
chi_redux = {}
for k in open_credit_freq:
    if open_credit_freq[k] >= 5:
        chi_redux[k] = open_credit_freq[k]
# Step 2: Calculate chi and p-value
chi, p = stats.chisquare(chi_redux.values())
print chi
print p
{'fraction_non_alphanumeric': 0.09847984137475214, 'fraction_numerical': 0.005948446794448116, 'mean_word_length': 5.179591836734694, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6622192', 'n_tokens_mistral': 521, 'n_tokens_neox': 490, 'n_words': 158}