<commit_before>'''
<NAME>, <NAME>
CS 229 Final Project
Main driver for the project. 
'''

import pandas as pd
import numpy as np
from sklearn.cross_validation import train_test_split
from sklearn import cross_validation, linear_model

import wrangle
import features

Load, clean, and wrangle data
df_training, df_test, df_ID_table = wrangle.load_data()
df_training, df_test = wrangle.clean_data(df_training, df_test)
df_training, df_test = wrangle.rename_data(df_training, df_test)
df_training_all, df_test_all = wrangle.merge_data(df_training, df_test, df_ID_table)
# print df_training_all.head()

# Load the data once and save the pickle files. Use df_training[0].to_pickle('./trainingreviews.pkl') etc.

#trainingreviews = pd.io.pickle.read_pickle('./trainingreviews.pkl')
#trainingbusiness = pd.io.pickle.read_pickle('./trainingbusiness.pkl')
#trainingusers = pd.io.pickle.read_pickle('./trainingusers.pkl')

#testreviews = pd.io.pickle.read_pickle('./testreviews.pkl')
#testbusiness = pd.io.pickle.read_pickle('./testbusiness.pkl')
#testusers = pd.io.pickle.read_pickle('./testusers.pkl')



# feature selection
#X, Y = features.quick_and_dirty(df_training[2])
#X, Y = features.not_so_quick(df_training[2],df_training[1],df_training[0])
TrainMatrix = trainingreviews.merge(trainingbusiness,on="business_id")
TrainMatrix = TrainMatrix.merge(trainingusers,on="user_id")
TestMatrix = testreviews.merge(testbusiness,on="business_id")
TestMatrix = TestMatrix.merge(testusers,on="user_id")<commit_msg>Replaced old code<commit_after>'''
<NAME>, <NAME>
CS 229 Final Project
Main driver for the project. 
'''

import pandas as pd
import numpy as np
from sklearn.cross_validation import train_test_split
from sklearn import cross_validation, linear_model

import wrangle
import features

Load, clean, and wrangle data
df_training, df_test, df_ID_table = wrangle.load_data()
df_training, df_test = wrangle.clean_data(df_training, df_test)
df_training, df_test = wrangle.rename_data(df_training, df_test)
df_training_all, df_test_all = wrangle.merge_data(df_training, df_test, df_ID_table)
# print df_training_all.head()

# Load the data once and save the pickle files. Use df_training[0].to_pickle('./trainingreviews.pkl') etc.
trainingreviews = df_training[0]
trainingbusiness = df_training[1]
trainingusers = df_training[2]
testreviews = df_test[0]
testusers = df_test[2]
testbusiness = df_test[1]
#trainingreviews = pd.io.pickle.read_pickle('./trainingreviews.pkl')
#trainingbusiness = pd.io.pickle.read_pickle('./trainingbusiness.pkl')
#trainingusers = pd.io.pickle.read_pickle('./trainingusers.pkl')

#testreviews = pd.io.pickle.read_pickle('./testreviews.pkl')
#testbusiness = pd.io.pickle.read_pickle('./testbusiness.pkl')
#testusers = pd.io.pickle.read_pickle('./testusers.pkl')



# feature selection
#X, Y = features.quick_and_dirty(df_training[2])
#X, Y = features.not_so_quick(df_training[2],df_training[1],df_training[0])
TrainMatrix = trainingreviews.merge(trainingbusiness,on="business_id")
TrainMatrix = TrainMatrix.merge(trainingusers,on="user_id")
TestMatrix = testreviews.merge(testbusiness,on="business_id")
TestMatrix = TestMatrix.merge(testusers,on="user_id")
{'fraction_non_alphanumeric': 0.12409448818897638, 'fraction_numerical': 0.006929133858267717, 'mean_word_length': 7.31413612565445, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1279436', 'n_tokens_mistral': 1104, 'n_tokens_neox': 1088, 'n_words': 207}