<jupyter_start><jupyter_text>Predicting revenue in training set<jupyter_code>from sklearn.linear_model import Ridge
from sklearn import linear_model
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import GridSearchCV
import scipy
import numpy as np

def gsv(train_x,train_y):
    parameters = {'max_depth':[5, 100]}
    clf = GridSearchCV(DecisionTreeRegressor(), parameters)
    clf.fit(train_x, train_y)
    return clf

def train(train_x,train_y):
    clf = Ridge(alpha=0.01)
    #clf=linear_model.Lasso(alpha=0.01)
    #clf=DecisionTreeRegressor(max_depth=10)
    clf.fit(train_x, train_y)
    return clf

def predict(clf, test_x): 
    return clf.predict(test_x)

def cal_mae(y_hat,y):
    return np.mean(abs(y_hat-y))


h = np.array([1,2])
print h
b = np.append(h,[[3,4]])
print b
r = gsv(fvec_train_ps,true_rev_train_ps)
print r
import xml.etree.ElementTree as ET
tree = ET.parse('dataset\\movies-data-v1.0\\movies-data-v1.0\\perscreen-7domains-train-dev.tl.xml')
root_traindev_ps = tree.getroot()
bow_per_movie_train_ps = bow_movie_nltk(root_traindev_ps,0,1147)

true_rev_train_ps = true_rev(0,1147,root_traindev_ps)
# only when changing vocab 
#####################################################################################3
vocab_ps=vocab_nltk(bow_per_movie_train_ps)

fvec_train_ps = fvec(bow_per_movie_train_ps,vocab_ps)
len(vocab_ps)
len(vocab_ps)<jupyter_output><empty_output><jupyter_text>Predicting revenue in development set<jupyter_code>f= open('C:\Users\Hope\NLP 585\project\NLP_fvec\per screen\\fvec_train_ps_nfs.txt', 'r')
x= pickle.load(f)
g= open('C:\Users\Hope\NLP 585\project\NLP_fvec\per screen\\fvec_dev_ps_nfs.txt', 'r')
y= pickle.load(g)
fvec_train_ps_nfs = x
fvec_dev_ps_nfs = y

trained_classifier_nfsi = train(fvec_train_ps_nfsi,true_rev_train_ps)
#############################
# insert prediction code here
pred_rev_dev_ps = predict(trained_classifier,fvec_dev_ps_nfsi) 
mae_rev_dev_ps= cal_mae(pred_rev_dev_ps,true_rev_dev_ps)
############################

print "The mean absolute error for dev set is ", mae_rev_dev_ps
dtree_r = r #used nltk tokenizer,stop word,lemmatizer list to build bow and vocab with correct f_vec<jupyter_output><empty_output><jupyter_text>Storing models, DO NOT Run these cells dtree_6082        mae=6082 , used nltk tokenizer and stop word list to build bow and vocabdtree_6179        mae=6179 , used nltk tokenizer and stop word list to build bow and vocabdtree_6099        mae=6099 , used nltk tokenizer,stop word,lemmatizer list to build bow and vocabdtree_6030        mae=6030 , used nltk tokenizer,stop word,lemmatizer list to build bow and vocabdtree_6028        mae=6028 , used nltk tokenizer,stop word,lemmatizer list to build bow and vocabdtree_r           mae=6074 , used nltk tokenizer,stop word,lemmatizer list to build bow and vocab and grid search cvdtree_6106       used nltk tokenizer,stop word,lemmatizer list to build bow and vocab with correct f_vecdtree_6037       used nltk tokenizer,stop word,lemmatizer list to build bow and vocab with correct f_vec<jupyter_code>dtree_6179 = trained_classifier<jupyter_output><empty_output><jupyter_text>Predicting revenue in test set<jupyter_code>import xml.etree.ElementTree as ET
tree = ET.parse('dataset\\movies-data-v1.0\\movies-data-v1.0\\perscreen-7domains-train-test.tl.xml')
root_traintest_ps = tree.getroot()
bow_per_movie_test_ps = bow_movie_nltk(root_traintest_ps,1147,1401)

# insert prediction code here
# pred_rev_test_ps =
#########################
fvec_test_ps = fvec(bow_per_movie_test_ps,vocab_ps)

#########################
true_rev_test_ps = true_rev(1147,1401,root_traintest_ps)
import pickle

f = open('NLP_fvec\\NFSI\\fvec_test_ps_nf.txt', 'w')
pickle.dump(fvec_test_ps, f)
f.close()

f= open('C:\Users\Hope\NLP 585\project\NLP_fvec\NFSI\\fvec_train_ps_nfs.txt', 'r')
fvec_train_ps_nfs = pickle.load(f)
f.close()
f= open('C:\Users\Hope\NLP 585\project\NLP_fvec\NFSI\\fvec_train_ps_nfsi.txt', 'r')
fvec_train_ps_nfsi = pickle.load(f)
f.close()
f= open('C:\Users\Hope\NLP 585\project\NLP_fvec\NFSI\\fvec_train_ps_nf.txt', 'r')
fvec_train_ps_nf = pickle.load(f)
f.close()
f= open('C:\Users\Hope\NLP 585\project\NLP_fvec\NFSI\\fvec_test_ps_nfs.txt', 'r')
fvec_test_ps_nfs = pickle.load(f)
f.close()
f= open('C:\Users\Hope\NLP 585\project\NLP_fvec\NFSI\\fvec_test_ps_nfsi.txt', 'r')
fvec_test_ps_nfsi = pickle.load(f)
f.close()
f= open('C:\Users\Hope\NLP 585\project\NLP_fvec\NFSI\\fvec_test_ps_nf.txt', 'r')
fvec_test_ps_nf = pickle.load(f)
f.close()
from sklearn.neural_network import MLPRegressor
def mpl(train_x,train_y,test_x):
    clf = MLPRegressor(hidden_layer_sizes=(100, 100,100,100),alpha=0.1)
    clf.fit(train_x, train_y) 
    return clf.predict(test_x)
def cal_mae(y_hat,y):
    return np.mean(abs(y_hat-y))<jupyter_output><empty_output><jupyter_text>Neural MAEs for test set for per screen revenue<jupyter_code>y_hat= mpl(fvec_train_ps_nfsi,true_rev_train_ps,fvec_test_ps_nfsi)

mae= cal_mae(true_rev_test_ps,y_hat)

print "The mean absolute error for test set nfsi is ", mae
y_hat= mpl(fvec_train_ps_nfs,true_rev_train_ps,fvec_test_ps_nfs)

mae= cal_mae(true_rev_test_ps,y_hat)

print "The mean absolute error for test set nfs is ", mae
y_hat= mpl(fvec_train_ps_nf,true_rev_train_ps,fvec_test_ps_nf)

mae= cal_mae(true_rev_test_ps,y_hat)

print "The mean absolute error for test set nf is ", mae<jupyter_output>The mean absolute error for test set nf is  6432.81568768<jupyter_text>Try any trained classifier here<jupyter_code>#############################
# insert prediction code here
pred_rev_test_ps = predict(trained_classifier,fvec_test_ps_nfsi) 
mae_rev_dev_ps= cal_mae(pred_rev_dev_ps,true_rev_dev_ps)
############################

print "The mean absolute error for test set nfsi is ", mae_rev_dev_ps

#############################
# insert prediction code here
pred_rev_test_ps = predict(trained_classifier,fvec_test_ps_nfs) 
mae_rev_dev_ps= cal_mae(pred_rev_dev_ps,true_rev_dev_ps)
############################

print "The mean absolute error for test set nfs is ", mae_rev_dev_ps

#############################
# insert prediction code here
pred_rev_test_ps = predict(trained_classifier,fvec_test_ps_nf) 
mae_rev_dev_ps= cal_mae(pred_rev_dev_ps,true_rev_dev_ps)
############################

print "The mean absolute error for test set nf is ", mae_rev_dev_ps<jupyter_output><empty_output><jupyter_text>Histogram showing depiction of training data<jupyter_code>mean = np.mean(true_rev_train_ps)
std= np.std(true_rev_train_ps)
print mean
print std

import matplotlib.pyplot as plt

fig = plt.figure(figsize=(10, 6))
plt.hist(true_rev_train_ps/10 **3,30)
plt.xlabel("Per Screen Opening Weekend Revenue (in $ Thousand) ")
plt.ylabel("Number of movies")
plt.axvline(x=mean/10**3 , c='r' , label ='Mean revenue = $8447')
plt.axvline(x=(mean+std)/10**3, c='g', label = '1 Standard Deviation from mean')
plt.axvline(x=(mean+2*std)/10**3, c='b', label = '2 Standard Deviations from mean')
plt.axvline(x=(mean+3*std)/10**3, c='y', label = '3 Standard Deviations from mean')
plt.title('Histogram showing distribution of per screen revenue of Training data')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1522433671343242, 'fraction_numerical': 0.02611473815807751, 'mean_word_length': 6.67590618336887, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '360711', 'license': 'MIT', 'n_tokens_mistral': 2979, 'n_tokens_neox': 2839, 'n_words': 539, 'path': 'graph_per screen revenue.ipynb', 'repo_name': 'DRZN/reviewsTakeOnRevenue', 'size': '7199'}