<jupyter_start><jupyter_text>Need to change X to X.toarra() for some classifiers<jupyter_code>def documents2feature_vectors(documents):
#     print "Generating feature vectors"
    X= []
    for d in documents:
        fs = vectorizer.features(d)
        fks = sorted(fs.keys())
        fv = [fs[f] for f in fks]
        X.append(fv)
    X = csr_matrix(np.asarray(X))
    return X

def getScores(sub):
    if len(sub)!=0:
        X = documents2feature_vectors(sub)
        scores = clf.predict(X)
        return scores

MODEL_FILENAME = 'politeness-svm.p'
clf = pickle.load(open(MODEL_FILENAME,'rb'))

vectorizer = PolitenessFeatureVectorizer()
adminRequests = pickle.load(open('data/parsed/adminRequests_parsed.p'))

before, after =[], []
i = 0
for admin, reqs in adminRequests.iteritems():
    if i%50==0:
        print i
    if len(reqs['before'])!=0 and len(reqs['after'])!=0:
        req['beforescores'] = getScores(reqs['before'])
        before.append(req['beforescores'])
        req['afterscores'] = getScores(reqs['after'])
        after.append(req['afterscores'])
    i+=1

a_mean, b_mean= [], []
for a in after:
    a_mean.append(np.mean(a))
for b in before:
    b_mean.append(np.mean(b))

print "Average admin politeness:\n"
print "Before Elections -\t", np.mean(b_mean)
print "After Elections -\t", np.mean(a_mean)<jupyter_output>Average admin politeness:

Before Elections -	0.609205391601
After Elections -	0.522061694135
{'fraction_non_alphanumeric': 0.14335421016005567, 'fraction_numerical': 0.025052192066805846, 'mean_word_length': 3.6537216828478964, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '528972', 'license': 'Apache-2.0', 'n_tokens_mistral': 547, 'n_tokens_neox': 516, 'n_words': 110, 'path': 'Figures-Exploration.ipynb', 'repo_name': 'nidishrajendran/computational-politeness', 'size': '1437'}