<commit_before>import os
import time
import sys

from ....utils import load_data as ld
from ....utils.model_helpers import LemmaTokenizer
import numpy as np
import pandas as pd
import sklearn
import sklearn.feature_extraction.text
import sklearn.multiclass
import sklearn.linear_model
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer

np.random.seed(1029384756)

if len(sys.argv) == 2:
    df = ld.load_data(nrows=int(sys.argv[1]))
elif len(sys.argv) == 1:
    df = ld.load_data()
else:
    raise Exception('BAD ARGUMENTS')

crime_df = df.ix[df.loc[:, 'OEMC':'TASR'].any(1), :]
crime_df = crime_df.append(df.ix[~df['relevant'], :].sample(n=min(3000, (~df['relevant']).sum()),
                                                            axis=0))

vectorizer = sklearn.feature_extraction.text.CountVectorizer(tokenizer=LemmaTokenizer(),
                                                             binary=True)

clf = sklearn.multiclass.OneVsRestClassifier(
    sklearn.linear_model.LogisticRegression(verbose=1)
)

X = vectorizer.fit_transform(crime_df['bodytext'].values)
Y = crime_df.loc[:, 'OEMC':'TASR'].values

clf.fit(X, Y)
<commit_msg>decrease vocab size to make model file smaller<commit_after>import os
import time
import sys

from ....utils import load_data as ld
from ....utils.model_helpers import LemmaTokenizer
import numpy as np
import pandas as pd
import sklearn
import sklearn.feature_extraction.text
import sklearn.multiclass
import sklearn.linear_model
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer

np.random.seed(1029384756)

if len(sys.argv) == 2:
    df = ld.load_data(nrows=int(sys.argv[1]))
elif len(sys.argv) == 1:
    df = ld.load_data()
else:
    raise Exception('BAD ARGUMENTS')

crime_df = df.ix[df.loc[:, 'OEMC':'TASR'].any(1), :]
crime_df = crime_df.append(
    df.ix[~df['relevant'], :].sample(n=min(3000, (~df['relevant']).sum()),
                                     axis=0)
)

vectorizer = sklearn.feature_extraction.text.CountVectorizer(tokenizer=LemmaTokenizer(),
                                                             binary=True,
                                                             max_features=40000)

clf = sklearn.multiclass.OneVsRestClassifier(
    sklearn.linear_model.LogisticRegression(verbose=1)
)

X = vectorizer.fit_transform(crime_df['bodytext'].values)
Y = crime_df.loc[:, 'OEMC':'TASR'].values

clf.fit(X, Y)

{'fraction_non_alphanumeric': 0.12463887742468015, 'fraction_numerical': 0.018572018159306643, 'mean_word_length': 3.046744574290484, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6647588', 'n_tokens_mistral': 841, 'n_tokens_neox': 783, 'n_words': 161}