<jupyter_start><jupyter_text>Raw tweets<jupyter_code>xData = data.text
yData = data.target
folds = data.fold<jupyter_output><empty_output><jupyter_text>Count Vectorizer<jupyter_code>fe     = CountVectorizer(max_features=5000, ngram_range=(1,3))
clf    = MultinomialNB()
metric = accuracy_score

score  = get_cross_val_score(xData, yData, folds, fe, clf, metric)
print(score)
hyper_params = {
    'max_features':[100, 1000, 2500, 5000, 7500],
    'max_ngram':[1,2,3,4],
}
runs = RunBuilder.get_runs(hyper_params)
def CountVectorizer_runs():
    results_CountVectorizer = []
    for run in runs:
        fe     = CountVectorizer(max_features=run['max_features'], ngram_range=(1,run['max_ngram']))
        clf    = MultinomialNB()
        metric = accuracy_score

        score  = np.mean(get_cross_val_score(xData, yData, folds, fe, clf, metric))
        result = {**run, 'score':score}
        results_CountVectorizer.append(result)

        display(pd.DataFrame(results_CountVectorizer))
        clear_output(wait=True)
    clear_output(wait=True)
    return results_CountVectorizer
results_CountVectorizer = CountVectorizer_runs()
pd.DataFrame(results_CountVectorizer).sort_values('score',ascending=False).head()<jupyter_output><empty_output><jupyter_text>Tfidf Vectorizer<jupyter_code>fe     = TfidfVectorizer(max_features=5000, ngram_range=(1,3))
clf    = MultinomialNB()
metric = accuracy_score
score  = get_cross_val_score(xData, yData, folds, fe, clf, metric)
print(score)
def TfidfVectorizer_runs():
    results_TfidfVectorizer = []
    for run in runs:
        fe     = TfidfVectorizer(max_features=run['max_features'], ngram_range=(1,run['max_ngram']))
        clf    = MultinomialNB()
        metric = accuracy_score

        score  = np.mean(get_cross_val_score(xData, yData, folds, fe, clf, metric))
        result = {**run, 'score':score}
        results_TfidfVectorizer.append(result)

        display(pd.DataFrame(results_TfidfVectorizer))
        clear_output(wait=True)
    clear_output(wait=True)
    return results_TfidfVectorizer
results_TfidfVectorizer = TfidfVectorizer_runs()
pd.DataFrame(results_TfidfVectorizer).sort_values('score',ascending=False).head()<jupyter_output><empty_output><jupyter_text>Pre-Processing<jupyter_code># pre-processing tweets
stemmer = PorterStemmer()
xData = xData.map(lambda tweet:preProc(tweet, stemmer))
results_CountVectorizer_proc = CountVectorizer_runs()
pd.DataFrame(results_CountVectorizer_proc).sort_values('score',ascending=False).head()
results_TfidfVectorizer_proc = TfidfVectorizer_runs()
pd.DataFrame(results_TfidfVectorizer_proc).sort_values('score',ascending=False).head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11764705882352941, 'fraction_numerical': 0.013862869988759834, 'mean_word_length': 5.828644501278772, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '469955', 'license': 'MIT', 'n_tokens_mistral': 986, 'n_tokens_neox': 941, 'n_words': 126, 'path': '02-NLP_with_Disaster_Tweets/_02_vectorizer_tuning.ipynb', 'repo_name': 'L-ashwin/Exploring-ml', 'size': '2669'}