<commit_before>from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
import numpy as np

def vectorize(data):    
    vectorizer = CountVectorizer(analyzer="word", tokenizer=None, preprocessor=None, stop_words=None, max_features=5000)
    vectorized_data = vectorizer.fit_transform(data['Tweet list'])
    vectorized_data = vectorized_data.toarray()
    vocab = vectorizer.get_feature_names()
    
    # view vocab
    #print(vocab)
    
    # view vocab count
    dist = np.sum(vectorized_data, axis = 0)<commit_msg>Update vectorizer.py<commit_after>from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
import numpy as np

def vectorize(data):    
    vectorizer = CountVectorizer(analyzer="word", tokenizer=None, preprocessor=None, stop_words=None, max_features=5000)
    vectorized_data = vectorizer.fit_transform(data['Tweet List'])
    vectorized_data = vectorized_data.toarray()
    vocab = vectorizer.get_feature_names()
    
    # view vocab
    #print(vocab)
    
    # view vocab count
    dist = np.sum(vectorized_data, axis = 0)
{'fraction_non_alphanumeric': 0.09057639524245197, 'fraction_numerical': 0.009149130832570906, 'mean_word_length': 4.553299492385787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3780646', 'n_tokens_mistral': 353, 'n_tokens_neox': 343, 'n_words': 74}