<jupyter_start><jupyter_text>X = vectorizer.fit_transform(cleaned_list)<jupyter_code>X
X.toarray()
X.shape
clf = MultinomialNB()
clf.fit(X, good_data['review'])
from sklearn.pipeline import Pipeline
text_clf = Pipeline([('tfidf', TfidfVectorizer()),
                     ('clf', MultinomialNB()),
])
bad_data = pd.read_csv("bad_reviews.csv")
bad_list = []
for item in bad_data['review']:
    new_entry = []
    # split into words
    tokens = word_tokenize(item)
    # convert to lowercase
    tokens = [w.lower() for w in tokens]
    # remove punctuation and abberations
    table = str.maketrans('', '', string.punctuation)
    stripped = [w.translate(table) for w in tokens]
    words = [word for word in stripped if word.isalpha()]
    # filter out stop words
    stop_words = set(stopwords.words('english'))
    words = [w for w in words if w not in stop_words]
    # words is now a list of the significant words
    sentence = ' '.join(words)
    bad_list.append(sentence)
    
    
y = vectorizer.transform(bad_list)
y.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1234684260131951, 'fraction_numerical': 0.0, 'mean_word_length': 3.72, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '100371', 'license': 'MIT', 'n_tokens_mistral': 367, 'n_tokens_neox': 360, 'n_words': 92, 'path': 'Day3_restart.ipynb', 'repo_name': 'TeamAlphaPrimeReviews/ReviewsToImprove', 'size': '1061'}