<commit_before>import pandas as pd 
import pprint
import numpy as np

df = pd.read_table('dataset/SMSSpamCollection', sep ='\t',  header =None, names = ['label', 'message'])

df['label'] = df.label.map({'ham':0, 'spam':1})

#print(df.shape[0])
# print(df.head())


documents = ['Hello, how are you!',
                'Win money, win from home.',
                'Call me now.',
                'Hello, Call hello you tomorrow?']

from sklearn.feature_extraction.text import CountVectorizer
count_vector = CountVectorizer()
count_vector.fit(documents)
#pprint.pprint(count_vector.get_feature_names())

doc_arr = count_vector.transform(documents).toarray()
#print(doc_arr)

doc_arr_df = pd.DataFrame(doc_arr, columns = count_vector.get_feature_names())
#print(doc_arr_df)

from sklearn.model_selection import train_test_split as tts

x_message_train, x_message_test, y_label_train, y_label_test = tts(df['message'], df['label'], random_state = 1)

print('total rows = '+format(df.shape[0]))
print('total rows in training = '+ format(x_message_train.shape[0]))
print('total rows in testing = ' + format(x_message_test.shape[0]))
<commit_msg>git problem<commit_after>import pandas as pd 
import pprint
import numpy as np

df = pd.read_table('dataset/SMSSpamCollection', sep ='\t',  header =None, names = ['label', 'message'])

df['label'] = df.label.map({'ham':0, 'spam':1})

#print(df.shape[0])
# print(df.head())


documents = ['Hello, how are you!',
                'Win money, win from home.',
                'Call me now.',
                'Hello, Call hello you tomorrow?']

from sklearn.feature_extraction.text import CountVectorizer
count_vector = CountVectorizer()
count_vector.fit(documents)
#pprint.pprint(count_vector.get_feature_names())

doc_arr = count_vector.transform(documents).toarray()
#print(doc_arr)

doc_arr_df = pd.DataFrame(doc_arr, columns = count_vector.get_feature_names())
#print(doc_arr_df)

from sklearn.model_selection import train_test_split as tts

x_message_train, x_message_test, y_label_train, y_label_test = tts(df['message'], df['label'], random_state = 1)

print('total rows = '+format(df.shape[0]))
print('total rows in training = '+ format(x_message_train.shape[0]))
print('total rows in testing = ' + format(x_message_test.shape[0]))

#another change
{'fraction_non_alphanumeric': 0.14454148471615721, 'fraction_numerical': 0.00611353711790393, 'mean_word_length': 5.125668449197861, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3162966', 'n_tokens_mistral': 791, 'n_tokens_neox': 765, 'n_words': 173}