<jupyter_start><jupyter_text>Loading the data from directory, subdirectory and files<jupyter_code>X = list()
Y = list()
for r,d,f in os.walk(r'''20_newsgroups'''):
    for file in f:
        X.append(os.path.join(r,file))
print(len(X))


for r,d,f in os.walk(r'''20_newsgroups'''):
    for dirc in d:
        Y.append(os.path.join(r,dirc))
print(len(Y))<jupyter_output>19998
20<jupyter_text>Test Train Split<jupyter_code>test_file = list()
train_file = X
for i in range(0,len(train_file),50):
    test_file.append(train_file[i])
for j in test_file:
    train_file.remove(j)
test_file.pop(0)
print(len(test_file))
len(train_file)<jupyter_output>399<jupyter_text>Creating a list of complete stop words<jupyter_code>from nltk.corpus import stopwords
stopwords = stopwords.words('english')
len(stopwords)
stopwords_i = [word.title() for word in stopwords]
stopwords_u = [word.upper() for word in stopwords]
others = [str(i) for i in range(101)] + [str(i)+"." for i in range(101)]
sp = ["''",'""',"'",'"','!','~','`','!','@','#','$','%','^','&','*','/','-','+','_',':)','**','***','_/',
      '----------------------------------------------------------------------',
      '[]','{}','()','\\','|','[',']','(','|>','>>','<<',
')','{','}',',','.','?','=','<=','>=','<','>',':',';','A','B','Q','W','E','R','T','Y','U','I','O','P','S','D','F',
      'G','H','J','K','L','Z','X','C','V','N','M','A.','B.','Q.','W.','E.','R.','T.','Y.','U.','I.','O.','P.',
      'S.','D.','F.','G.','H.','J.','K.','L.','Z.','X.','C.','V.','N.','M.']
stopwords = stopwords + stopwords_i + stopwords_u + others + sp
len(stopwords)<jupyter_output><empty_output><jupyter_text>Building the Vocabulary<jupyter_code>from nltk.tokenize import TweetTokenizer
tknz = TweetTokenizer()
vocab = {}
for k in train_file:
    #with open(k, 'rb') as f:
    #    text = f.read()
        text = tknz.tokenize(open(k,errors = "replace").read())
    #with open(k, encoding="utf8", errors='ignore') as f:
        #text = f.read()
        for t in text:
            if t not in stopwords:
                if vocab.get(t) != None:
                    vocab[t] += 1
                else:
                    vocab[t] = 1
    
print(len(vocab.keys()))
import matplotlib.pyplot as plt
x_axis = list(range(len(vocab.keys())))
y_axis = list(vocab.values()) 
plt.scatter(x_axis,y_axis)
#plt.axis([0,350000,0,50000])
plt.show<jupyter_output><empty_output><jupyter_text>sorting the vocab list in descending order on the basis of values<jupyter_code>voc = vocab
voc = sorted(voc.items(), key=lambda x: x[1], reverse=True) #Sorting the words according to their frequencies in the dataset
col = voc[0:2000] #Choosing top 2000 most occuring words
col = dict(col) 
columns = list(col.keys()) #list of all 2000 words<jupyter_output><empty_output><jupyter_text>creating a np array with  zero values with shape total no. of training files as rows and total no. of vocab selected as columns<jupyter_code>x_train = np.array([[0] * len(col) for i in range(len(train_file))])
x_train.shape<jupyter_output><empty_output><jupyter_text>Traversing through each file again and checking each word if it is in vocab then incrementing its frequency<jupyter_code>for k in range(len(train_file)):
    text = tknz.tokenize(open(train_file[k],errors = 'replace').read())
    for t in text:
        if t in columns:
            pos = columns.index(t)
            x_train[k][pos] += 1
x_train.shape
df_train = pd.DataFrame(x_train,columns = columns)
y_train = list()<jupyter_output><empty_output><jupyter_text>Creating a list of targets (y) for trainging data<jupyter_code>for i in train_file:
    clas = i.split('/')[1]
    y_train.append(clas)
y_train = np.array(y_train)
y_train.shape<jupyter_output><empty_output><jupyter_text>storing the train datasets in csv for submission<jupyter_code>np.savetxt('train_text_targets_2.csv', y_train, delimiter = ',', fmt = "%s")

df_train.to_csv("train_text_20.csv", sep = ',')<jupyter_output><empty_output><jupyter_text>Creating array for x_test and y_test as well<jupyter_code>x_test = np.array([[0] * len(col) for i in range(len(test_file))])
for k in range(len(test_file)):
    text = tknz.tokenize(open(test_file[k],errors = 'replace').read())
    for t in text:
        if t in columns:
            pos = columns.index(t)
            x_test[k][pos] += 1
x_test.shape
df_test = pd.DataFrame(x_test, columns = columns)
y_test = list()
for i in test_file:
    clas = i.split('/')[1]
    y_test.append(clas)
y_test = np.array(y_test)
y_test.shape<jupyter_output><empty_output><jupyter_text>storing the test datasets in csv for submission<jupyter_code>np.savetxt('test_text_targets_2.csv', y_test, delimiter = ',', fmt = "%s")

df_test.to_csv("test_text_20.csv", sep = ',')<jupyter_output><empty_output><jupyter_text>Implementation using sklearn naive bayes<jupyter_code>from sklearn.naive_bayes import MultinomialNB
clf = MultinomialNB()
clf.fit(x_train,y_train)
y_pred_sklearn = clf.predict(x_test)
from sklearn.metrics import classification_report, confusion_matrix
print(classification_report(y_test, y_pred_sklearn))
print(confusion_matrix(y_test, y_pred_sklearn))
'''#  Making a dictionary of groups to give them an integer value which shows class
#  I have classified 20 groups into 6 classes viz, Religion, Computer, Vehicles, Sports, Politics and for_sale
dic = {'alt.atheism':1,
 'comp.graphics':2,
 'comp.os.ms-windows.misc':2,
 'comp.sys.ibm.pc.hardware':2,
 'comp.sys.mac.hardware':2,
 'comp.windows.x':2,
 'misc.forsale':3,
 'rec.autos':4,
 'rec.motorcycles':4,
 'rec.sport.baseball':4,
 'rec.sport.hockey':4,
 'sci.crypt':5,
 'sci.electronics':5,
 'sci.med':5,
 'sci.space':5,
 'soc.religion.christian':1,
 'talk.politics.guns':6,
 'talk.politics.mideast':6,
 'talk.politics.misc':6,
 'talk.religion.misc':1,
 '.DS_Store':3
      }
      '''
'''
y_train_six = list()
y_test_six = list()
#test_file.pop(0)
for group in train_file:
    g = group.split('/')[1]
    y_train_six.append(dic[g])
                    
for gr in test_file:
    g1 = gr.split('/')[1]
    
    
    y_test_six.append(dic[g1])
'''
'''clf.fit(x_train,y_train_six)
y_pred_sklearn_six = clf.predict(x_test)
print(classification_report(y_test_six, y_pred_sklearn_six))
print(confusion_matrix(y_test_six, y_pred_sklearn_six))<jupyter_output>precision    recall  f1-score   support

          1       0.91      0.90      0.91        59
          2       0.99      0.70      0.82       100
          3       0.50      0.95      0.66        21
          4       0.93      0.97      0.95        80
          5       0.86      0.96      0.91        80
          6       0.89      0.85      0.87        60

avg / total       0.90      0.87      0.87       400

[[53  0  0  0  0  6]
 [ 1 70 15  3 11  0]
 [ 0  1 20  0  0  0]
 [ 0  0  2 78  0  0]
 [ 1  0  1  1 77  0]
 [ 3  0  2  2  2 51]]<jupyter_text>implementation of naive bayes from scratch<jupyter_code># Building dictionary
def fit(x_train, y_train):
    dic = {}
    possible_classes = set(y_train)
    num_features = x_train.shape[1]
    for current_class in possible_classes:
        dic[current_class] = {}
        current_class_rows = (y_train == current_class)
        x_train_current = x_train[current_class_rows]
        y_train_current = y_train[current_class_rows]
        dic["total_data"] = len(y_train)
        output = 0
        dic[current_class]["y_total_count"] = len(y_train_current)
        for j in range(1, num_features+1):
            dic[current_class][j] = x_train_current[:,j-1].sum()
            output += dic[current_class][j]
        dic[current_class]["total_count"] = output
         
    return dic        
    
# function for probability
def probability(dictionary,x,current_class):
    op = np.log(dictionary[current_class]["y_total_count"]) - np.log(dictionary["total_data"])
    num_features = len(dictionary[current_class].keys()) - 2
    for j in range(1,num_features + 1):
        if x[j-1] != 0:
            count_current_class_current_word = dictionary[current_class][j] + 1
            count_current_class = dictionary[current_class]["total_count"] + num_features
            current_probability = np.log(count_current_class_current_word) - np.log(count_current_class)
            op = op + current_probability
    return op
def predict_single_class(dictionary, x):
    all_classes = dictionary.keys()
    best_prob = -1000
    best_class = ""
    for current_class in all_classes:
        if current_class == "total_data":
            continue
        current_class_prob = probability(dictionary,x,current_class)
        if current_class_prob > best_prob:
            best_prob = current_class_prob
            best_class = current_class
    return best_class
def predict(dictionary, x_test):
    y_pred = list()
    for x in x_test:
        final_class = predict_single_class(dictionary,x)
        y_pred.append(final_class)
    return y_pred
dic = fit(x_train,y_train)
y_pred_own_algo = predict(dic,x_test) 
print(classification_report(y_test,y_pred_own_algo))
print(confusion_matrix(y_test,y_pred_own_algo))<jupyter_output>precision    recall  f1-score   support

                               0.00      0.00      0.00         0
             alt.atheism       0.74      0.70      0.72        20
           comp.graphics       0.56      0.75      0.64        20
 comp.os.ms-windows.misc       0.00      0.00      0.00        20
comp.sys.ibm.pc.hardware       0.60      0.75      0.67        20
   comp.sys.mac.hardware       0.82      0.90      0.86        20
          comp.windows.x       0.88      0.75      0.81        20
            misc.forsale       0.76      0.95      0.84        20
               rec.autos       1.00      1.00      1.00        20
         rec.motorcycles       0.87      1.00      0.93        20
      rec.sport.baseball       0.95      0.95      0.95        20
        rec.sport.hockey       1.00      0.95      0.97        20
               sci.crypt       0.94      0.80      0.86        20
         sci.electronics       0.78      0.90      0.84        20
         [...]
{'fraction_non_alphanumeric': 0.15736495913892765, 'fraction_numerical': 0.03926649392066972, 'mean_word_length': 2.841883614088821, 'pattern_counts': {'":': 1, '<': 53, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '647178', 'license': 'Apache-2.0', 'n_tokens_mistral': 3862, 'n_tokens_neox': 3642, 'n_words': 737, 'path': '20_newsgroups_text_classification_naive_bayes_project.ipynb', 'repo_name': 'kushalguptadeveloper/Text-Classification-using-Naive-Bayes', 'size': '10034'}