<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
import matplotlib as plt
import re

df = pd.read_csv("/content/Apple Sentiment Tweets.csv")
df.head(20)
f = open('/content/stop_words.txt','r')

words = f.read()
words = words.split('\n')
print(words)
numbers = ['0','1','2','3','4','5','6','7','8','9']
punc_marks = ['?','!','#', '^','$','%',';',':','/',"_","-",'.']

print(words)
import regex as re
tweets = df['text'].tolist()
new = ''
new_tweets = []
# print(tweets[0])
num_pattern = '[0-9]'
user_pattern = '@\S+'
punctuation_pattern = '[^\w\s]'
hyperlink_pattern = 'http\S*'



for tweet in tweets:
  # print('------new------')
  # print(f'before ---> {tweet}')
  new = re.sub(user_pattern, '', tweet)
  
  new = re.sub(hyperlink_pattern, '', new)
 
  new = re.sub(num_pattern, '',new)
  
  new = re.sub(punctuation_pattern, '', new)
  new = new.replace("\r\n",' ')
  new = new.lower()
  # print(f'after ---> {new}')
  new_tweets.append(new)


tweet_words = []


for tweet in new_tweets:
  
  tweet = tweet.split(' ')
  
  # print(' ')
  # print(tweet)
  tweet_words += tweet

tweet_words = list(set(tweet_words))
tweet_words.remove('')
tweet_words = np.array(tweet_words)
print(tweet_words)
bow = []

def get_bow(tweet):
  

  tweet = tweet.split(' ')
  tweet = list(set(tweet))
  if '' in tweet:

    tweet.remove('')
  tweet = np.array(tweet)
  # print(tweet_words)
  # print(tweet)

  bag = np.zeros(len(tweet_words))

  arr, x, y = np.intersect1d(tweet_words, tweet, return_indices=True)
  bag[x] = 1
  return bag


bow = np.array(bow)
bow = get_bow(new_tweets[0])
for tweet in new_tweets:
  arr = get_bow(tweet)
  bow = np.vstack((bow, arr))

print(bow)
  
print(bow.shape)
bow_list = []
for i in bow[1:]:
  bow_list.append(i.tolist())
print(len(bow_list))
print(np.count_nonzero(bow[0] == 1))
# d = {'tweets': df['text']}
# print(d)
df_bow = pd.DataFrame(bow,columns = tweet_words, index = np.arange(0,1631) )
df_bow
df_bow['tweets'] = df['text']
df_bow
# print(df_bow['tweets'])
# col = df_bow.pop("tweets")
df_bow.insert(0, 'Tweets', df['text'])
df_bow
# df_bow['sentiment'] = df['sentiment'] 
# col = df_bow.pop("sentiment")
df_bow.insert(3831, 'Sentiments', df['sentiment'])
df_bow
from google.colab import drive

drive.mount('/content/drive')

df_bow.to_csv('/content/drive/MyDrive/ML/df_bagOfWords.csv')
def calculate_distances(x_train,x_test, euclidean = True):
  len_x = x_train.shape[0]
  len_y = x_test.shape[0]

  dists = np.zeros((len_x,len_y))



  

  if euclidean:

    for i in range(len_x):
      
      a = x_train[i]
      b = np.transpose(x_test)
      # print(a.shape)
      # print(b.shape)
      c = np.square(a + np.transpose(b))
      print(c.shape)
      d = np.sqrt(np.sum(c,axis = 1))
      print(d)
      
      dists[i] = d

  else:
    for i in range(len_x):
      
      a = x_train[i]
      b = np.transpose(x_test)
      # print(a.shape)
      # print(b.shape)
      c = np.abs(a - np.transpose(b))
      d = np.sum(c,axis = 1)
      
      dists[i] = d




  # print(dists)
  return dists
x_train = np.random.randint(10, size = (5,4))
x_test = np.random.randint(10, size = (3,4))


calculate_distances(x_train, x_test)
def predict(dists, y_train, k):

  num_train, num_test = dists.shape
  y_pred = np.zeros(num_test)


  for i in range(num_test):
    if k == 1:

      y_pred[i] = y_train[np.argsort[dists[:,i]][0]]

    else:
      nearest_k = np.argsort[dists[:,i]][0:k]
      top_k = y_train[nearest_k]
      label = np.argmax[np.bincount[top_k]]
  

def evaluate(x_test, y_test,y_knn,k):
  correct_labels = np.count_nonzero(y_test == y_knn) 
  accuracy = correct_labels / (x_test.shape[0])

  tp = np.count_nonzero(y_test == 1 && y_knn == 1)

  fp = np.count_nonzero(y_test == 0 && y_knn == 1)

  fn = np.count_nonzero(y_test == 1 && y_knn == 0)

  tn = np.count_nonzero(y_test == 0 && y_knn == 0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1918158567774936, 'fraction_numerical': 0.014066496163682864, 'mean_word_length': 3.5318655851680187, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '420409', 'license': 'MIT', 'n_tokens_mistral': 1737, 'n_tokens_neox': 1665, 'n_words': 293, 'path': 'Untitled3.ipynb', 'repo_name': 'hamdjalil/ChordDiagram', 'size': '3910'}