<jupyter_start><jupyter_text>Build a similarity matrix for all tweets in the dataset (post processing the dataset)Takes about 1 hour to build the similarity matrix for 600k x 600k (all tweets x all tweets ) andTakes about 15 minutes to build the similarity matrix for 1 x 325k (one tweet row x all bot tweets)<jupyter_code># Not enough memory to pre-compile similarity matrices - even for just all_tweets x bot_tweets - 
# We compute similarity matrices in batches of 10,000 and save only the top 4 similar tweets per tweet

tqdm.pandas()
tqdm.__init__ = partialmethod(tqdm.__init__, disable=False)

print("total tweets", len(data))

time_start = time()

# data_subset = data[['tweet','label']].sample(n=300000, random_state=11)
data_subset = data.groupby('label').apply(lambda x: x.sample(180000, random_state=1))

all_tweets = data_subset['tweet'].to_list()
all_tweet_labels = data_subset['label'].to_list()

bot_tweets = data_subset[data_subset['label']==1]['tweet'].to_list()
bot_tweet_labels = data_subset[data_subset['label']==1]['label'].to_list()

human_tweets = data_subset[data_subset['label']==0]['tweet'].to_list()
human_tweet_labels = data_subset[data_subset['label']==0]['label'].to_list()

print("all_tweets", len(all_tweets))
print("bot_tweets", len(bot_tweets))
print("human_tweets", len(human_tweets))
print("verify bot+human tweet size", len(bot_tweets) + len(human_tweets))
pd.option_context('display.max_rows', None, 'display.max_columns', None)
print(data_subset.iloc[0])
print(len(data_subset))
batch_size = 9000
num_batches = int(len(all_tweets)/batch_size)
print(num_batches)

for i in range(0,num_batches):
  print("{}/{}".format(i+1, num_batches))
  # Get the data batch
  data_batch = all_tweets[i*batch_size:(i+1)*batch_size]
  print("data_batch size", len(data_batch))
  time_start = time()
  # Compute the bot sim matrix for the batch
  sim_mat_bot = model.similarity(data_batch, bot_tweets)
  print("sim_mat_bot size", len(sim_mat_bot)) #should be equal to batch_size

  # use np.argpartition to get the top 4 similar tweets
  num = 4
  sim_mat_bot = np.argpartition(sim_mat_bot, -num, axis=1)[:, -num:]
  # These are the top 4 tweets in ascending order. Sort them while accessing them later on.
  
  np.save(data_triplet_matrices_folder+'train/sim_mat_bot_' + str(i*batch_size) + '_'+ str((i+1)*batch_size) + '.npy', sim_mat_bot)
  print("time taken to write bot" + str(i*batch_size) +"_" + str((i+1)*batch_size) + "npy file", time() - time_start)

  # delete bot sim matrix
  del sim_mat_bot
  # del top4_sim_mat_bot
  gc.collect()
  
  time_start = time()
  # Compute the human sim matrix for the batch
  sim_mat_human = model.similarity(data_batch, human_tweets)
  print("sim_mat_human size", len(sim_mat_human)) #should be equal to batch_size

  # use np.argpartition to get the top 4 similar tweets
  num = 4
  sim_mat_human = np.argpartition(sim_mat_human, -num, axis=1)[:, -num:]
  # These are the top 4 tweets in ascending order. Sort them while accessing them later on.


  np.save(data_triplet_matrices_folder+'train/sim_mat_human_' + str(i*batch_size) + '_' + str((i+1)*batch_size) + '.npy', sim_mat_human)
  print("time taken to write human" + str(i*batch_size) + "_" + str((i+1)*batch_size) + "npy file", time() - time_start)

  # delete human sim matrix
  del sim_mat_human
  # del top4_sim_mat_human
  gc.collect()<jupyter_output><empty_output><jupyter_text>Functions for searching similar sentences - using sim matrix<jupyter_code>def sim_00(row, tweet, label):
  top3_bot_indices = global_bot_sim_mat[row.name][::-1] 
  top3_human_indices = global_human_sim_mat[row.name][::-1] 

  if label == 0:
    # Current tweet is a human tweet
    # Just hardcode the label? For now let's get it from the actual list and see if the right values are coming
    return human_tweets[top3_human_indices[0]], human_tweet_labels[top3_human_indices[0]], \
            bot_tweets[top3_bot_indices[0]], bot_tweet_labels[top3_bot_indices[0]] 
  elif label == 1:
    # Current tweet is a bot tweet
    return bot_tweets[top3_bot_indices[0]], bot_tweet_labels[top3_bot_indices[0]], \
            human_tweets[top3_human_indices[0]], human_tweet_labels[top3_human_indices[0]]
  else:
    return 'Should not come here'

def sim_01(row, tweet, label):
  top3_bot_indices = global_bot_sim_mat[row.name][::-1] 
  top3_human_indices = global_human_sim_mat[row.name][::-1] 

  if label == 0:
    # Current tweet is a human tweet
    # Just hardcode the label? For now let's get it from the actual list and see if the right values are coming
    return human_tweets[top3_human_indices[0]], human_tweet_labels[top3_human_indices[0]], \
            bot_tweets[top3_bot_indices[1]], bot_tweet_labels[top3_bot_indices[1]]
  elif label == 1:
    # Current tweet is a bot tweet
    return bot_tweets[top3_bot_indices[0]], bot_tweet_labels[top3_bot_indices[0]], \
            human_tweets[top3_human_indices[1]], human_tweet_labels[top3_human_indices[1]]
  else:
    return 'Should not come here'

def sim_10(row, tweet, label):
  top3_bot_indices = global_bot_sim_mat[row.name][::-1] 
  top3_human_indices = global_human_sim_mat[row.name][::-1] 

  if label == 0:
    # Current tweet is a human tweet
    # Just hardcode the label? For now let's get it from the actual list and see if the right values are coming
    return human_tweets[top3_human_indices[1]], human_tweet_labels[top3_human_indices[1]], \
            bot_tweets[top3_bot_indices[0]], bot_tweet_labels[top3_bot_indices[0]] 
  elif label == 1:
    # Current tweet is a bot tweet
    return bot_tweets[top3_bot_indices[1]], bot_tweet_labels[top3_bot_indices[1]], \
            human_tweets[top3_human_indices[0]], human_tweet_labels[top3_human_indices[0]]
  else:
    return 'Should not come here'

def sim_11(row, tweet, label):
  top3_bot_indices = global_bot_sim_mat[row.name][::-1] 
  top3_human_indices = global_human_sim_mat[row.name][::-1] 

  if label == 0:
    # Current tweet is a human tweet
    # Just hardcode the label? For now let's get it from the actual list and see if the right values are coming
    return human_tweets[top3_human_indices[1]], human_tweet_labels[top3_human_indices[1]], \
            bot_tweets[top3_bot_indices[1]], bot_tweet_labels[top3_bot_indices[1]] 
  elif label == 1:
    # Current tweet is a bot tweet
    return bot_tweets[top3_bot_indices[1]], bot_tweet_labels[top3_bot_indices[1]], \
            human_tweets[top3_human_indices[1]], human_tweet_labels[top3_human_indices[1]]
  else:
    return 'Should not come here'<jupyter_output><empty_output><jupyter_text>Building the Dataset<jupyter_code>tqdm.pandas()
tqdm.__init__ = partialmethod(tqdm.__init__, disable=False)
## **Note: Editing the encode function of SimCSE.tools to silence TQDM there**
## Edited but nothing happened! progress bar still appears. Maybe a restart is needed but I don't want to restart now

def build_dataset(file, data):
  
  triplet_csv_build_start = time()
  
  # num_similar_tweets = 2
  # out_data = out_data.loc[out_data.index.repeat(2**num_similar_tweets)].reset_index(drop=True)


  out_data_00 = data[['tweet','label']].copy()
  out_data_00 = out_data_00.rename(columns={'tweet': 'sent0', 'label': 'sent0_label'})
  print(len(out_data_00))

  out_data_01 = data[['tweet','label']].copy()
  out_data_01 = out_data_01.rename(columns={'tweet': 'sent0', 'label': 'sent0_label'})

  out_data_10 = data[['tweet','label']].copy()
  out_data_10 = out_data_10.rename(columns={'tweet': 'sent0', 'label': 'sent0_label'})

  out_data_11 = data[['tweet','label']].copy()
  out_data_11 = out_data_11.rename(columns={'tweet': 'sent0', 'label': 'sent0_label'})

  file_00_time_start = time()
  print("preparing 00 file (most similar sent1, most similar hard_neg)")
  out_data_00['sent1'], out_data_00['sent1_label'], out_data_00['hard_neg'], out_data_00['hard_neg_label'] = \
                                                            zip(*out_data_00.apply(lambda x: sim_00(x, x.sent0, x.sent0_label), axis=1))  
  file_00_time_end = time()
  print("Time taken for file 00 :", file_00_time_end - file_00_time_start)

  file_01_time_start = time()
  print("preparing 01 file (most similar sent1, second most similar hard_neg)")
  out_data_01['sent1'], out_data_01['sent1_label'], out_data_01['hard_neg'], out_data_01['hard_neg_label'] = \
                                                            zip(*out_data_01.apply(lambda x: sim_01(x, x.sent0, x.sent0_label), axis=1))
  file_01_time_end = time()
  print("Time taken for file 01 :", file_01_time_end - file_01_time_start)

  file_10_time_start = time()
  print("preparing 10 file (second most similar sent1, most similar hard_neg)")
  out_data_10['sent1'], out_data_10['sent1_label'], out_data_10['hard_neg'], out_data_10['hard_neg_label'] = \
                                                            zip(*out_data_10.apply(lambda x: sim_10(x, x.sent0, x.sent0_label), axis=1))

  file_10_time_end = time()
  print("Time taken for file 10 :", file_10_time_end - file_10_time_start)

  file_11_time_start = time()
  print("preparing 10 file (second most similar sent1, second most similar hard_neg)")
  out_data_11['sent1'], out_data_11['sent1_label'], out_data_11['hard_neg'], out_data_11['hard_neg_label'] = \
                                                            zip(*out_data_11.apply(lambda x: sim_11(x, x.sent0, x.sent0_label), axis=1))

  file_11_time_end = time()
  print("Time taken for file 11 :", file_11_time_end - file_11_time_start)

  out_data = pd.concat([out_data_00, out_data_01, out_data_10, out_data_11], ignore_index=True)
  # with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also
  #     display(out_data)

  # /content/drive/MyDrive/CSCI_544_NLP_Project/Twitter_Bot_Detection/Data/Model_TwiBot_Similarity_With_Metadata_360k_Tweets/twiBot-360k-similarity-metadata-1epoch
  out_data.to_csv(data_folder + "Model_TwiBot_Similarity_With_Metadata_360k_Tweets/" + file + "_all_data_triplets_with_similarity.csv", index=False)
  print('Time taken to build CSV:', (time() - triplet_csv_build_start))<jupyter_output><empty_output><jupyter_text>Create Global Similarity Matrices<jupyter_code># /content/drive/MyDrive/CSCI_544_NLP_Project/Twitter_Bot_Detection/Data/Model_TwiBot_Similarity_With_Metadata_360k_Tweets/data_triplet_matrices

# triplet_folder = "/content/drive/MyDrive/CSCI_544_NLP_Project/Twitter_Bot_Detection/Data/Model_TwiBot_Similarity_With_Metadata_360k_Tweets/data_triplet_matrices/train/"
triplet_folder = "/content/drive/MyDrive/CSCI_544_NLP_Project/Twitter_Bot_Detection/Data/Model_TwiBot_Similarity_With_Metadata_360k_Tweets/data_triplet_matrices/dev/"

global_bot_sim_mat_orig = []
global_human_sim_mat_orig = []

batch_size = 9000
num_batches = int(len(all_tweets)/batch_size)
print(num_batches)

for i in range(num_batches):
  print("{}/{}".format(i+1, num_batches))
  sim_mat_bot = np.load(triplet_folder+'sim_mat_bot_' + str(i*batch_size) + '_' + str((i+1)*batch_size) + '.npy')
  sim_mat_human = np.load(triplet_folder+'sim_mat_human_' + str(i*batch_size) + '_' + str((i+1)*batch_size) + '.npy')
  
  global_bot_sim_mat_orig.extend(sim_mat_bot)
  global_human_sim_mat_orig.extend(sim_mat_human)


print("len(global_bot_sim_mat_orig)", len(global_bot_sim_mat_orig))
print("len(global_human_sim_mat_orig)", len(global_human_sim_mat_orig))<jupyter_output><empty_output><jupyter_text>Delete the instances of the index in sim matrix if it matches with row index ( we don't want tweets that are similar to themselves )<jupyter_code>print(global_bot_sim_mat_orig[0])
print(global_human_sim_mat_orig[0])

global_bot_sim_mat = [[x for x in global_bot_sim_mat_orig[row] if x != row] for row in range(len(global_bot_sim_mat_orig))]
global_human_sim_mat = [[x for x in global_human_sim_mat_orig[row] if x != row] for row in range(len(global_human_sim_mat_orig))]

print(global_bot_sim_mat[0])
print(global_human_sim_mat[0])<jupyter_output><empty_output><jupyter_text>Call the build_dataset function<jupyter_code># Build the train triplet set
pd.options.mode.chained_assignment = None  # default='warn'

# we shuffle the data in SimCSE code

data_subset.reset_index(drop = True, inplace = True)

build_dataset("train", data_subset)<jupyter_output><empty_output><jupyter_text>Verify that the created file has the right number of rows (expected 1440000 rows)<jupyter_code>all_data = pd.read_csv('/content/drive/MyDrive/CSCI_544_NLP_Project/Twitter_Bot_Detection/Data/Model_TwiBot_Similarity_With_Metadata_360k_Tweets/train_all_data_triplets_with_similarity_new.csv')
print(len(all_data))<jupyter_output><empty_output><jupyter_text>Build the dataset for SimCSE (only tweets - no labels)<jupyter_code>all_data_only_tweets = all_data[['sent0','sent1','hard_neg']]
all_data_only_tweets = all_data_only_tweets.sample(frac=1, random_state=1).reset_index(drop=True)
all_data_only_tweets.to_csv("/content/drive/MyDrive/CSCI_544_NLP_Project/Twitter_Bot_Detection/Data/Model_TwiBot_Similarity_With_Metadata_360k_Tweets/train_all_data_triplets_with_similarity_for_simcse.csv", index=False)
time_start = time()

sim_mat_bot_file = tables.open_file('/content/sim_mat_bot_0_20000.hdf', 'r')
sim_mat_bot = sim_mat_bot_file.root.somename
sim_mat_bot_rows = sim_mat_bot[:10,:] #only one row gets loaded into memory 
print (len(sim_mat_bot_rows))

print("time taken to load bot hdf file", time() - time_start)
time_start = time()

# print(sim_mat_bot_rows)

num = 4
top4_sim_mat_bot_rows = np.argpartition(sim_mat_bot_rows, -num, axis=1)[:, -num:]
print(sim_mat_bot_rows[np.arange(sim_mat_bot_rows.shape[0])[:, None], top4_sim_mat_bot_rows])

# print(top)
# top = np.argpartition(sim_mat_bot_rows, -num, axis=1)[-num:]
# x = sim_mat_bot_rows[np.arange(sim_mat_bot_rows.shape[0])[:, None], top]
# top4_sim_mat_bot_rows = np.argpartition(sim_mat_bot_rows, -num, axis=1)[-num:]
np.save(data_folder+'data_triplets/train/sim_mat_bot_0_10000.npy', top4_sim_mat_bot_rows)
print("time taken to write bot npy file", time() - time_start)<jupyter_output><empty_output><jupyter_text>Old/Unused - Build a similarity matrix for all tweets in the dataset (post processing the dataset) Try using the build_index() and search() functionsBuilding the two indices takes around 30 mins. Index is built as a model property (self.index). See if we can save it when we save the model.<jupyter_code>all_tweets = data['tweet'].to_list()
print(len(all_tweets))

bot_tweets = data[data['label']==1]['tweet'].to_list()
bot_tweet_labels = data[data['label']==1]['label'].to_list()

human_tweets = data[data['label']==0]['tweet'].to_list()
human_tweet_labels = data[data['label']==0]['label'].to_list()


print("bot_tweets", len(bot_tweets))
print("human_tweets", len(human_tweets))
print(len(bot_tweets) + len(human_tweets))
sim_mat_time_start = time()

#build index for bot tweets and human tweets
# model_idx = model.build_index(all_tweets)
bot_index_model.build_index(bot_tweets)
human_index_model.build_index(human_tweets)

print("\nTime taken to build index", time() - sim_mat_time_start)
print("\ntweet 0", all_tweets[0])
results = bot_index_model.search(human_tweets[0], threshold = 0, top_k = 6)
print("\nSimilar Bot tweets:",results)

print("\ntweet 0", all_tweets[0])
results = human_index_model.search(human_tweets[0], threshold = 0, top_k = 6)
print("\nSimilar Human tweets:",results)

# print(results[1][0])
# print(human_tweets.index(results[1][0]))<jupyter_output><empty_output><jupyter_text>Old/Unused Build similarity matrices for 50k tweets. One sim matrix for tweets x bot_tweets, another for tweets x human_tweets<jupyter_code># Include labels while writing to the triplets CSV!

sim_mat_time_start = time()
# data['tweet'] = data['tweet'].apply(lambda x: ' '.join(x))

# As of now, we sample 50000 random rows with seed = 1. If we wanna progressively build the entire dataset, we can take the first 50000 rows, and so on.
data_subset = data[['tweet','label']].sample(n=50000, random_state=1)
tweet_subset = data_subset['tweet'].to_list()
label_subset = data_subset['label'].to_list()
print(data_subset[:5])
print(data_subset[-5:])
bot_tweets = data_subset[data_subset['label']==1]['tweet'].to_list()
bot_tweet_labels = data_subset[data_subset['label']==1]['label'].to_list()
human_tweets = data_subset[data_subset['label']==0]['tweet'].to_list()
human_tweet_labels = data_subset[data_subset['label']==0]['label'].to_list()
print("bot_tweets", len(bot_tweets))
print("human_tweets", len(human_tweets))
print("total tweets", len(bot_tweets) + len(human_tweets))
# import numpy as np

# sim_mat_bot = np.load(data_folder+'similarity_matrix_bot_50k.npy')
# sim_mat_human = np.load(data_folder+'similarity_matrix_human_50k.npy')<jupyter_output><empty_output><jupyter_text>Old/Unused Build triplet CSV (dataset size = 50k right now)  Prev exec : 6622.105570077896 seconds<jupyter_code>import heapq
import numpy as np

triplet_csv_build_start = time()

out = pd.DataFrame(columns=['sent0', 'sent1', 'hard_neg', 'sent0_label', 'sent1_label', 'hard_neg_label'])
print("tweets in tweet_subset", len(tweet_subset))
for idx in range(len(data_subset)):
  if idx % 100 == 0:
    print("{}/{}".format(idx, len(data_subset)))
  # find top n similar bot tweets
  top3_bot_indices = heapq.nlargest(3, range(len(sim_mat_bot[idx])), key=sim_mat_bot[idx].__getitem__)
  # # find top n similar human tweets
  top3_human_indices = heapq.nlargest(3, range(len(sim_mat_human[idx])), key=sim_mat_human[idx].__getitem__)

  # combine all the permutations of the most similar bot and human tweets
  # ignore the first most similar bot tweet if current tweet is a bot tweet
  # else ignore the first most similar human tweet
  if label_subset[idx]==1:
    for bot_idx in top3_bot_indices[:2]:
      for human_idx in top3_human_indices[:2]:
        # print("<bot>",tweet_subset[idx], tweet_subset[bot_idx], tweet_subset[human_idx])
        out = out.append({'sent0': tweet_subset[idx], 'sent1': bot_tweets[bot_idx], 'hard_neg': human_tweets[human_idx],\
                          'sent0_label': label_subset[idx], 'sent1_label': bot_tweet_labels[bot_idx], 'hard_neg_label': human_tweet_labels[human_idx]}, ignore_index=True)
        # print(out_line)
        # out.append(out_line)

  elif label_subset[idx]==0:
    for bot_idx in top3_bot_indices[:2]:
      for human_idx in top3_human_indices[1:]:
        # print("<human>",tweet_subset[idx], tweet_subset[bot_idx], tweet_subset[human_idx])
        out = out.append({'sent0': tweet_subset[idx], 'sent1': human_tweets[human_idx], 'hard_neg': bot_tweets[bot_idx],\
                          'sent0_label': label_subset[idx], 'sent1_label': human_tweet_labels[human_idx], 'hard_neg_label': bot_tweet_labels[bot_idx]}, ignore_index=True)
        # print(out_line)
        # out.append(out_line)
  else:
    print("something went wrong in the label comparison")

#*******************************************************************************
# If the above does not work due to memory issues, 
# we loop across each tweet and find top n similar bot 
# tweets and top n similar human tweets for the given tweet.

# for tweet_idx, tweet in enumerate(all_tweets.iloc[0]):
#   print(tweet_idx, tweet)

# for index, row in data_subset[:5].iterrows():
#     print(row['tweet'], row['label'])


  # find top n similar bot tweets
  # similar_bot_tweets = model.similarity(tweet, bot_tweets)
  # top3_bot_indices = heapq.nlargest(3, range(len(similar_bot_tweets[0])), key=similar_bot_tweets[0].__getitem__)

  # # find top n similar human tweets
  # similar_human_tweets = model.similarity(tweet, human_tweets)
  # top3_human_indices = heapq.nlargest(3, range(len(similar_human_tweets[0])), key=similar_human_tweets[0].__getitem__)

  # # combine all the permutations of the most similar bot and human tweets
  # # ignore the first most similar bot tweet if current tweet is a bot tweet
  # # else ignore the first most similar human tweet
  # if tweet['label']==1:
  #   for bot_idx in top3_bot_indices[1:]:
  #     for human_idx in top3_human_indices[:2]:
  #       print(tweet, all_tweets[bot_idx], all_tweets[human_idx])
  # elif tweet['label']==0:
  #   for bot_idx in top3_bot_indices[:2]:
  #     for human_idx in top3_human_indices[1:]:
  #       print(tweet, all_tweets[bot_idx], all_tweets[human_idx])
  # else:
  #   print("something went wrong in the label comparison")
#*******************************************************************************

print(out)

out.to_csv(data_folder+"train_triplets_with_similarity.csv", index=False)

print('Time taken to build CSV:', (time() - triplet_csv_build_start))<jupyter_output><empty_output><jupyter_text>Old/Unused Functions for searching similar sentences - using build_index and search<jupyter_code>def similar_00(tweet, label):
  top3_bot_tweets = bot_index_model.search(tweet, threshold = 0, top_k = 3)
  top3_human_tweets = human_index_model.search(tweet, threshold = 0, top_k = 3)

  if label == 0:
    # Current tweet is a human tweet
    # Just hardcode the label? For now let's get it from the actual list and see if the right values are coming
    return top3_human_tweets[1][0], human_tweet_labels[human_tweets.index(top3_human_tweets[1][0])], \
            top3_bot_tweets[0][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[0][0])] 
  elif label == 1:
    # Current tweet is a bot tweet
    return top3_bot_tweets[1][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[1][0])], \
            top3_human_tweets[0][0], human_tweet_labels[human_tweets.index(top3_human_tweets[0][0])]
  else:
    return 'Should not come here'

def similar_01(tweet, label):
  top3_bot_tweets = bot_index_model.search(tweet, threshold = 0, top_k = 3)
  top3_human_tweets = human_index_model.search(tweet, threshold = 0, top_k = 3)

  if label == 0:
    # Current tweet is a human tweet
    # Just hardcode the label? For now let's get it from the actual list and see if the right values are coming
    return top3_human_tweets[1][0], human_tweet_labels[human_tweets.index(top3_human_tweets[1][0])], \
            top3_bot_tweets[1][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[1][0])]
  elif label == 1:
    # Current tweet is a bot tweet
    return top3_bot_tweets[1][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[1][0])], \
            top3_human_tweets[1][0], human_tweet_labels[human_tweets.index(top3_human_tweets[1][0])]
  else:
    return 'Should not come here'

def similar_10(tweet, label):
  top3_bot_tweets = bot_index_model.search(tweet, threshold = 0, top_k = 3)
  top3_human_tweets = human_index_model.search(tweet, threshold = 0, top_k = 3)

  if label == 0:
    # Current tweet is a human tweet
    # Just hardcode the label? For now let's get it from the actual list and see if the right values are coming
    return top3_human_tweets[2][0], human_tweet_labels[human_tweets.index(top3_human_tweets[2][0])], \
            top3_bot_tweets[0][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[0][0])] 
  elif label == 1:
    # Current tweet is a bot tweet
    return top3_bot_tweets[2][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[2][0])], \
            top3_human_tweets[0][0], human_tweet_labels[human_tweets.index(top3_human_tweets[0][0])]
  else:
    return 'Should not come here'

def similar_11(tweet, label):
  top3_bot_tweets = bot_index_model.search(tweet, threshold = 0, top_k = 3)
  top3_human_tweets = human_index_model.search(tweet, threshold = 0, top_k = 3)

  if label == 0:
    # Current tweet is a human tweet
    # Just hardcode the label? For now let's get it from the actual list and see if the right values are coming
    return top3_human_tweets[2][0], human_tweet_labels[human_tweets.index(top3_human_tweets[2][0])], \
            top3_bot_tweets[1][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[1][0])] 
  elif label == 1:
    # Current tweet is a bot tweet
    return top3_bot_tweets[2][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[2][0])], \
            top3_human_tweets[1][0], human_tweet_labels[human_tweets.index(top3_human_tweets[1][0])]
  else:
    return 'Should not come here'

# def similar_0_sent1(tweet, label):
#   # Human tweet
#   top3_bot_tweets = bot_index_model.search(tweet, threshold = 0, top_k = 3)
#   top3_human_tweets = human_index_model.search(tweet, threshold = 0, top_k = 3)

#   if label == 0:
#     # Current tweet is a human tweet
#     # Just hardcode the label? For now let's get it from the actual list and see if the right values are coming
#     return top3_human_tweets[1][0], human_tweet_labels[human_tweets.index(top3_human_tweets[1][0])] 
#   elif label == 1:
#     # Current tweet is a bot tweet
#     return top3_bot_tweets[1][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[1][0])]
#   else:
#     return 'Should not come here'

# def similar_0_hard_neg(tweet, label):
#   # Human tweet
#   top3_bot_tweets = bot_index_model.search(tweet, threshold = 0, top_k = 3)
#   top3_human_tweets = human_index_model.search(tweet, threshold = 0, top_k = 3)

#   if label == 0:
#     # Current tweet is a human tweet
#     return top3_bot_tweets[0][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[0][0])]
#   elif label == 1:
#     # Current tweet is a bot tweet
#     return top3_human_tweets[0][0], human_tweet_labels[human_tweets.index(top3_human_tweets[0][0])]
#   else:
#     return 'Should not come here'

# def similar_1_sent1(tweet, label):
#   # Human tweet
#   top3_bot_tweets = bot_index_model.search(tweet, threshold = 0, top_k = 3)
#   top3_human_tweets = human_index_model.search(tweet, threshold = 0, top_k = 3)

#   if label == 0:
#     # Current tweet is a human tweet
#     return top3_human_tweets[2][0], human_tweet_labels[human_tweets.index(top3_human_tweets[2][0])]
#   elif label == 1:
#     # Current tweet is a bot tweet
#     return top3_bot_tweets[2][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[2][0])]
#   else:
#     return 'Should not come here'

# def similar_1_hard_neg(tweet, label):
#   # Human tweet
#   top3_bot_tweets = bot_index_model.search(tweet, threshold = 0, top_k = 3)
#   top3_human_tweets = human_index_model.search(tweet, threshold = 0, top_k = 3)

#   if label == 0:
#     # Current tweet is a human tweet
#     return top3_bot_tweets[1][0], bot_tweet_labels[bot_tweets.index(top3_bot_tweets[1][0])]
#   elif label == 1:
#     # Current tweet is a bot tweet
#     return top3_human_tweets[1][0], human_tweet_labels[human_tweets.index(top3_human_tweets[1][0])]
#   else:
#     return 'Should not come here'<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12647954007439974, 'fraction_numerical': 0.02810656446097772, 'mean_word_length': 5.4238474535360846, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '509764', 'license': 'MIT', 'n_tokens_mistral': 10228, 'n_tokens_neox': 9709, 'n_words': 2011, 'path': 'DatasetBuilder.ipynb', 'repo_name': 'vrundha/bot-on-my-watch', 'size': '26613'}