<jupyter_start><jupyter_text>WordSimilarity<jupyter_code>p = 'Evaluation_Benchmarks/Word_Similarity'
p1 = p+ '/WS353_Exact/EN-WS-353-ALL.txt'
p2 = p + '/RG65/EN-RG-65.txt'
p3 = p + '/MEN/EN-MEN-TR-3k.txt'

p1_w = 'ws353_word_string.txt'
p2_w = 'rg_word_string.txt'
p3_w = 'men_word_string.txt'

WS = open(p1,'r')
RG = open(p2,'r')
MEN = open(p3,'r')

count=0
word = ''
print(len(word))
for index,line in tqdm(enumerate(WS)):
  #if count ==2: break
  count+=1
  line = line.strip().lower().split()
  word = word + line[0] + ' ' + line[1] + ' '

file = open(p1_w,'w')
file.write(word)
file.close()

count=0
word2 = ''
#print(len(word))
for index,line in tqdm(enumerate(RG)):
  #if count ==2: break
  count+=1
  line = line.strip().lower().split()
  word2 = word2 + line[0] + ' ' + line[1] + ' '

file = open(p2_w,'w')
file.write(word2)
file.close()

count=0
word3 = ''
#print(len(word))
for index,line in tqdm(enumerate(MEN)):
  #if count ==2: break
  count+=1
  line = line.strip().lower().split()
  word3 = word3 + line[0] + ' ' + line[1] + ' '

file = open(p3_w,'w')
file.write(word3)
file.close()<jupyter_output>0<jupyter_text>SA<jupyter_code>import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))
psa_tr = 'Evaluation_Benchmarks/Sentiment_Analysis/Dataset_Exact/train.txt'
psa_ts = 'Evaluation_Benchmarks/Sentiment_Analysis/Dataset_Exact/test.txt'

psa_tr_w = 'SA-train_word_string.txt'
psa_ts_w = 'SA-test_word_string.txt'

tr_obj = open(psa_tr,'r')
ts_obj = open(psa_ts,'r')
count = 0
str_word = ''
for index,line in tqdm(enumerate(tr_obj)):
  #if count ==20: break
  count += 1
  line = line.strip().split()
  #temp = []
  for word in line[1:]:
    #print("word:", word)
    if word.isalpha() and word not in stop_words:
      #print("hahahaha")
      str_word = str_word + word.lower() + ' '

tr_obj.close()

file = open(psa_tr_w,'w')
file.write(str_word)
file.close()

count = 0
str_word2 = ''
for index,line in tqdm(enumerate(ts_obj)):
  #if count ==20: break
  count += 1
  line = line.strip().split()
  #temp = []
  for word in line[1:]:
    #print("word:", word)
    if word.isalpha() and word not in stop_words:
      #print("hahahaha")
      str_word2 = str_word2 + word.lower() + ' '

ts_obj.close()

file = open(psa_ts_w,'w')
file.write(str_word2)
file.close()




total_sa_string = str_word+str_word2
print(len(str_word))
print(len(str_word2))
print(len(total_sa_string))
file = open('SA-traintest_word_string.txt','w')
file.write(total_sa_string)
file.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18132295719844357, 'fraction_numerical': 0.025680933852140077, 'mean_word_length': 4.688053097345133, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '144732', 'license': 'MIT', 'n_tokens_mistral': 1117, 'n_tokens_neox': 1098, 'n_words': 194, 'path': 'ESA_Retrofit/Eval_Dataset_String.ipynb', 'repo_name': 'pp30/NLP-Wikipedia_Relatedness', 'size': '2570'}