<jupyter_start><jupyter_text>**Doc2Vec and distilBERT**<jupyter_code>#Using the GPU
import torch
if torch.cuda.is_available():    

    # Tell PyTorch to use the GPU.    
    device = torch.device("cuda")

    print('There are %d GPU(s) available.' % torch.cuda.device_count())

    print('We will use the GPU:', torch.cuda.get_device_name(0))

# If not...
else:
    print('No GPU available, using the CPU instead.')
    device = torch.device("cpu")
import io
from google.colab import files 
import pandas as pd
uploaded = files.upload()
df= pd.read_csv(io.BytesIO(uploaded['case_docs.csv']))
documents=list(df['Content'])
print(df['Content'])

#Query doc
uploaded2 = files.upload()
qdf= pd.read_csv(io.BytesIO(uploaded2['embedded_queries.csv']))
queries=list(qdf['query'])
print(qdf)<jupyter_output><empty_output><jupyter_text>**distilBERT**<jupyter_code>#Cleaning the fetched documents,queries and removing unnecessary tokens
#Trying doc2vec

import nltk
nltk.download('punkt')
nltk.download('stopwords')
from nltk.tokenize import word_tokenize
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from nltk.corpus import stopwords
import string
import pandas as pd
import json
def get_cleaned_doc(doc):
  #maintains order and removes unnecessary words
  doc=word_tokenize(doc.lower())
  unnecessary = set(stopwords.words('english'))
  for p in list(string.punctuation):
    unnecessary.add(p)
  cleaned=[i for i in doc if i not in unnecessary]
  cleaned=[token for token in cleaned if not token.isdigit()] #remove numbers
  return cleaned

tokenized_documents=[get_cleaned_doc(d) for d in documents]
documents=[" ".join(d) for d in tokenized_documents]
queries=[get_cleaned_doc(q) for q in queries] 

print(tokenized_documents[0])

!pip install -U sentence-transformers
#Get doc embeddings using sentence transformers (DistilBERT)
from sentence_transformers import SentenceTransformer
import numpy as np
model = SentenceTransformer('distilbert-base-nli-mean-tokens')
doc_embeddings = model.encode(documents)
print(doc_embeddings.shape)


#download embeddings
doc_embeddings=np.array(doc_embeddings)
print(doc_embeddings.shape)
np.save('case_docs_embeddings.npy', doc_embeddings)
files.download('case_docs_embeddings.npy')
#Get Query Embeddings using sentence trasnformers

queries=list(qdf["query"])
query_embeddings=model.encode(queries)
print(query_embeddings.shape)

#download embeddings
query_embeddings=np.array(query_embeddings)
np.save('query_embeddings.npy', query_embeddings)
files.download('query_embeddings.npy')<jupyter_output>(50, 768)<jupyter_text>**Doc2Vec**<jupyter_code>k=25 #number of rel docs to pick!
Q=50
from gensim.models.doc2vec import Doc2Vec, TaggedDocument
tagged_docs = [TaggedDocument(d, [i]) for i, d in enumerate(tokenized_documents)]

model = Doc2Vec(tagged_docs, vector_size = 100, window = 2, min_count = 1, epochs = 100,workers=4)
query_results=[]
for i,query in enumerate(queries):
  query_vector = model.infer_vector(query)
  top_k=model.docvecs.most_similar(positive = [query_vector],topn=k)
  top_k_docs=[d for d,cs in top_k]

  given=json.loads(qdf.loc[(i,"relevant")])
  matches=0
  for ele in top_k_docs:
    if(ele in given):
      matches+=1
  precision_at_k=round(matches/k,2)
  recall_at_k=round(matches/len(given),2)
  query_results.append({"index":"AILA_Q"+str(i+1),"top_k":top_k_docs,"relevant":given,"precision@k":precision_at_k,"recall@k":recall_at_k})


ans_df=pd.DataFrame(query_results)
print(ans_df)
ap=ans_df["precision@k"].sum()/Q
ar=ans_df["recall@k"].sum()/Q
print("The Avg. precision@k for all queries— ",ap)
print("The Avg. recall@k for all queries— ",ar)
f_score=(2*ap*ar)/ (ap+ar)
print("The F-score is ",f_score)<jupyter_output>index  ... recall@k
0    AILA_Q1  ...     0.00
1    AILA_Q2  ...     0.00
2    AILA_Q3  ...     0.00
3    AILA_Q4  ...     0.00
4    AILA_Q5  ...     0.00
5    AILA_Q6  ...     0.00
6    AILA_Q7  ...     0.00
7    AILA_Q8  ...     0.00
8    AILA_Q9  ...     0.00
9   AILA_Q10  ...     0.00
10  AILA_Q11  ...     0.00
11  AILA_Q12  ...     0.00
12  AILA_Q13  ...     0.00
13  AILA_Q14  ...     0.00
14  AILA_Q15  ...     0.00
15  AILA_Q16  ...     0.00
16  AILA_Q17  ...     0.00
17  AILA_Q18  ...     0.00
18  AILA_Q19  ...     0.00
19  AILA_Q20  ...     0.00
20  AILA_Q21  ...     0.00
21  AILA_Q22  ...     0.00
22  AILA_Q23  ...     0.00
23  AILA_Q24  ...     0.00
24  AILA_Q25  ...     0.00
25  AILA_Q26  ...     0.12
26  AILA_Q27  ...     0.00
27  AILA_Q28  ...     0.00
28  AILA_Q29  ...     0.00
29  AILA_Q30  ...     0.00
30  AILA_Q31  ...     0.00
31  AILA_Q32  ...     0.00
32  AILA_Q33  ...     0.00
33  AILA_Q34  ...     0.00
34  AILA_Q35  ...     0.00
35  AILA_Q36  ...     0.00
3[...]
{'fraction_non_alphanumeric': 0.13279592702588036, 'fraction_numerical': 0.056851930420025454, 'mean_word_length': 3.677579365079365, 'pattern_counts': {'":': 5, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '176533', 'license': 'MIT', 'n_tokens_mistral': 2005, 'n_tokens_neox': 1826, 'n_words': 395, 'path': 'distilBERT_+_Doc2Vec (1).ipynb', 'repo_name': 'thecodearrow/Artificial-Intelligence-for-Legal-Assistance', 'size': '4714'}