<jupyter_start><jupyter_text>This sheets answers the question of: out of the best models, how similar are the top X% of the results.  The best models are mpnet_base_v2, roberta, and scispacy, since all three of these models have the greatest z-scores from the noise.  See Noise_to_Related_Claims_Histogram_DrugLabelsandPatent.ipynb.<jupyter_code>from google.colab import drive
drive.mount('/content/drive', force_remount=True)
zip_path='/content/drive/MyDrive/Colab Notebooks/zip/db2file.zip'
!cp "{zip_path}" .
!cp "/content/drive/MyDrive/Colab Notebooks/requirements.txt" .
!unzip -q db2file.zip
!rm db2file.zip
!rm -r en_core_sci_lg-0.4.0.zip
!cp "/content/drive/MyDrive/Colab Notebooks/zip/en_core_sci_lg-0.4.0.zip" .
!unzip -q en_core_sci_lg-0.4.0.zip
!rm en_core_sci_lg-0.4.0.zip
!pip install -r '/content/requirements.txt'
import random
import os
random.seed(30)
db2files = "/content/db2file/"
NDA_list=[f for f in os.listdir(db2files)]
def get_lines_in_file(file_name):
  if os.path.exists(file_name):
    f = open(file_name, "rb")
    return_list = [str(line.decode('unicode_escape')) for line in f if line.decode('unicode_escape').strip()]
    f.close()
    return return_list
  else:
    return []
def flat_list(lst):
  return [item for sublist in lst for item in sublist]

def get_additions(NDA, additions_folder_name):
  """ 
  Return all additions as a list for the set-id with most additions for a NDA 
  excluding the first addition.
  Parameters:
      NDA (string): NDA dir
      additions_folder_name (string): either 'just_additions' or 'additions_with_context'
  """
  if additions_folder_name not in ['just_additions', 'additions_with_context']:
    print(f"Parameter {additions_folder_name} not in ['just_additions', 'additions_with_context']")
    return []
  NDA_dir=db2files+str(NDA)+'/'
  set_id_dirs=[f for f in os.listdir(NDA_dir)]
  try:
    set_id_dirs.remove('patents')
  except ValueError:
    pass
  additions_list=[]
  for set_id_dir in set_id_dirs:
    additions_dir=NDA_dir+set_id_dir+'/'+additions_folder_name+'/'
    if os.path.exists(additions_dir):
      additions_files=sorted([additions_dir+f for f in os.listdir(additions_dir)])[1:]
      additions_list_tmp=flat_list([get_lines_in_file(file) for file in additions_files])
      if len(additions_list_tmp)> len(additions_list):
        additions_list=additions_list_tmp
  return additions_list

def get_patent_claims(NDA, patents_folder_name):
  """Return a list of patents claims for a NDA
  Parameters:
      NDA (string): NDA dir
      patents_folder_name (string): either 'patents' or 'patents_longhand'
  """
  patent_dir=db2files+str(NDA)+'/'+patents_folder_name+'/'
  if os.path.exists(patent_dir):
    patent_files=[patent_dir+f for f in os.listdir(patent_dir)]
    return flat_list([get_lines_in_file(file) for file in patent_files])
  return []
    
random_NDA_list=random.sample(NDA_list, int(len(NDA_list)*.33))
print(len(NDA_list), len(random_NDA_list), random_NDA_list[1])
# narrow 1/3 of random data to NDA with patents and NDAs with additions.  If either is missing, we cannot check quality of additions to related patents.
random_NDA_list=[x for x in random_NDA_list if get_patent_claims(x, 'patents') and get_additions(x, 'additions_with_context')]
print(len(random_NDA_list))
def return_match(NDA, additions_folder_name, patent_folder_name, scoring_method_list, cutoff_percentage=.1):
  """
  This method returns {("method_name", NDA, len(additions), len(claims)):[[claim_num, claim_num],],}
  Parameters:
    random_NDA_list (list): list of NDA numbers
    additions_folder_name (string): either 'patents' or 'patents_longhand'
    patent_folder_name (string): either 'just_additions' or 'additions_with_context'
    scoring_method_list (list): list of [[function that is use to score similarity, optional_scoring_method_field],..]
  """

  # scoring_method_result_dict={ "scoring_method.__name__": {"additions_len": X, "claims_len": X, "matches":[(addition, claim), }}
  scoring_method_result_dict={}

  for i in range(len(scoring_method_list)):
    scoring_method=scoring_method_list[i][0]
    optional_scoring_method_field=scoring_method_list[i][1]
    optional_scoring_method_field_name=scoring_method_list[i][2]

    claims = get_patent_claims(NDA, patent_folder_name)
    additions = get_additions(NDA, additions_folder_name)

    matrix=scoring_method(additions, claims, optional_scoring_method_field)
    # get top 10 percent of each row/addition of the matrix.
    top_10 = top_10_percent_matrix(matrix, cutoff_percentage)
    method_name = optional_scoring_method_field_name if optional_scoring_method_field else scoring_method.__name__
    key = (method_name, NDA, len(additions), len(claims))
    scoring_method_result_dict[key]=top_10
  return scoring_method_result_dict


import math
def top_10_percent_matrix(matrix, cutoff_percentage):
  """ Return a matrix: ie. [[claim_num_index, claim_num_index],] for top 10% of each match.
        The rows represents additions.

  Parameters:
    matrix (list of list): a matrix where the rows represents additions, and the columns, claims for all additions to all claims
  """
  return_matrix_of_index=[]
  for row in matrix:
    length_10percent=math.ceil(len(row)*cutoff_percentage)
    # get indices of top 10 in max_scores_list
    indexes=sorted(range(len(row)), key=lambda i: row[i], reverse=True)[:length_10percent]
    return_matrix_of_index.append(indexes)
  return return_matrix_of_index

from sentence_transformers import SentenceTransformer, util

def scoring_method_bert(additions, claims, model):
  """ Returns a list of [[cosine_score,cosine_score,],], where rows indicate addition, columns are claims.
  """
  # Compute embedding for both lists
  additions_embeddings = model.encode(
      additions,
      convert_to_tensor=True,
  )
  claims_embeddings = model.encode(
      claims,
      convert_to_tensor=True,
  )
  # Compute cosine-similarity for every additions to every claim
  cosine_scores = util.pytorch_cos_sim(
      additions_embeddings, claims_embeddings
  ).tolist()
  return cosine_scores

import spacy
gpu = spacy.prefer_gpu()
print('GPU:', gpu)
!pip install -U spacy[cuda101]
_N_PROCESS = 1
_en_core_sci_lg_nlp = spacy.load("/content/en_core_sci_lg-0.4.0/en_core_sci_lg/en_core_sci_lg-0.4.0")
!python -m spacy download en_core_web_trf
nlp_en = spacy.load('en_core_web_trf')

def preprocess_with_spacy_nlp(text_list, steps, nlp=nlp_en):
    """
    This method can remove punctuation,
    Parameters:
        text_list (list): list of strings
        steps (list): one of ["punct", "lemma", "stopwords"]
    """
    # make a copy of text_lis
    return_list = text_list
    if any(item in ["punct", "lemma", "stopwords"] for item in steps):
        # 'lemmatizer' required 'tagger' and 'attribute_ruler'
        nlp_list = list(
            nlp.pipe(
                return_list,
                disable=["tok2vec", "ner"],
                n_process=_N_PROCESS,
            )
        )
        return_list = [
            " ".join(
                [
                    token.lemma_ if "lemma" in steps else token.text
                    for token in doc
                    if (
                        (
                            ("punct" in steps and not token.is_punct)
                            or "punct" not in steps
                        )
                        and (
                            ("stopwords" in steps and not token.is_stop)
                            or "stopwords" not in steps
                        )
                    )
                ]
            )
            for doc in nlp_list
        ]
    return return_list


def similarity_matrix(embed_A_list, embed_B_list):
    """
    This method returns a matrix such as:
        [[X, X, X],
        [X, X, X]]
    wherein each row represents the similarity measurement between an embedding
    from embed_A_list to each of the embeddings in embed_B_list.

    Parameters:
        embed_A_list (list): list of NLP object generated by spaCy
        embed_B_list (list): list of NLP object generated by spaCy to be
                             compared to embed_A
    """
    matrix = [[0] * len(embed_B_list) for y in range(len(embed_A_list))]
    for i in range(len(embed_A_list)):
        for j in range(len(embed_B_list)):
            matrix[i][j] = embed_A_list[i].similarity(embed_B_list[j])
    return matrix


def scoring_method_spacy(additions, claims, nlp=nlp_en):
  """ Scores with spaCy; return a matrix of similar scores
  """
  additions=preprocess_with_spacy_nlp(additions, ["punct", "lemma", "stopwords"], nlp)
  claims=preprocess_with_spacy_nlp(claims, ["punct", "lemma", "stopwords"], nlp)
  # Compute embedding for both lists
  # tokenization only requires tok2vec
  disabled_list = ["tagger", "attribute_ruler", "lemmatizer", "parser", "ner"]
  additions_embeddings = list(
      nlp.pipe(
          additions, disable=disabled_list, n_process=_N_PROCESS
      )
  )
  claims_embeddings = list(
      nlp.pipe(
          claims, disable=disabled_list, n_process=_N_PROCESS
      )
  )
  # Compute cosine-similarity for every additions to every claim
  return similarity_matrix(additions_embeddings, claims_embeddings)

_device = None

model_mpnet_base_v2 = SentenceTransformer("stsb-mpnet-base-v2", device=_device)
model_mpnet_base_v2.zero_grad()
# Will limit size since CUDA runs out of memory
model_mpnet_base_v2.max_seq_length=512

model_roberta_base_v2 = SentenceTransformer("stsb-roberta-base-v2", device=_device)
model_roberta_base_v2.zero_grad()
# Will limit size since CUDA runs out of memory
model_roberta_base_v2.max_seq_length=512



scoring_method_list=[
                     [scoring_method_bert, model_mpnet_base_v2, "model_mpnet_base_v2"],
                     [scoring_method_bert,model_roberta_base_v2, "model_roberta_base_v2"],
                     [scoring_method_spacy,nlp_en, "en_core_web_trf"],
                     [scoring_method_spacy,_en_core_sci_lg_nlp, "_en_core_sci_lg_nlp"],]

NDA=random_NDA_list[3]

# return_match_result={("method_name", NDA, len(additions), len(claims)):[[claim_num, claim_num],],}, 
# where the value are (row) additions and (col) index/specific line in the patent file when all files are joined together
return_match_result=return_match(NDA, "additions_with_context", "patents", scoring_method_list, cutoff_percentage=.1)
print(return_match_result)

# Remove en_core_web_trf, since results indicate that there are no vectors
scoring_method_list=[
                     [scoring_method_bert, model_mpnet_base_v2, "model_mpnet_base_v2"],
                     [scoring_method_bert, model_roberta_base_v2, "model_roberta_base_v2"],
                     [scoring_method_spacy, _en_core_sci_lg_nlp, "_en_core_sci_lg_nlp"],]

multi_return_match_result=[]
for i in range(len(random_NDA_list)):
  print(f"processing {str(i)} of {str(len(random_NDA_list))}")
  NDA=random_NDA_list[i]
  multi_return_match_result.append(return_match(NDA, "additions_with_context", "patents", scoring_method_list, cutoff_percentage=.1))
# print(multi_return_match_result[0])
# import json
# json=json.dumps(multi_return_match_result)
# f=open("multi_match_result.json", "w")
# f.write(json)
# f.close()
def mean(lst):
  return sum(lst)/len(lst)

def return_mean_percent_same_for_each_nda(multi_return_match_result):
  # calculate percentage that match
  # percent_same_for_each_nda is a list of percentages of claims that are the same for each addtion for each NDA
  percent_same_for_each_nda=[]
  for NDA in multi_return_match_result:
    # matches = [[[claim_index1,claim_index2,],],] wherein the outermost list represents list for each model, the second outermost list represented the additions
    matches=[]
    model_names=[]
    # breakup dict into lists; the format of the dict is {(method_name, NDA, len(additions), len(claims)):matches, ...}
    for key, value in NDA.items():
      model_names.append(key[0])
      matches.append(value)
    if not matches or not matches[0] or not matches[0][0]:
      continue

    models_len=len(matches)
    additions_len=len(matches[0])
    claims_len=len(matches[0][0])

    # additions_percent_same is a list of the percentage of additions that have same claim_indexes across multiple models for each addition in an NDA
    additions_percent_same=[]

    for a in range(additions_len):
      # addition_matches=[[claim_index1,claim_index2,],] wherein each row represents a different model
      addition_matches=[]
      for m in range(models_len):
        addition_matches.append(matches[m][a])
      # count number of claim_index that is the same across multiple models for same addition
      same_claims=addition_matches[0]
      same_claims_orig_len=len(same_claims)
      assert same_claims_orig_len==claims_len
      for i in range(1,len(addition_matches)):
        same_claims=list(set(same_claims).intersection(addition_matches[i]))
      percent_same=len(same_claims)/same_claims_orig_len
      additions_percent_same.append(percent_same)
    
    percent_same_for_each_nda.append(mean(additions_percent_same))
  return mean(percent_same_for_each_nda)

print("Mean percent_same_for_each_nda: ", return_mean_percent_same_for_each_nda(multi_return_match_result))<jupyter_output>Mean percent_same_for_each_nda:  0.3022827958802241<jupyter_text>The above indicates that if three models are used, the top 10% of the results are shared by 30% of the models.  The following indicates if 2 models are using the results are shared by 55% of the two models.<jupyter_code>scoring_method_list=[
                     [scoring_method_bert, model_mpnet_base_v2, "model_mpnet_base_v2"],
                     [scoring_method_spacy,_en_core_sci_lg_nlp, "_en_core_sci_lg_nlp"],
                     ]

multi_return_match_result=[]
for i in range(len(random_NDA_list)):
  # print(f"processing {str(i)} of {str(len(random_NDA_list))}")
  NDA=random_NDA_list[i]
  multi_return_match_result.append(return_match(NDA, "additions_with_context", "patents", scoring_method_list, cutoff_percentage=.1))

print("Mean percent_same_for_each_nda: ", return_mean_percent_same_for_each_nda(multi_return_match_result))
scoring_method_list=[
                     [scoring_method_bert, model_mpnet_base_v2, "model_mpnet_base_v2"],
                     [scoring_method_bert,model_roberta_base_v2, "model_roberta_base_v2"],
                     ]

multi_return_match_result=[]
for i in range(len(random_NDA_list)):
  # print(f"processing {str(i)} of {str(len(random_NDA_list))}")
  NDA=random_NDA_list[i]
  multi_return_match_result.append(return_match(NDA, "additions_with_context", "patents", scoring_method_list, cutoff_percentage=.1))

print("Mean percent_same_for_each_nda: ", return_mean_percent_same_for_each_nda(multi_return_match_result))<jupyter_output>Mean percent_same_for_each_nda:  0.5502248546898962<jupyter_text>If a single model is use, the following should report 1, since all matches are the same to itself.<jupyter_code>scoring_method_list=[
                     [scoring_method_bert, model_mpnet_base_v2, "model_mpnet_base_v2"],
                     ]

multi_return_match_result=[]
for i in range(len(random_NDA_list)):
  # print(f"processing {str(i)} of {str(len(random_NDA_list))}")
  NDA=random_NDA_list[i]
  multi_return_match_result.append(return_match(NDA, "additions_with_context", "patents", scoring_method_list, cutoff_percentage=.1))

print("Mean percent_same_for_each_nda: ", return_mean_percent_same_for_each_nda(multi_return_match_result))<jupyter_output>Mean percent_same_for_each_nda:  1.0<jupyter_text>If the cutoff percentage increases, then the number of matching claims should increase.<jupyter_code>scoring_method_list=[
                      [scoring_method_bert, model_mpnet_base_v2, "model_mpnet_base_v2"],
                     [scoring_method_bert,model_roberta_base_v2, "model_roberta_base_v2"],
                     [scoring_method_spacy,_en_core_sci_lg_nlp, "_en_core_sci_lg_nlp"]
                     ]

multi_return_match_result=[]
for i in range(len(random_NDA_list)):
  # print(f"processing {str(i)} of {str(len(random_NDA_list))}")
  NDA=random_NDA_list[i]
  multi_return_match_result.append(return_match(NDA, "additions_with_context", "patents", scoring_method_list, cutoff_percentage=.2))

print("Mean percent_same_for_each_nda: ", return_mean_percent_same_for_each_nda(multi_return_match_result))
scoring_method_list=[
                      [scoring_method_bert, model_mpnet_base_v2, "model_mpnet_base_v2"],
                     [scoring_method_bert,model_roberta_base_v2, "model_roberta_base_v2"],
                     [scoring_method_spacy,_en_core_sci_lg_nlp, "_en_core_sci_lg_nlp"]
                     ]

multi_return_match_result=[]
for i in range(len(random_NDA_list)):
  # print(f"processing {str(i)} of {str(len(random_NDA_list))}")
  NDA=random_NDA_list[i]
  multi_return_match_result.append(return_match(NDA, "additions_with_context", "patents", scoring_method_list, cutoff_percentage=.3))

print("Mean percent_same_for_each_nda: ", return_mean_percent_same_for_each_nda(multi_return_match_result))
scoring_method_list=[
                      [scoring_method_bert, model_mpnet_base_v2, "model_mpnet_base_v2"],
                     [scoring_method_bert,model_roberta_base_v2, "model_roberta_base_v2"],
                     [scoring_method_spacy,_en_core_sci_lg_nlp, "_en_core_sci_lg_nlp"]
                     ]

multi_return_match_result=[]
for i in range(len(random_NDA_list)):
  # print(f"processing {str(i)} of {str(len(random_NDA_list))}")
  NDA=random_NDA_list[i]
  multi_return_match_result.append(return_match(NDA, "additions_with_context", "patents", scoring_method_list, cutoff_percentage=.1))
def print_top_percentage_multi(multi_return_match_result, NDA_, additions_folder_name, patent_folder_name):
  """
  Prints the same matches from an NDA
  Parameters:
    multi_return_match_result=[{ ("method_name", NDA, len(additions), len(claims)): [[additions_num, claim_num],], },] for a list of NDA
  """
  # calculate percentage that match
  # percent_same_for_each_nda is a list of percentages of claims that are the same for each addtion for each NDA
  for NDA_set in multi_return_match_result:
    # print(str(NDA)[:50])
    key_=list(NDA_set.keys())[0]
    value_ = NDA_set.values()
    if key_[1]!=NDA_:
      continue
    claims = get_patent_claims(NDA, patent_folder_name)
    additions = get_additions(NDA, additions_folder_name)

    # matches = [[[claim_index1,claim_index2,],],] wherein the outermost list represents list for each model, the second outermost list represented the additions
    matches=[]

    # breakup dict into lists; the format of the dict is {(method_name, NDA, len(additions), len(claims)):matches, ...}
    for key, value in NDA_set.items():
      matches.append(value)
    if not matches or not matches[0] or not matches[0][0]:
      continue

    models_len=len(matches)
    additions_len=len(matches[0])
    claims_len=len(matches[0][0])

    print(f"==NDA: {NDA_}")
    patent_dir=db2files+str(NDA)+'/'+patent_folder_name+'/'
    if os.path.exists(patent_dir):
      patent_files=[patent_dir+f for f in os.listdir(patent_dir)]
      patent_files=[f.split('/')[-1] for f in patent_files]
    print(f"==patents: {patent_files}")

    for a in range(additions_len):

      # addition_matches=[[claim_index1,claim_index2,],] wherein each row represents a different model
      addition_matches=[]
      for m in range(models_len):
        addition_matches.append(matches[m][a])
      # count number of claim_index that is the same across multiple models for same addition
      same_claims=addition_matches[0]
      same_claims_orig_len=len(same_claims)
      assert same_claims_orig_len==claims_len
      for i in range(1,len(addition_matches)):
        same_claims=list(set(same_claims).intersection(addition_matches[i]))

      if same_claims:
        print(f"==addition: {additions[a]}")
        print(f'==same_claims_list: {same_claims}')
        
        for c in same_claims:
          print(f"********claim: \n{claims[c]}")
  
    
    break
    
  
# Change i to see all addition to claim matching 
i=1
NDA=random_NDA_list[0]

print_top_percentage_multi(multi_return_match_result, NDA, "additions_with_context", "patents")
i=1
NDA=random_NDA_list[i]

print_top_percentage_multi(multi_return_match_result, NDA, "additions_with_context", "patents")
i=2
NDA=random_NDA_list[i]

print_top_percentage_multi(multi_return_match_result, NDA, "additions_with_context", "patents")
i=3
NDA=random_NDA_list[i]

print_top_percentage_multi(multi_return_match_result, NDA, "additions_with_context", "patents")<jupyter_output>==NDA: 208051
==patents: ['8669273', '9630946', '10035788', '8518446', '9139558', '9211291', '8790708', '9265784', '7399865', '7982043']
==addition: NERLYNX is indicated for the extended adjuvant treatment of adult patients with early stage HER2-overexpressed/amplified breast cancer, to follow adjuvant trastuzumab based therapy [
  see Clinical Studies (
   14)
  ].

==same_claims_list: [0, 101, 103, 109, 142, 112, 115, 118, 22]
********claim: 
1. A method for treating an ErbB-2 positive metastatic breast cancer in a subject; comprising administering to the subject neratinib and capecitabine; wherein neratinib and capecitabine act synergistically.

********claim: 
1. A regimen for treatment of early stage HER-2 positive breast cancer comprising delivering neratinib therapy to early stage HER-2 positive breast cancer patients at the completion of at least twelve cycles of trastuzumab adjuvant therapy; wherein the neratinib therapy is started about 2 weeks to about one year from the comp[...]<jupyter_text>The following is the result of a bad data grab.  All claims are group together as one unit in database.  Data grab is part of unrelated module.<jupyter_code>i=4
NDA=random_NDA_list[i]

print_top_percentage_multi(multi_return_match_result, NDA, "additions_with_context", "patents")
i=5
NDA=random_NDA_list[i]

print_top_percentage_multi(multi_return_match_result, NDA, "additions_with_context", "patents")
i=6
NDA=random_NDA_list[i]

print_top_percentage_multi(multi_return_match_result, NDA, "additions_with_context", "patents")
i=7
NDA=random_NDA_list[i]

print_top_percentage_multi(multi_return_match_result, NDA, "additions_with_context", "patents")
i=8
NDA=random_NDA_list[i]

print_top_percentage_multi(multi_return_match_result, NDA, "additions_with_context", "patents")
i=9
NDA=random_NDA_list[i]

print_top_percentage_multi(multi_return_match_result, NDA, "additions_with_context", "patents")
i=10
NDA=random_NDA_list[i]

print_top_percentage_multi(multi_return_match_result, NDA, "additions_with_context", "patents")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09896291088064686, 'fraction_numerical': 0.013798558621901916, 'mean_word_length': 4.490229191797346, 'pattern_counts': {'":': 4, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '337309', 'license': 'Apache-2.0', 'n_tokens_mistral': 8037, 'n_tokens_neox': 7689, 'n_words': 1656, 'path': 'analysis/Matching_Top10Percent_DrugLabelsandPatent.ipynb', 'repo_name': 'pharmaDB/scoring_data_processor', 'size': '22756'}