<jupyter_start><jupyter_text>Extracting LitCovid in local env<jupyter_code>!wget https://ftp.ncbi.nlm.nih.gov/pub/lu/LitCovid/litcovid2pubtator.xml.gz
!gunzip litcovid2pubtator.xml.gz
!cat litcovid2pubtator.xml | grep C063533
!wc -l /content/drive/MyDrive/Research/BLAH7/sentences.txt
pip install bioc
filename = 'litcovid2pubtator.xml'
from nltk import tokenize
import nltk
nltk.download('punkt')
import collections
import uuid 
def get_lens_passage(passage):
  sentences = tokenize.sent_tokenize(passage.text)
  lens_sentences = []
  ids_sentences = []
  for sentence in sentences:
    lens_sentences.append(len(sentence))
    ids_sentences.append(uuid.uuid4())
  return(lens_sentences,sentences, ids_sentences)
def extract_annotations_passage(passage, type_of_annotation='MESH'):
  annotations_processed = []
  for annotation in passage.annotations:
    tmp =  {k.lower(): v for k, v in annotation.infons.items()}
    if type_of_annotation in tmp['identifier']:
      identifier = tmp['identifier']
      type_of_identifier = tmp['type']
      locations = []
      for location in annotation.locations:
        locations.append(location.offset)
      text = annotation.text
      annotations_processed.append({'identifier': identifier,
                                    'type_of_identifier': type_of_identifier,
                                    'locations': locations,
                                    'text': text})
  if len(annotations_processed) > 0:
    return(annotations_processed)
  else:
    return None
def match_annotations_sentences(passage, file_handler=None, dict_annotations=None):
  annotations_processed = extract_annotations_passage(passage)
  lens_sentences, sentences,ids_sentences = get_lens_passage(passage)
  if file_handler is not None:
    for i in range(len(sentences)):
      file_handler.write(str(ids_sentences[i]) + '\t' + sentences[i] + '\n')
  else:
    for i in range(len(sentences)):
      print(str(ids_sentences[i]) + '\t' + sentences[i])
  #dict_annotations = collections.defaultdict(set)
  offsets_sentences = []
  offset_passage = passage.offset
  for i in range(len(lens_sentences)):
    if i == 0:
      offsets_sentences.append(lens_sentences[i]+passage.offset) 
    else:
      offsets_sentences.append(lens_sentences[i]+offsets_sentences[i-1])
  if annotations_processed is not None:
    for annotation in annotations_processed:
      min_location = min(annotation['locations'])
      #print(min_location)
      location_annotation = next((x[0] for x in enumerate(offsets_sentences) if x[1] >= min_location), len(offsets_sentences)-1)
      dict_annotations[annotation['identifier']].add(str(ids_sentences[location_annotation]))
  return(dict_annotations)

import bioc
import io
# read from a ByteIO
dict_all = collections.defaultdict(set)
f_out = output = io.StringIO()
with open(filename, 'rb') as fp:
    reader = bioc.BioCXMLDocumentReader(fp)
    collection_info = reader.get_collection_info()
    i = 0
    for document in reader:
        for passage in document.passages:
          dict_all = match_annotations_sentences(passage, file_handler=f_out, dict_annotations=dict_all)
#          if tmp is not None:
#            dict_all = mergeDict(dict_all,tmp)
        i+=1
        print(i)
        if i % 30000==0:
          break

import pickle
with open('/content/drive/MyDrive/Research/BLAH7/filename.pickle', 'wb') as handle:
    pickle.dump(dict_all, handle, protocol=pickle.HIGHEST_PROTOCOL)

import shutil

with open('/content/drive/MyDrive/Research/BLAH7/sentences.txt', 'w') as fd:
  f_out.seek(0)
  shutil.copyfileobj(f_out, fd, -1)
import pickle
with open('/content/drive/MyDrive/Research/BLAH7/filename.pickle', 'rb') as handle:
  dict_all = pickle.load(handle)
tmp_passage1 = extract_annotations_passage(document.passages[1])
with open('/content/drive/MyDrive/Research/BLAH7/occurences_v1.tsv','w') as f:
  for key in dict_all:
    f.write(key + '\t' +  str(len(dict_all[key])) + '\n')
with open('/content/drive/MyDrive/Research/BLAH7/initial_terms_to_translate.txt','r') as f:
  terms_to_translate = f.readlines()
del(f_out)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1112167300380228, 'fraction_numerical': 0.008555133079847909, 'mean_word_length': 3.723905723905724, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '591346', 'license': 'MIT', 'n_tokens_mistral': 1442, 'n_tokens_neox': 1367, 'n_words': 261, 'path': 'Approach_with_tags.ipynb', 'repo_name': 'soares-f/O-JMeSH', 'size': '4208'}