<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/gdrive')
!pip install langid
!pip install tqdm
import zipfile
import os
import sys
import pandas as pd
import numpy as np
import gc
from urllib import request
import json
import itertools
import gzip
import shutil
import ast
import pickle 
import nltk
from langid.langid import LanguageIdentifier, model
from tqdm import tqdm
# Function to unip a file from semantic scholar to a json file, then open that json file
def dlandopen(url, file, json ):
    request.urlretrieve(url, file )
    with gzip.open(file, 'rb') as f_in:
        with open(json, 'wb') as f_out:
            shutil.copyfileobj(f_in, f_out)

    os.remove(file)
    f = open(json)
    mylistH = f.readlines()
    del f
    return mylistH
# This is probably not the best way to go about it, but this is the code to convert the opened file into a pandas dataframe
#Akul, it may be worth figuring out if you can make a dataframe directly, without going line-by-line 
def firstDataframe(mylist, start, finish):
    data = []

    for line in itertools.islice(mylist , start , finish):
       data.append(json.loads(line))

    df = pd.DataFrame(data)
    del data
    gc.collect()
    return df
def medicalRefinement(dftbr):
  dftbr['strsources'] = dftbr['sources'].astype('str')
  dftbr['totalCites2'] = dftbr['outCitations'] + dftbr['inCitations'] 
  dftbr = dftbr[dftbr['totalCites2'].str.len() >= 1]  #outCitations - all papers where papers cited is greater than 1
  dftbr = dftbr[ (dftbr.pmid != '' ) | (dftbr.strsources.str.contains("Medline"))]
  dftbr.drop(['journalVolume','journalPages','year','authors','sources','doiUrl', 'strsources'], axis = 1)
  return dftbr
def remove_nonenglish(df, jSet, vSet):
  # Gets list of English Journals
  listOfEnglishJournals = []
  identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)
  count = 0
  for i in jSet:
    lang = identifier.classify(str(i))
    langstr = str(lang)
    if langstr.find("en") != -1 and lang[1] > 0.5:
      listOfEnglishJournals.append(i)
  
  # Gets List of English Venues
  listOfEnglishVenues = []
  count = 0
  for i in vSet:
    lang = identifier.classify(str(i))
    langstr = str(lang)
    if langstr.find("en") != -1 and lang[1] > 0.5:
      listOfEnglishVenues.append(i)

  filter1 = df_refined['journalName'].isin(listOfEnglishJournals)
  filter2 = df_refined['venue'].isin(listOfEnglishVenues)

  dfFinal = df_refined[df_refined['journalName'].isin(listOfEnglishJournals) | df_refined['venue'].isin(listOfEnglishVenues)]

  return dfFinal
url_template = 'https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2019-11-01/s2-corpus-{:03}.gz'
file_template = 'json{:03}.gz'
json_template = 'json{:03}'

i=55
mylist = dlandopen(url_template.format(i), file_template.format(i), json_template.format(i) )
df = firstDataframe(mylist, 0, len(mylist))
df_refined = medicalRefinement(df)
print("REFINED")
df_refined.shape
journalSet = set()
venueSet = set()
journalSet = journalSet|set(df_refined['journalName'])
venueSet = journalSet|set(df_refined['venue'])
english_only_df = remove_nonenglish(df_refined, journalSet, venueSet)
print("NONENGLISH REMOVED")
english_only_df.shape
english_only_df.to_json("/content/gdrive/My Drive/text2doc_data/med_data/med-{:03}.json".format(i), orient='index')
url_template = 'https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2019-11-01/s2-corpus-{:03}.gz'
file_template = 'json{:03}.gz'
json_template = 'json{:03}'

all_length = 0
english_length = 0

i = 175

while i <= 178:
  mylist = dlandopen(url_template.format(i), file_template.format(i), json_template.format(i) )
  df = firstDataframe(mylist, 0, len(mylist))
  df_refined = medicalRefinement(df)
  df_refined.to_json("/content/gdrive/My Drive/text2doc_data/med_data_all/med-{:03}.json".format(i), orient='index')
  all_length += df_refined.shape[0]
  print("REFINED")
  journalSet = set()
  venueSet = set()
  journalSet = journalSet|set(df_refined['journalName'])
  venueSet = journalSet|set(df_refined['venue'])
  english_only_df = remove_nonenglish(df_refined, journalSet, venueSet)
  print("NONENGLISH REMOVED")
  english_length += english_only_df.shape[0]
  english_only_df.to_json("/content/gdrive/My Drive/text2doc_data/med_data_english_only/med-{:03}.json".format(i), orient='index')
  i+=1

print("All length: " + all_length)
print("English length: " + english_length)<jupyter_output>REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
REFINED
NONENGLISH REMOVED
{'fraction_non_alphanumeric': 0.1167422599092881, 'fraction_numerical': 0.015578781305462433, 'mean_word_length': 5.246305418719212, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '163049', 'license': 'MIT', 'n_tokens_mistral': 1934, 'n_tokens_neox': 1875, 'n_words': 410, 'path': 'notebooks/data/medical_preprocessing.ipynb', 'repo_name': 'EXYNOS-999/NaturalLanguageRecommendations', 'size': '5071'}