<jupyter_start><jupyter_text>Set-up Code<jupyter_code>!pip install git+https://github.com/bmabey/pyLDAvis.git@master#egg=pyLDAvis -q
!pip install whoosh -q

import pyLDAvis.sklearn

import pandas as pd
import numpy as np
import os
from pathlib import Path, PurePath
from ipywidgets import interact
import ipywidgets as widgets
from collections import defaultdict
import json

import whoosh
from whoosh.qparser import *
from whoosh.fields import Schema, TEXT, KEYWORD, ID, STORED, NUMERIC, NGRAMWORDS
from whoosh.analysis import StemmingAnalyzer,StandardAnalyzer, NgramFilter
from whoosh import index
# set data paths, this requires local drive to have a folder calld "COVID-19" with the clean_metadata.csv file
# returns a string to the local path setup
def setup_local_data():
  from google.colab import drive
  drive.mount('/content/drive')
  drive_path=PurePath('/content/drive/My Drive')
  input_dir = drive_path/'COVID-19'
  print(list(Path(input_dir).glob('*')))
  return input_dir
#read the metadata file into df
def read_metadata_csv(input_dir):
  metadata_path = input_dir / 'clean_metadata.csv'
  metadata = pd.read_csv(metadata_path,
                         dtype={'cord_uid':str,
                                'sha':str,
                                'publish_time': str, 
                                'authors':str,
                                'title': str,
                                'abstract':str,
                                'url': str},
                         parse_dates = ['publish_time']
                         )
  #set the abstract to the paper title if it is null
  metadata['abstract'] = metadata['abstract'] .fillna(metadata['title'])
  metadata['publish_year'] = pd.DatetimeIndex(metadata['publish_time']).year
  metadata = metadata.drop('Unnamed: 0', axis=1)
  return metadata
def read_full_data_csv(input_dir):
  path = input_dir / 'covid_data_full_v5.csv'
  full_data = pd.read_csv(path)
  return full_data
def read_query_dictionary(input_dir):
  path = input_dir / 'covid_query_dictionary.json'
  with open(path) as f:
    query_dict = json.load(f)
  return query_dict
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.feature_extraction.text import CountVectorizer

def get_count_vectorizer():
  vectorizer = CountVectorizer(min_df =1, max_df=1, #min_df=5, max_df=0.9, 
                             stop_words='english', lowercase=True, 
                             token_pattern='[a-zA-Z\-][a-zA-Z\-]{2,}')
  return vectorizer

def get_lda_model(num_topics):
  lda_model = LatentDirichletAllocation(n_components=num_topics, max_iter=10, learning_method='online')
  return lda_model
#get hardcoded schema for the index
def get_search_schema():
  schema = Schema(uid = TEXT(stored=True),
                  sha = TEXT(stored=True),
                  year=NUMERIC(stored=True),
                  author=TEXT(stored=True),
                  title=TEXT(analyzer=StandardAnalyzer(),stored=True),
                  abstract=TEXT(analyzer=StandardAnalyzer(),stored=True),
                  url=TEXT(stored=True))
  return schema

# creates an index in a dictionary (only need to run once)
# noop if it's already created
def create_search_index(drive_path, search_schema):
  if not os.path.exists(drive_path/'indexdir'):
      os.mkdir(drive_path/'indexdir')
  ix = index.create_in(drive_path/'indexdir', search_schema)
  # #open an existing index object
  # ix = index.open_dir(drive_path/'indexdir')
  return ix


def add_documents_to_index(ix, metadata):
  #cancel writer in case re-indexing is needed
  if 'writer' in locals():
    writer.cancel()
  #create a writer object to add documents to the index
  writer = ix.writer()

  #now we can add documents to the index
  uid = metadata['cord_uid']
  sha = metadata['sha']
  year = metadata['publish_year']
  author = metadata['authors']
  title = metadata['title']
  abstract = metadata['abstract']
  url = metadata['url']

  for UID, SHA, YEAR, AUTHOR, TITLE, ABSTRACT, URL in zip(uid, sha, year,author,title,abstract,url):
    writer.add_document(uid = str(UID),
                        sha= str(SHA),
                        year= int(YEAR),
                        author=str(AUTHOR),
                        title=str(TITLE),
                        abstract=str(ABSTRACT),
                        url=str(URL))

  #close the writer and save the added documents in the index
  #you should call the commit() function once you finish adding the documents otherwise you will cause an error-
  #when you try to edit the index next time and open another writer. 
  writer.commit()

  # need to cancel writer if error or need to reset
  # writer.cancel()
  return

# get a multifield parser for the list of inptted fields
def get_multifield_parser(fields, search_schema):
  parser = MultifieldParser(fields, schema=search_schema)
  parser.add_plugin(GtLtPlugin())
  parser.add_plugin(SequencePlugin())
  parser.add_plugin(PhrasePlugin())
  return parser

# this takes in a parser and query string to return the actual query that'll be sent to the searcher
def get_parser_query(parser, query):
  result = parser.parse(query) # use boolean operators in quotation
  print(result)
  return result

# Convert the doi to a url
# def doi_url(d): 
#     return f'http://{d}' if d.startswith('doi.org') else f'http://doi.org/{d}'
    
# this method takes in a search index and query to return a dataframe of results
# ix is the document index we created before
# query is the string found from the parser
def get_search_results(ix, query):
  #you can open the searcher using a with statement so the searcher is automatically closed when you’re done with it
  with ix.searcher() as searcher:
      results = searcher.search(query, limit=None)#The Results object acts like a list of the matched documents
      print('Total Hits: {}\n'.format(len(results)))
      # print(results)
      output_dict = defaultdict(list)
      num = 0
      for result in results:
        # print(result)
        num = num + 1
        output_dict['cord_uid'].append(result['uid'])
        output_dict['sha'].append(result['sha'])
        output_dict['bm25_score'].append(result.score)
        output_dict['title'].append(result['title'])
        output_dict['abstract'].append(result['abstract'])
        output_dict['publish_year'].append(result['year'])
        output_dict['authors'].append(result['author'])
        output_dict['url'].append(result['url'])
      print("Loop ran {} times.".format(num))

  # print(len(output_dict['title']))
  output_df = pd.DataFrame(output_dict)
  print(output_df.shape)
  # output_df['doi'] = output_df['doi'].apply(lambda x: doi_url(x) if x !='nan' else x)  
  return output_df
from ipywidgets import interact, Layout, HBox, VBox, Box
from IPython.display import HTML, display, clear_output
import ipywidgets as widgets
from IPython.display import update_display

def get_new_text_box():
  textW = widgets.Textarea(
        value='',
        placeholder='Type something like "covid" or incubation',
        description='',
        disabled=False,
        layout=Layout(width='100%', height='50px')
    )
  return textW

def get_new_plus_button():
  button = widgets.Button(description="+")
  return button

def get_new_dropdown():
  dropdown = widgets.Dropdown(
      options=['AND', 'OR', 'NOT'],
      value='AND',
      description='Operator: ',
      disabled=False,
    )
  return dropdown

def dynamic_search_query(parser, ix):
  textW = widgets.Textarea(
        value='',
        placeholder='Type something like "covid" or incubation',
        description='',
        disabled=False,
        layout=Layout(width='100%', height='50px')
    )
  
  button = widgets.Button(description="+")
  search_rows_list = []
  search_rows_list.append( HBox([textW, button], layout=Layout(align_items='center')) )
  # vboxes = [
  #           VBox(search_rows_list, layout=Layout(align_items='center')),
  #           VBox([button], layout=Layout(object_position='bottom'))
  #           ]
  # display_handle = display(vboxes, display_id='disp')
  display_handle = display(VBox(search_rows_list, layout=Layout(align_items='center')), display_id='disp')
  # display(HBox([textW, button], layout=Layout(align_items='center')))

  #search_rows_list is a list of HBox objects
  # the first index will just be a text box and '+' button
  # subsequent rows will have operator, text box, and '+' button
  def on_button_clicked(b):
    global STORED_SEARCH_QUERY
    # global search_rows_list
    clear_output(wait=True)
    new_text_box = get_new_text_box()
    dropdown = get_new_dropdown()
    # new_plus_button = get_new_plus_button()
    search_rows_list.append( HBox([dropdown, new_text_box, button], layout=Layout(align_items='center')) )
    # print(len(search_rows_list))
    # display(VBox(search_rows_list, layout=Layout(align_items='center')))
    display_handle.update(VBox(search_rows_list, layout=Layout(align_items='center')))

    combined = ''
    for i in range(0, len(search_rows_list)-1): #we do len - 1 since newet row has no values
      row = search_rows_list[i]
      # print(row)
      # print(i)
      if i == 0:
        temp = combined + row.children[0].value
        combined = temp
      else:
        temp = combined + ' ' + row.children[0].value + ' ' + row.children[1].value
        combined = temp
    
    print("Current raw search query:\n" + combined)
    print("Current query from parser:")
    query = get_parser_query(parser, combined) #already prints in method
    STORED_SEARCH_QUERY = query
    # update_display(display_id)

  button.on_click(on_button_clicked)

def get_query_dropdown():
  query_dictonary = read_query_dictionary(local_dir)
  dropdown = widgets.Dropdown(
      options=list(query_dictonary.keys()),
      value='What is the range of incubation periods?',
      description='Select a question:',
      disabled=False,
  )
  return dropdown
def precoded_search_query(parser, ix):
  query_dictonary = read_query_dictionary(local_dir)
  select_question = get_query_dropdown()
  def on_change(change):
    if change['name'] == 'value' and (change['new'] != change['old']): #### bug to fix 
      global STORED_PRECODED_QUERY
      precoded_query = query_dictonary[select_question.value]
      query = get_parser_query(parser, precoded_query) #already prints in method
      STORED_PRECODED_QUERY = query
  
  select_question.observe(on_change)
  display(select_question)<jupyter_output><empty_output><jupyter_text>Main Code Search Engine<jupyter_code>local_dir = setup_local_data()
metadata = read_metadata_csv(local_dir)
print(metadata.info())
print(metadata.shape)
print(metadata.head(5))

search_schema = get_search_schema()
##uncomment this line if indexing database for the first time
# ix = create_search_index(local_dir, search_schema)
# add_documents_to_index(ix, metadata)
#read saved index from directory
ix = index.open_dir(local_dir/'indexdir')

#set default search fields to title OR abstract
fields = ["title", "abstract"] 
parser = get_multifield_parser(fields, search_schema)
STORED_SEARCH_QUERY = '' #this is stored as a global so the last search query from the parser can be used in later cells<jupyter_output><empty_output><jupyter_text>Search InstructionsThe search engine searches in both title and abstract by default. To search keywords in title or abstract separtely, see example below:**title:("covid-19") AND abstract:("incubation period" OR exposure)**Notes: Enclose hyphenated words or phrases with quoatationEnclose a group of words to search within a field with parenthesis") Manual dynamic search<jupyter_code>#STORED_SEARCH_QUERY
dynamic_search_query(parser, ix)
print(STORED_SEARCH_QUERY)<jupyter_output><jupyter_text>Select question from list<jupyter_code>#STORED_PRECODED_QUERY
precoded_search_query(parser, ix)
print(STORED_PRECODED_QUERY)
#plug in STORED_SEARCH_QUERY if using manual search
#plug in STORED_PRECODED_QUERY if using a precoded-query
output = get_search_results(ix, STORED_PRECODED_QUERY)
# print(output.shape)
# output
# data = pd.concat([metadata['title'], metadata['abstract']], axis=1)
# metadata = metadata.dropna(subset=['title', 'abstract'])
# data = metadata['title'] + ' ' + metadata['abstract'] #space ensures title_last_word and abstract_first_word are stored as separate words

# output = output.dropna(subset=['title', 'abstract'])
print(output.shape)
data = output['title'] + ' ' + output['abstract']

print(type(data))
print(len(data))
data = data.tolist()
print(data[0])
output.head()
#save to csv
if not os.path.exists(local_dir/'SearchEngineOutput'):
  os.mkdir(local_dir/'SearchEngineOutput')
output.to_csv(local_dir/'SearchEngineOutput'/'search_engine_duration_infectiousness.csv', index=False)<jupyter_output><empty_output><jupyter_text>Use topic model to find similar articles<jupyter_code>#vectorize search output
count_vectorizer = get_count_vectorizer()
data_vectorized = count_vectorizer.fit_transform(data)
#train LDA using vectorized search output
print(count_vectorizer.get_feature_names())
print(len(count_vectorizer.get_feature_names()))
lda_model = get_lda_model(num_topics=10)
lda_Z = lda_model.fit_transform(data_vectorized)
print(lda_Z.shape)  # (NO_DOCUMENTS, NO_TOPICS)
print(lda_Z[0]) #we can see that this adds up to 1 since LDA is a summation of topic probabilities
print(lda_model.components_) 
print(lda_model.components_.shape) # this is num_components/topics by num_features, num_features comes from feature names of count vectorizer
#get average topic distribution of all search output
avg_topic = lda_Z.mean(axis=0)
print(avg_topic)
print(avg_topic.shape)
#print topic components
def print_topics(model, vectorizer, top_n=10):
    for idx, topic in enumerate(model.components_):
        print("Topic %d:" % (idx))
        print([(vectorizer.get_feature_names()[i], topic[i])
                        for i in topic.argsort()[:-top_n - 1:-1]])
        
print("LDA Model:")
print_topics(lda_model, count_vectorizer)
# !pip install --upgrade pandas
# visualize topics using pyLDAvis
pyLDAvis.enable_notebook()
panel = pyLDAvis.sklearn.prepare(lda_model, data_vectorized, count_vectorizer, mds='tsne')
panel
from scipy.spatial import distance
print(distance.cosine(avg_topic, lda_Z[0]))

# For one d arrays below, it doesn't make sense to transpose as 2d since calc is always 0
# Cosine score of 0 means 2 vectors are at 90degrees/orthagonal, have no match
# avg_topic = np.array(avg_topic).reshape(-1,1)
# print(avg_topic.shape)
# distances = distance.cdist(avg_topic, avg_topic, metric='cosine')
# print(distances)
# print(avg_topic)
# print(np.array(lda_Z[0]).reshape(-1, 1))
# example = distance.cdist(avg_topic, np.array(lda_Z[0]).reshape(-1, 1), metric='cosine')
# print(example)
#vectorize all title + abstract in metadata
print(data_vectorized.shape)
metadata = metadata.dropna(subset=['title', 'abstract'])
metadata_condensed = metadata['title'] + ' ' + metadata['abstract'] #space ensures title_last_word and abstract_first_word are stored as separate words
metadata_condensed = metadata_condensed.tolist()
print(metadata_condensed[0])

metadata_condensed_vectorized = count_vectorizer.transform(metadata_condensed)
print(metadata_condensed_vectorized.shape)
#derive topic distributions for each document in corpus
lda_corpus = lda_model.transform(metadata_condensed_vectorized)
#for each entry in the corpus, add a column of cosine distance compared to the avg topic from search query
print(lda_corpus.shape)
for entry in lda_corpus[:5]:
  print(entry)

distances = []
for entry in lda_corpus:
  distances.append(distance.cosine(avg_topic, entry))
print(distances[0])
metadata['cosine_dist_from_topic'] = np.asarray(distances)
metadata.head(5)
#then display top results from overall corpus
metadata_sorted = metadata.sort_values('cosine_dist_from_topic', ascending=True) #ascending because smaller distance is better
# metadata_sorted.head(5)
metadata_sorted[:20]['title']
metadata_sorted['query'] = 'How effective are personal protective equipment?'
#save to csv
if not os.path.exists(local_dir/'TopicModelOutput'):
  os.mkdir(local_dir/'TopicModelOutput')
metadata_sorted[:50].to_csv(local_dir/'TopicModelOutput'/'PPE_top50.csv', index=False)

frames = []
for f in list(Path(local_dir/'TopicModelOutput').glob('*')):
  frames.append(pd.read_csv(f))
AllOutput = pd.concat(frames).reset_index(drop=True)
AllOutput.info()
AllOutput = AllOutput[['cord_uid','sha','cosine_dist_from_topic','title','abstract','publish_year','authors','url','query']]
AllOutput.to_csv(local_dir/'PrelimResults'/'LDA_baseline_results.csv', index=False)
AllOutput.to_json(local_dir/'PrelimResults'/'LDA_baseline_results.json', orient='records')<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 250 entries, 0 to 249
Data columns (total 20 columns):
 #   Column                       Non-Null Count  Dtype  
---  ------                       --------------  -----  
 0   cord_uid                     250 non-null    object 
 1   sha                          250 non-null    object 
 2   source_x                     250 non-null    object 
 3   title                        250 non-null    object 
 4   doi                          250 non-null    object 
 5   pmcid                        134 non-null    object 
 6   pubmed_id                    184 non-null    float64
 7   license                      250 non-null    object 
 8   abstract                     250 non-null    object 
 9   publish_time                 250 non-null    object 
 10  authors                      247 non-null    object 
 11  journal                      217 non-null    object 
 12  Microsoft Academic Paper ID  12 non-null     float64
 13  WHO #Covidence      [...]
{'fraction_non_alphanumeric': 0.10772529766943174, 'fraction_numerical': 0.008182382484058462, 'mean_word_length': 3.8834389639019014, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '203808', 'license': 'MIT', 'n_tokens_mistral': 5603, 'n_tokens_neox': 5382, 'n_words': 1380, 'path': 'task_ties/TIES_LDA_POC.ipynb', 'repo_name': 'CoronaWhy/infectivity', 'size': '17721'}