<jupyter_start><jupyter_text>**summarization.py**<jupyter_code>import os
os.environ["COVID_WEBAPP_DATA_DIR"] = "/Users/eugenetang/Development/data/covid_19/cord19/20200609"
import summarization
bert_summarizer = summarization.BertSummarizer()

%load_ext autoreload
%autoreload 2
sample_text = '<NAME> Elizabeth took a taxi to attend the night party in the city. While in the party, Elizabeth collapsed and was rushed to the hospital. Since she was diagnosed with a brain injury, the doctor told Peter to stay besides her until she gets well. Therefore, Peter stayed with her at the hospital for 3 days without leaving.'
summarization.summarization_tf(sample_text, num_sentences=2)
bert_summarizer.summarize_text(sample_text)
bert_summarizer.extract_doc_embedding(sample_text)
#Build two sets of embeddings

sample_text = '<NAME> Elizabeth took a taxi to attend the night party in the city. While in the party, Elizabeth collapsed and was rushed to the hospital. Since she was diagnosed with a brain injury, the doctor told Peter to stay besides her until she gets well. Therefore, Peter stayed with her at the hospital for 3 days without leaving.'
sample_text2 = 'Peter and Elizabeth took a taxi to attend the night party in the city. While in the party, Elizabeth collapsed and was rushed to the hospital. Since she was diagnosed with a brain injury.'
sample_text3 = 'I am so excited to be finally completing my MIDS program.'

embedding1 = bert_summarizer.extract_doc_embedding(sample_text)
embedding2 = bert_summarizer.extract_doc_embedding(sample_text2)
embedding3 = bert_summarizer.extract_doc_embedding(sample_text3)
print("Similar", summarization.summarization_evaluation(embedding1,embedding2))
print("Different", summarization.summarization_evaluation(embedding1,embedding3))<jupyter_output>Similar [[0.9868824]]
Different [[0.8636333]]<jupyter_text>Code below is untested<jupyter_code>import sys
sys.path.append('../modules/')
import summarization
import pickle
import os

# CONFIG

#DATA_DIR = '/home/ubuntu/efs-mnt/latest/'
DATA_DIR = '/Users/mursil.makhani/Drive_MIDS/workspace/mids/w210_capstone/'

#DATA_INPUT = 'df_covid_with_text.pkl'
DATA_INPUT = 'sample_data_with_text.pkl'
DATA_OUTPUT = 'df_covid_with_summary.pkl'

COLUMN_INPUT = 'text'

# Load Data

with open(os.path.join(DATA_DIR,DATA_INPUT),'rb') as f:
    df_covid = pickle.load(f)
    text_data = df_covid[COLUMN_INPUT]
text_data.reset_index().text[0]
model, tokenizer = summarization.build_scibert_objects()
summarizer = summarization.build_summarizer_scibert(model=model, model_tokenizer=tokenizer)
embedding_extractor = summarization.build_embedding_extractor_scibert(model=model, model_tokenizer=tokenizer)

#df_covid['scibert_embedding'] = text_data.apply(summarization.extract_embedding_scibert,extractor=embedding_extractor) 
#df_covid['scibert_summary'] = text_data.apply(summarization.summarization_scibert,summarizer=summarizer)

summarization.extract_embedding_scibert(text_data.reset_index().text[1])<jupyter_output>running extraction
{'fraction_non_alphanumeric': 0.08336089976844195, 'fraction_numerical': 0.01587826662256037, 'mean_word_length': 7.330578512396694, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '197801', 'license': 'MIT', 'n_tokens_mistral': 1003, 'n_tokens_neox': 968, 'n_words': 249, 'path': 'modules/module_testing.ipynb', 'repo_name': 'eugenet12/covidresearchviewer', 'size': '3029'}