<jupyter_start><jupyter_text>Using Scispacy<jupyter_code>### chose to do this from cli
#%pip install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.4.0/en_core_sci_sm-0.4.0.tar.gz
import pandas as pd

import scispacy
import spacy

from scispacy.linking import EntityLinker

import sys
directory_path = "/Users/dragonzord/Documents/SchoolDocs/codeup-data-science/exercises/methodologies2/capstone/nlp-capstone/src"
sys.path.append(directory_path)
for p in sys.path:
    print(p)
    
# these imports must come after adding to path
import prepare_jag
from casenums import UniqueDataFrames
patient_notes = pd.read_csv("data/patient_notes.csv")
features = pd.read_csv("data/features.csv")

print(patient_notes[patient_notes["case_num"] == 0][:10])
features[features["case_num"] == 0][:10]
case_num_zero = patient_notes["pn_history"][patient_notes["case_num"] == 0][:1500]
whole_case_num_zero = "\n".join(case_num_zero.to_list())
len(whole_case_num_zero)
whole_case_num_zero[:1000]
clean_note = prepare_jag.basic_clean3(whole_case_num_zero)
clean_note = prepare_jag.remove_stopwords(clean_note, exclude_words=["no"])
len(clean_note)<jupyter_output><empty_output><jupyter_text>run this always<jupyter_code>sci_nlp = spacy.load("en_ner_bc5cdr_md")
# This line takes a while, because we have to download ~1GB of data
# and load a large JSON file (the knowledge base). Be patient!
# Thankfully it should be faster after the first time you use it, because
# the downloads are cached.
print("works, yay!")<jupyter_output>works, yay!<jupyter_text>From https://gbnegrini.com/post/biomedical-text-nlp-scispacy-named-entity-recognition-medical-records/<jupyter_code>doc = sci_nlp(clean_note)
print("TEXT", "START", "END", "ENTITY TYPE")
for ent in doc.ents:
    print(ent.text, ent.start_char, ent.end_char, ent.label_)<jupyter_output><empty_output><jupyter_text>- case num 0 patient notes<jupyter_code>case_num_zero_notes = patient_notes[patient_notes["case_num"] == 0]
case_num_zero_notes.head(3)<jupyter_output><empty_output><jupyter_text>- case num 0 feature text<jupyter_code>case_num_zero_features = features[features["case_num"] == 0]
case_num_zero_features.head(3)<jupyter_output><empty_output><jupyter_text>merge of the notes and features  Create function to repeat this for each note in the dataframe<jupyter_code>def get_doc_ents(clean_text):
    
    sci_doc = sci_nlp(clean_text)
    # print("TEXT", "START", "END", "ENTITY TYPE")
    return {ent.text: [ent.start_char, ent.end_char] for ent in sci_doc.ents}
case_num_feature_text = {}
for i in range(10):
    case_num_feature_text[i] = features["feature_text"][features["case_num"] == i].to_list()
case_num_feature_text
features[["case_num", "feature_num", "feature_text"]]<jupyter_output><empty_output><jupyter_text>creating dictionary of dictionaries- first dictionary key is the case_num- first dictionary value (second dictionary key) is the feature_num- second dictionary value is the feature_text<jupyter_code># looking at feature_num per case_num
for i in features["case_num"].unique():
    print(i, features["feature_num"][features["case_num"] == i].values)
# testing how to get only strings
for a in features["feature_num"][features["case_num"] == 0]:
    print(features["feature_text"][features["feature_num"] == a].to_string()[5:].strip())
d = {}
for i in features["case_num"].unique():
    d[i] = {a: features["feature_text"][features["feature_num"] == a].to_string()[5:].strip() for a in features["feature_num"][features["case_num"] == i]}
d
df = pd.DataFrame(data=[d.keys(), d.values()]).T
df.columns = ["case_num", "features"]
df
notes_features2 = patient_notes.merge(right=df, left_on="case_num", right_on="case_num")
notes_features2
notes_features2.case_num.value_counts()<jupyter_output><empty_output><jupyter_text>redo cleaning and finding entities<jupyter_code>notes_features2["clean_pn_history"] = notes_features2["pn_history"].apply(prepare_jag.basic_clean3).apply(prepare_jag.remove_stopwords, exclude_words=["no"])
notes_features2.head()
notes_features2["note_entities"] = notes_features2["clean_pn_history"].apply(get_doc_ents)
notes_features2.head()
num_records_show = int(input("Please select how many records to show. Must be less than 1000, but the smaller the better due to timing and length: "))

case_num_one = notes_features2["pn_history"][notes_features2["case_num"] == 1][:num_records_show]
whole_case_num_one = "\n".join(case_num_one.to_list())

sci_doc = sci_nlp(whole_case_num_one)
spacy.displacy.render(sci_doc, style='ent', jupyter=True);<jupyter_output><empty_output><jupyter_text>From Scispacy GitHub<jupyter_code># NOTE: The resolve_abbreviations parameter is optional, and requires that
# the AbbreviationDetector pipe has already been added to the pipeline. Adding
# the AbbreviationDetector pipe and setting resolve_abbreviations to True means
# that linking will only be performed on the long form of abbreviations.
sci_nlp1 = spacy.load("en_ner_bc5cdr_md")
sci_nlp1.add_pipe("scispacy_linker", config={"linker_name": "umls"}) # {"resolve_abbreviations": True,}<jupyter_output>/opt/miniconda3/envs/scispacy/lib/python3.9/site-packages/sklearn/base.py:329: UserWarning: Trying to unpickle estimator TfidfTransformer from version 0.20.3 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:
https://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations
  warnings.warn(
/opt/miniconda3/envs/scispacy/lib/python3.9/site-packages/sklearn/base.py:329: UserWarning: Trying to unpickle estimator TfidfVectorizer from version 0.20.3 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:
https://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations
  warnings.warn(<jupyter_text>with case_num == 1<jupyter_code>sci_doc1 = sci_nlp(whole_case_num_one)
# Let's look at a random entity!
# sci_doc1 comes from above
entity1 = sci_doc1.ents[1]
print(f"Name: {entity1}")
# Each entity is linked to UMLS with a score
# (currently just char-3gram matching).
linker = sci_nlp1.get_pipe("scispacy_linker")
for umls_ent in entity1._.kb_ents:
	print(linker.kb.cui_to_entity[umls_ent[0]])<jupyter_output><empty_output><jupyter_text>with case_num == 0<jupyter_code>case_num_zero = notes_features2["pn_history"][notes_features2["case_num"] == 0][:num_records_show]
whole_case_num_zero = "\n".join(case_num_zero.to_list())

sci_doc1 = sci_nlp(whole_case_num_zero)
# Let's look at a random entity!
# sci_doc1 comes from above
entity1 = sci_doc1.ents[1]
print(f"Name: {entity1}")
entities = sci_doc1.ents[0]
linker = sci_nlp1.get_pipe("scispacy_linker")
for umls_ent in entities._.kb_ents:
    print(linker.kb.cui_to_entity[umls_ent[0]])<jupyter_output><empty_output><jupyter_text>comparing locations to features<jupyter_code>notes_features2[["case_num", "features", "note_entities"]]
d_keys = []
for dic in notes_features2["features"]:
    if dic.keys() not in d_keys:
        d_keys.append(dic.keys())
d_keys<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12449350286432863, 'fraction_numerical': 0.015788738298169624, 'mean_word_length': 7.3426573426573425, 'pattern_counts': {'":': 2, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '139657', 'license': 'MIT', 'n_tokens_mistral': 2622, 'n_tokens_neox': 2523, 'n_words': 516, 'path': 'mvp_scipspacy_appendix.ipynb', 'repo_name': 'codeup-nlp-capstone/nlp-capstone', 'size': '7157'}