<jupyter_start><jupyter_text>End of main code...------------------------<jupyter_code># Special hacked code for the weird ADT table file format
# DEPRECATED
dfs = []
dfd=pd.read_excel(basepath+"ADT/ADTEventHierarchy AND LocationHierarchy for Each site i2b2 June 21 2018_update.xlsx",
                  sheet_name=None)
for k,v in dfd.items():
    shortf=k[0:k.find(' ',k.find(' ')+1)].replace(' ','_')
    print(shortf)
    df=v.dropna(axis='columns',how='all')
    df = df.drop(['C_TOOLTIP','c_tooltip'],axis=1,errors='ignore')
    print(df.columns)
    df = OntProcess('CONCERN\\'+shortf,df)
    ndf = OntBuild(df)
    dfs.append(ndf)
    ndf.to_csv(outpath+shortf+"_autoprocessed.csv")
#tname = 'out_'+shortf
#globals()[tname]=ndf
#%sql DROP TABLE $tname
#%sql PERSIST $tname
# One-off a prespecified file
k = "VISIT i2b2 HIERARCHY.xlsx"
dfs = ontExcelConverter("/Users/jeffklann/Downloads/"+k)
outdf = pd.concat(dfs)
outdf.to_csv("/Users/jeffklann/Downloads/VISIT_HIERARCHY"+"_autoprocessed.csv")
# Example of persisting table with SQL Magic
testdict={"animal":["dog",'cat'],'size':[30,15]}
zoop = pd.DataFrame(testdict)
tname = 'zoop'
%sql DROP TABLE $tname
%sql PERSIST $tname
%sql SELECT * from autoprocessed_i2b2ontology
#engine.execute("SELECT * FROM autoprocessed_i2b2ontology").fetchall()
#Workspace, working on folder check code
mydf = dfs[0]

mymerge = mydf.merge(mydf,left_on=mydf.columns[3],right_on=mydf.columns[4],how='inner',suffixes=['','_r']).groupby('c_fullname').size().reset_index().rename(lambda x: 'size' if x==0 else x,axis='columns')
mymerge=mydf.merge(mymerge,
                 left_on='c_fullname',right_on='c_fullname',how='left')
print(mymerge['size'] > 0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18054746651135703, 'fraction_numerical': 0.015142690739662202, 'mean_word_length': 5.3161764705882355, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '499727', 'license': 'Apache-2.0', 'n_tokens_mistral': 690, 'n_tokens_neox': 667, 'n_words': 114, 'path': 'ontology_tools/concern_build_ont.ipynb', 'repo_name': 'jklann/jgk-i2b2tools', 'size': '1717'}