<jupyter_start><jupyter_text>Load metadata from HD5 ECGs<jupyter_code>fpaths_mrns_old = os.path.expanduser("~/dropbox/ecg/explore/mgh/tensors_all_union.csv")
df_old = pd.read_csv(fpaths_mrns_old)
print(f"Loaded {fpaths_mrns_old} into df")<jupyter_output><empty_output><jupyter_text>Load new VM metadata (MRNs, ECG datetime, location)<jupyter_code># If current CSV does nost exist, create it by merging MRN + locations list
fpath_mrns_new      = os.path.expanduser("~/dropbox/ecg/new-vms/mgh-updated.csv")
fpath_mrns_new_locs = os.path.expanduser("~/dropbox/ecg/new-vms/mgh-updated-locations.csv")
fpath_mrns_updated  = os.path.expanduser("~/dropbox/ecg/new-vms/mgh-updated-merged.csv")

ecg_datetime_key = 'ECG_datetime'
ecg_mrn_key = 'PatientID'

if os.path.exists(fpath_mrns_updated):
    df_new = pd.read_csv(fpath_mrns_updated)

    # Convert column to datetime format
    df_new[ecg_datetime_key] = pd.to_datetime(df_new[ecg_datetime_key])
    
    print(f"Loaded {fpath_mrns_updated} into df with keys {df_new.keys()}")
else:
    df_new = dd.read_csv(fpath_mrns_new)
    print(f"Read {fpath_mrns_new} to dask dataframe with {len(df_new.index)} rows")
        
    df_new_locs = dd.read_csv(fpath_mrns_new_locs)
    print(f"Read {fpath_mrns_new_locs} to dask dataframe with {len(df_new_locs.index)} rows")
    
    # Merge the csv files.
    df_merged = dd.merge(df_new, df_new_locs, how='outer', on=['PatientID', 'ECG_datetime'])
    print(f'Merged two Dask dfs into one df with length {len(df_merged.index)}')
    
    # Convert Dask DF to Pandas DF, and overwrite df_new
    df_new = df_merged.compute()
    
    # Convert column to datetime format
    df_new[ecg_datetime_key] = pd.to_datetime(df_new[ecg_datetime_key])
    
    # Write the output
    df_new.to_csv(fpath_mrns_updated, index=False)
    print(f"Saved merged df to {fpath_mrns_updated}")
# Preview merged new DF
df_new.head()<jupyter_output><empty_output><jupyter_text>Load CSV of reference cohort<jupyter_code># STS
# fpath_ref = os.path.expanduser("~/dropbox/sts-data/mgh-all-features-labels.csv")
# cohort_name = "sts" 
# date_key = "surgdt"
# ref_key = "medrecn" 
# outcome_key = 'mtopd'

# Apollo
fpath_ref = os.path.expanduser("~/dropbox/apollo/ecg_pressures_labs_processed.csv")
cohort_name = "apollo"
date_key = "Date_of_Cath"
ref_key = "Patient_ID" 

df_ref = pd.read_csv(fpath_ref)
print(f"Loaded {fpath_ref} into df")
df_ref[date_key] = pd.to_datetime(df_ref[date_key])
df_ref
# Get all MRNs from reference CSV
mrn_ref = pd.to_numeric(df_ref[ref_key], errors="coerce")
mrn_ref_unique = np.unique(mrn_ref)
df_ref[ref_key] = mrn_ref
print(f'Reference: {len(mrn_ref)} total MRNs and {len(mrn_ref_unique)} unique MRNs')
# Get all MRNs in tensors_all
mrn_old = pd.to_numeric(df_old['ecg_patientid_clean'], errors="coerce")
mrn_old_unique = np.unique(mrn_old)
df_old['ecg_patientid_clean'] = mrn_old
print(f'Existing HD5 dataset: {len(mrn_old)} total MRNs and {len(mrn_old_unique)} unique MRNs')

mrn_intersect_old_ref = np.intersect1d(mrn_ref_unique, mrn_old_unique)
print(f'Intersect between existing HD5 dataset and reference cohort: {len(mrn_intersect_old_ref)} unique MRNs found in both')
# Get all MRNs from new VM CSV
mrn_new = pd.to_numeric(df_new['PatientID'], errors="coerce")
mrn_new_unique = np.unique(mrn_new)
df_new['PatientID'] = mrn_new
print(f'CSV from new VM: {len(mrn_new)} total MRNs and {len(mrn_new_unique)} unique MRNs')
# Get intersect between reference cohort and MRN
mrn_intersect_new_ref = np.intersect1d(mrn_ref_unique, mrn_new_unique)
print(f'Intersect between new VM CSVs and reference cohort: {len(mrn_intersect_new_ref)} unique MRNs found in both')
new_mrns = set(mrn_intersect_new_ref) - set(mrn_intersect_old_ref)
new_mrns = list(new_mrns)
print(f"Found {len(new_mrns)} new reference cohort MRNs in new VM CSV")
fpaths_mrns_prioritize = os.path.expanduser(f"~/dropbox/ecg/new-vms/mrns-to-prioritize-mgh-{cohort_name}.csv")
mrns_formatted = [f"{int(mrn):09}" for mrn in new_mrns]
pd.DataFrame(mrns_formatted).to_csv(fpaths_mrns_prioritize, index=False, header=False)
print(f"Saved new reference cohort MRNs to {fpaths_mrns_prioritize}")
# Print a few formatted MRNs to ensure zero padding
mrns_formatted[0:10]
# ecg_hits = {}

# for mrn in tqdm(new_mrns):
    
#     ecg_hits[mrn] = {}
    
#     # Isolate DF rows for this MRN from reference cohort
#     df_ref_mrn = df_ref[df_ref[ref_key] == mrn]
    
#     # Save label for this MRN
#     ecg_hits[mrn][outcome_key] = int(df_ref_mrn[outcome_key])
    
#     # Isolate DF rows for this MRN from CSV from new VM
#     df_new_ecgs = df_new[df_new[ecg_mrn_key] == mrn]
    
#     # Get date of surgery for latest row of reference data for this patient
#     start_date = df_ref_mrn[date_key].iloc[-1] - pd.Timedelta(value=30, unit='days')
#     end_date = df_ref_mrn[date_key].iloc[-1]
    
#     # Get boolean mask of all ECGs that are within window
#     mask = (df_new_ecgs[ecg_datetime_key] > start_date) & (df_new_ecgs[ecg_datetime_key] < end_date)

#     # If any hits, get the first date of the hit
#     if mask.any():
#         ecg_hits[mrn]['hit_dates'] = df_new_ecgs[ecg_datetime_key][mask].to_list()
#     else:
#          ecg_hits[mrn]['hit_dates'] = []

#     # Add info to dicts
#     ecg_hits[mrn]['start_date'] = start_date
#     ecg_hits[mrn]['end_date'] = end_date
#     ecg_hits[mrn]['hit_count'] = sum(mask)

# df_ecg_hits = pd.DataFrame(ecg_hits).T
# df_ecg_hits
# print(f"Reference MRNs (total new from VM): {df_ecg_hits.shape[0]}")
# print(f"Reference MRNs (1+ ECG in window): {sum(df_ecg_hits['hit_count'] > 0)}")
# print(f"Total ECGs in window: {df_ecg_hits['hit_count'].sum()}")
# print(f"Positive labels: {df_ecg_hits[outcome_key].sum()}")
# print(f"Positive labels (with ECGs in window): {df_ecg_hits[df_ecg_hits['hit_count'] > 0]['mtopd'].sum()}")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12642698926563298, 'fraction_numerical': 0.003066962003748509, 'mean_word_length': 5.311827956989247, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '488428', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2432, 'n_tokens_neox': 2358, 'n_words': 452, 'path': 'notebooks/ecg/assess-ecg-coverage.ipynb', 'repo_name': 'mit-ccrg/ml4c3-mirror', 'size': '5869'}