<jupyter_start><jupyter_text>Add GAN data to dataset<jupyter_code>train = pd.read_csv(path + "train_preprocessed.csv")
# test = pd.read_csv(path + "test_train_preprocessed.csv")
base_path = "CheXpert-v1.0-small/train/patient65000/study1/"
label_path = "/om/user/shobhita/src/chexpert/data/"
with open(path + "gan_labels_65000_prop.pkl", "rb") as handle:
    gan_labels = pickle.load(handle)

gan_data = []
    
for img_id, label in gan_labels.items():
    img_filename = "fake_{}.png".format(img_id)
    img_path = base_path + img_filename
    img_row = {}
    img_row["Path"] = img_path
    img_row["Sex"] = "Female"
    img_row["Age"] = 21
    img_row["Frontal/Lateral"] = "Frontal"
    img_row["AP/PA"] = "AP"
    for i, name in enumerate(names):
        img_row[name] = label[i]
    img_row["ID"] = 65000
    gan_data.append(img_row)
gan_df = pd.DataFrame(gan_data)
full_train_df = pd.concat([train, gan_df], ignore_index=True)
full_train_df.to_csv(path + "train_with_gan_preprocessed.csv", index=False)
train_gan = pd.read_csv(path + "train_with_gan_preprocessed.csv")
train_gan.shape
names = ['No Finding', 'Enlarged Cardiomediastinum', 'Cardiomegaly', 'Lung Opacity',
       'Lung Lesion', 'Edema', 'Consolidation', 'Pneumonia', 'Atelectasis',
       'Pneumothorax', 'Pleural Effusion', 'Pleural Other', 'Fracture',
       'Support Devices']
print(label)<jupyter_output>[0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]<jupyter_text>Split dataset into train and test<jupyter_code>train["ID"][0]
train_ids = train["Path"].apply(lambda x: int(x.split("patient")[1].split("/")[0]))
test_ids = test["Path"].apply(lambda x: int(x.split("patient")[1].split("/")[0]))
max(train_ids.max(), test_ids.max())
valid=pd.read_csv(path+'valid.csv')
label_cols=train.columns[5:]
valid[label_cols]=valid[label_cols].fillna(0.0)
valid[label_cols]=valid[label_cols].replace(-1.0,1.0)
valid.to_csv(path+"valid_preprocessed.csv")<jupyter_output><empty_output><jupyter_text>Downsample train dataset  Downsample to 50%<jupyter_code>path = "/om/user/shobhita/src/chexpert/data/CheXpert-v1.0-small/"
train = pd.read_csv(path + "train_preprocessed.csv")
test = pd.read_csv(path + "test_train_preprocessed.csv")
train_gan = pd.read_csv(path + "train_with_gan_preprocessed.csv")
train_subset_patients, _ = train_test_split(train["ID"].unique(), test_size=0.5)
train_subset = train[train["ID"].isin(train_subset_patients)]
len(train_subset)/len(train)
gan_data = train_gan[train_gan["ID"] == 64600]
train_subset_with_gan = pd.concat([train_subset, gan_data], ignore_index=True)
train_subset.to_csv(path + "train_preprocessed_subset.csv", index=False)
train_subset_with_gan.to_csv(path + "train_preprocessed_subset_with_gan.csv", index=False)<jupyter_output><empty_output><jupyter_text>Downsample to 10%<jupyter_code>path = "/om/user/shobhita/src/chexpert/data/CheXpert-v1.0-small/"
train = pd.read_csv(path + "train_preprocessed.csv")
test = pd.read_csv(path + "test_train_preprocessed.csv")
train_gan = pd.read_csv(path + "train_with_gan_preprocessed.csv")

train_subset_patients, _ = train_test_split(train["ID"].unique(), test_size=0.9)
train_subset = train[train["ID"].isin(train_subset_patients)]

print("Ratio: ", len(train_subset)/len(train))
splits = get_class_split(train_subset)
splits
train_subset.to_csv(path + "train_preprocessed_subset_10.csv", index=False)
train = pd.read_csv(path + "train_preprocessed_subset_50.csv")
len(train)
patient_id = 64900
gan_label_path = "/om/user/shobhita/src/chexpert/data/gan_labels_{}_prop.pkl".format(patient_id)
base_path = "CheXpert-v1.0-small/train/patient{}/study1/".format(patient_id)

with open(gan_label_path, "rb") as handle:
    gan_labels = pickle.load(handle)

gan_data = []
    
for img_id, label in gan_labels.items():
    img_filename = "{}.jpg".format(img_id)
    img_path = base_path + img_filename
    img_row = {}
    img_row["Path"] = img_path
    img_row["Sex"] = "Female"
    img_row["Age"] = 21
    img_row["Frontal/Lateral"] = "Frontal"
    img_row["AP/PA"] = "AP"
    for i, name in enumerate(names):
        img_row[name] = label[i]
    img_row["ID"] = patient_id
    gan_data.append(img_row)
len(gan_data)
gan_df = pd.DataFrame(gan_data)
full_train_df = pd.concat([train, gan_df], ignore_index=True)
full_train_df.to_csv(path + "train_preprocessed_subset_50_with_gan.csv", index=False)
train_gan = pd.read_csv(path + "train_preprocessed_subset_50_with_gan.csv")
len(train)
len(train_gan)
train_gan[train_gan["ID"] == 64900]["Path"].iloc[0]<jupyter_output><empty_output><jupyter_text>Downsample to 5%<jupyter_code>path = "/om/user/shobhita/src/chexpert/data/CheXpert-v1.0-small/"
train = pd.read_csv(path + "train_preprocessed.csv")
test = pd.read_csv(path + "test_train_preprocessed.csv")
train_gan = pd.read_csv(path + "train_with_gan_preprocessed.csv")

train_subset_patients, _ = train_test_split(train["ID"].unique(), test_size=0.95)
train_subset = train[train["ID"].isin(train_subset_patients)]

print("Ratio: ", len(train_subset)/len(train))

splits = get_class_split(train_subset, proportion=False)
print(splits)
train_subset.to_csv(path + "train_preprocessed_subset_5.csv", index=False)

train = pd.read_csv(path + "train_preprocessed_subset_5.csv")
len(train)
patient_id = 65100
gan_label_path = "/om/user/shobhita/src/chexpert/data/gan_labels_5.pkl"
base_path = "CheXpert-v1.0-small/train/patient{}/study1/".format(patient_id)

with open(gan_label_path, "rb") as handle:
    gan_labels = pickle.load(handle)

gan_data = []
    
for img_id, label in gan_labels.items():
    img_filename = "{}.jpg".format(img_id)
    img_path = base_path + img_filename
    img_row = {}
    img_row["Path"] = img_path
    img_row["Sex"] = "Female"
    img_row["Age"] = 21
    img_row["Frontal/Lateral"] = "Frontal"
    img_row["AP/PA"] = "AP"
    for i, name in enumerate(names):
        img_row[name] = label[i]
    img_row["ID"] = patient_id
    gan_data.append(img_row)
gan_df = pd.DataFrame(gan_data)
full_train_df = pd.concat([train, gan_df], ignore_index=True)
full_train_df.to_csv(path + "train_preprocessed_subset_5_with_gan.csv", index=False)
len(full_train_df)
len(gan_df)<jupyter_output><empty_output><jupyter_text>Downsample to 1%<jupyter_code>path = "/om/user/shobhita/src/chexpert/data/CheXpert-v1.0-small/"
train = pd.read_csv(path + "train_preprocessed.csv")
train_gan = pd.read_csv(path + "train_with_gan_preprocessed.csv")

train_subset_patients, _ = train_test_split(train["ID"].unique(), test_size=0.99)
train_subset = train[train["ID"].isin(train_subset_patients)]

print("Size: ", len(train_subset))
print("Ratio: ", len(train_subset)/len(train))

splits = get_class_split(train_subset, proportion=False)
print(splits)
train_subset.to_csv(path + "train_preprocessed_subset_1.csv", index=False)
train = pd.read_csv(path + "train_preprocessed_subset_1.csv")
len(train)
patient_id = 65200
gan_label_path = "/om/user/shobhita/src/chexpert/data/gan_labels_1.pkl"
base_path = "CheXpert-v1.0-small/train/patient{}/study1/".format(patient_id)

with open(gan_label_path, "rb") as handle:
    gan_labels = pickle.load(handle)

gan_data = []
    
for img_id, label in gan_labels.items():
    img_filename = "{}.jpg".format(img_id)
    img_path = base_path + img_filename
    img_row = {}
    img_row["Path"] = img_path
    img_row["Sex"] = "Female"
    img_row["Age"] = 21
    img_row["Frontal/Lateral"] = "Frontal"
    img_row["AP/PA"] = "AP"
    for i, name in enumerate(names):
        img_row[name] = label[i]
    img_row["ID"] = patient_id
    gan_data.append(img_row)
gan_df = pd.DataFrame(gan_data)
full_train_df = pd.concat([train, gan_df], ignore_index=True)
full_train_df.to_csv(path + "train_preprocessed_subset_1_with_gan.csv", index=False)
len(full_train_df)
"""
1) For each label vector choose one class, bring them to uniform distribution
2) Pick a category and target number
"""<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15417830835661672, 'fraction_numerical': 0.016383032766065533, 'mean_word_length': 5.969026548672566, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '376883', 'license': 'MIT', 'n_tokens_mistral': 3162, 'n_tokens_neox': 3030, 'n_words': 439, 'path': 'notebooks/preprocessing_chexpert.ipynb', 'repo_name': 'ssundaram21/6.819FinalProjectRAMP', 'size': '7874'}