<jupyter_start><jupyter_text>For our target distribution, hdl seems balanced between the classes, hgb is inbalanced in two classes(ok, high), ldl is inbalanced on one class(high)Cholesterol_hdl levels in users is balanced between low, ok, highhemoglobin levels in users is grossly inbalanced. over 80% of users have low hamoglobin levelspatients do not have high cholesterol_ldl levelshdl_cholesterol_human - hdlhemoglobin(hgb)_human - hgbcholesterol_ldl_human - ldl<jupyter_code>no_patients = len( train["donation_id"].value_counts() )
no_patients
# 486 patients
train.columns<jupyter_output><empty_output><jupyter_text>- variation between each individual persons scans(temperature, humidity)<jupyter_code>train.donation_id.values
patients = train.groupby(["donation_id"])
id_2250 = patients.get_group(2250)

id_2250.head(5)
id_2250.describe()
# id_2250["hdl"].value_counts().values

print( id_2250.describe().loc["std", "temperature"] )
print( id_2250.describe().loc["std"].sort_values() )
donation_idx = train["donation_id"].unique()
# hdl_cholesterol_human - hdl
# hemoglobin(hgb)_human - hgb
# cholesterol_ldl_human - ldl

# Get patients individual dataframes to inspect for inconsistencies in data

patients_df = {}

avg_temp_per_patient = []
avg_humid_per_patient = []

for i in donation_idx:

    df = patients.get_group(i)
    patients_df[i] = df

    
    hgb_count = df["hgb"].value_counts()
    hdl_count = df["hdl"].value_counts()
    ldl_count = df["ldl"].value_counts()

    if len(hgb_count) > 1:
        print(f"ID:{i} ---- UNSTABLE hgb(hemoglobin_human) count")
    
    if len(hdl_count) > 1:
        print(f"ID:{i} ---- UNSTABLE hdl(hdl_cholesterol_human) count")
    
    if len(ldl_count) > 1:
        print(f"ID:{i} --- UNSTABLE ldl(cholesterol_ldl_human) count")
    
    temp_std = df.describe().loc["std", "temperature"]
    humidity_std = df.describe().loc["std", "humidity"]

    if temp_std > 1.5 and temp_std < 2.0:
        print(f"ID:{i} ---- UNSTABLE TEMPERATURE (between 1.5 and 2.0)")
    
    if temp_std >= 2.0:
        print(f"ID:{i} ---- UNSTABLE TEMPERATURE (above 2.0)")
    
    temp = df.describe().loc["mean", "temperature"] #get the average temperature of each patient
    humid = df.describe().loc["mean", "humidity"] #get the average humidity of each patient

    avg_temp_per_patient.append(temp)
    avg_humid_per_patient.append(humid)

# approx 13m
plt.subplots(figsize=(10, 6))
sns.set_style("whitegrid")
sns.lineplot(x=donation_idx, y=avg_temp_per_patient)<jupyter_output><empty_output><jupyter_text>Let's take first 28 scans per id to and see how the aveerage temperature differs per id<jupyter_code>avg_temp_per_patient_28 = []

for i in donation_idx:

    df = patients_df[i][0:28]

    temp = df["temperature"]
    avg_temp_per_patient_28.append(np.mean(temp))


# plt.subplots(figsize=(10, 6))
# sns.set_style("whitegrid")
# sns.lineplot(x=donation_idx, y=avg_temp_per_patient_28)
donation_idx[:10]
print( np.mean(patients_df[6824][0:28]["temperature"]) )
print( np.mean(patients_df[6824]["temperature"] ))
plt.subplots(figsize=(10, 6))
sns.set_style("whitegrid")
sns.lineplot(x=donation_idx, y=avg_temp_per_patient_28)
plt.subplots(figsize=(10, 6))
sns.set_style("whitegrid")
sns.lineplot(x=donation_idx, y=avg_humid_per_patient)<jupyter_output><empty_output><jupyter_text>Temperature and Humidity are inversely proportionalan increase in temperature would lead to an increase in humidityTemperature and Humidity are Noisy asf<jupyter_code>plt.subplots(figsize=(11, 7))
sns.set_style("whitegrid")

sns.lineplot(x=pd.Series(donation_idx), y=pd.Series(avg_temp_per_patient) , color="red")
sns.lineplot(x=pd.Series(donation_idx), y=pd.Series(avg_humid_per_patient), color="blue")<jupyter_output><empty_output><jupyter_text>clean users with high avg temperaturesdrop users with high tempslook at spectrum of high temps<jupyter_code># Renaming columns: (X) Wavelength to absorbance

hgb_spectra = pd.read_csv("hemoglobin_Spectra.csv", header=1).rename(columns={"(X) Wavelength":"X" , "(Y)Absorbance Coefficient  (cm-1)":"absorbance"})
ldl_spectra = pd.read_csv("Cholesterol_Spectra.csv", header=1).rename(columns={"(X) Wavelength":"X" , "(Y)Absorbance Coefficient  (cm-1)":"absorbance"})
hdl_spectra = pd.read_csv("Glucose.csv", header=1).rename(columns={"(X) Wavelength":"X" , "(Y)Absorbance Coefficient  (cm-1)":"absorbance"})
hgb_spectra.head(2)
ldl_spectra.head(2)
hdl_spectra.head(2)
## plotting spectrum data_with absorbance levels


def ab_spec(id_):
    plt.subplots(figsize=(11, 7))
    sns.set_style("whitegrid")
    id_df = patients_df[id_]

    id_df = id_df.drop(["temperature", "humidity", "hdl", "hgb", "ldl", "donation_id", "id", "std"], axis=1)

    for i in range(60):
        id_scan = id_df.iloc[i]
        id_df_scan = pd.DataFrame({"X":hgb_spectra["X"] , "absorbance":id_scan[11:-11].values })

        id_df_scan["absorbance"] = id_df_scan["absorbance"].astype("float32")

        sns.lineplot(x=pd.Series( id_df_scan["X"] ) , y=pd.Series( id_df_scan["absorbance"] ))

    
    # sns.lineplot(x=pd.Series(hgb_spectra["X"]), y=pd.Series(hgb_spectra["absorbance"]), color='red')
    # sns.lineplot(x=pd.Series(ldl_spectra["X"]), y=pd.Series(ldl_spectra["absorbance"]), color='g')
    # sns.lineplot(x=pd.Series(hdl_spectra["X"]), y=pd.Series(hdl_spectra["absorbance"]), color='y')
    
    # del id_df

ab_spec(7152)<jupyter_output><empty_output><jupyter_text>Let's pick patients1. with low low low2. with low low ok3. with low low high4. with low ok low5. with low ok ok6. with low ok highwith<jupyter_code>train.head(3)
# IDs with "low" as hdl

train[train["hdl"] == "low"]["donation_id"].unique()
len(train[train["hdl"] == "low"]["donation_id"].unique())
def con_tar_onehot(targets):

    targets = pd.get_dummies(targets)

    return targets

train_targets = con_tar_onehot(train[['hdl', 'hgb', 'ldl']])

train_targets.head(3)
def undummify(df, prefix_sep="_"):

    cols2collapse = { item.split(prefix_sep)[0]: (prefix_sep in item) for item in df.columns }

    series_list = []

    for col, needs_to_collapse in cols2collapse.items():
        if needs_to_collapse:
            undummified = (df.filter(like=col).idxmax(axis=1).apply(lambda x: x.split(prefix_sep, maxsplit=1)[1]).rename(col) )
            series_list.append(undummified)
        else:
            series_list.append(df[col])
        undummified_df = pd.concat(series_list, axis=1)
    
    return undummified_df

train_targets = undummify(train_targets)<jupyter_output><empty_output><jupyter_text>Absorbance Spectrum for patients with low hdl levels.<jupyter_code># for i in train[train["hdl"] == "low"]["donation_id"].unique():
#     ab_spec(i)
# for i in [2250, 8790, 2960, 3381, 7668, 2550, 6362, 3593, 10790]:

#     ab_spec(i)<jupyter_output><empty_output><jupyter_text>These values with large variations in absorbance, let's check their temperature.<jupyter_code>train[train["hdl"] == "high"]["donation_id"].unique()<jupyter_output><empty_output><jupyter_text>Let's see the difference in absorbance spectrum when hdl is low vs when hdl is high  Absorbance Spectrum of patients with high hdl levels<jupyter_code># for i in train[train["hdl"] == "high"]["donation_id"].unique():
#     ab_spec(i)<jupyter_output><empty_output><jupyter_text>Perform PCA on absorbance features<jupyter_code>test.columns
test_reading_id = test.Reading_ID
test_donation_id = test.donation_id
test_id = test.id
test_std = test["std"]
test_temperature = test.temperature
test_humidity = test.humidity

test_absorbance = test.drop( ["Reading_ID", "donation_id", "id", "std", "temperature", "humidity"], axis=1 )

test_absorbance.head(3)
[i for i in test.columns if test[i].dtype == "object"]
train.columns
train_absorbance = train.iloc[:, 0:170]

train_absorbance.head(4)
train.head(1)
train.columns
from sklearn.decomposition import PCA

# hdl, hgb, ldl
pca = PCA(n_components=28)

pca_model = pca.fit(train_absorbance)

train_absorbance_pca = pca_model.transform(train_absorbance)
test_absorbance_pca = pca_model.transform(test_absorbance)
train_absorbance_pca = pd.DataFrame( train_absorbance_pca)
test_absorbance_pca = pd.DataFrame( test_absorbance_pca )

# train

train_absorbance_pca["donation"] = train["donation_id"]
train_absorbance_pca["hdl"] = train["hdl"]
train_absorbance_pca["hgb"] = train["hgb"]
train_absorbance_pca["ldl"] = train["ldl"]
train_absorbance_pca["humidity"] = train["humidity"]
train_absorbance_pca["temperature"] = train["temperature"]
train_absorbance_pca["std"] = train["std"]
train_absorbance_pca["id"] = train["id"]

# Test
# ["Reading_ID", "donation_id", "id", "std", "temperature", "humidity"]
test_absorbance_pca["donation_id"] = test_donation_id
test_absorbance_pca["Reading_ID"] = test_reading_id
test_absorbance_pca["id"] = test_id
# test_absorbance_pca["std"] = test_std
test_absorbance_pca["temperature"] = test_temperature
test_absorbance_pca["humidity"] = test_humidity

train_absorbance_pca.head(3)
test_absorbance_pca.head(3)
assert train_absorbance_pca.shape[0] == train.shape[0], "train_absorbance_pca.shape != train.shape"
pca_patients = train_absorbance_pca.groupby(["donation"])
pca_test_patients = test_absorbance_pca.groupby(["donation_id"])
pca_patients_df = {}
pca_patients_test_df = {}


for i in donation_idx:
    # this is for train_data
    df = pca_patients.get_group(i)
    pca_patients_df[i] = df

for i in test_donation_id.values:
    # this is for test_data
    df = pca_test_patients.get_group(i)
    pca_patients_test_df[i] = df
donation_idx[0:10]
plt.imshow( pca_patients_df[3068].iloc[0:60, 0:28] )
plt.imshow( pca_patients_df[2250].iloc[0:60, 0:28] )
plt.imshow( pca_patients_df[6370].iloc[0:60, 0:28] )
# hdl, hgb, ldl
print( pca_patients_df[3068][["hdl", "hgb", "ldl"]].value_counts() )
print( pca_patients_df[2250][["hdl", "hgb", "ldl"]].value_counts() )
print( pca_patients_df[6370][["hdl", "hgb", "ldl"]].value_counts() )<jupyter_output>hdl  hgb  ldl
ok   ok   low    60
dtype: int64
hdl  hgb  ldl 
low  ok   high    60
dtype: int64
hdl  hgb  ldl 
ok   ok   high    60
dtype: int64<jupyter_text>Let's prepare data for training<jupyter_code># def trim_scans(data, n_of_scans):
    
#     new_data = data
#     count = 0
#     for i in donation_idx:
#         print(count)
#         df = patients_df[i].iloc[n_of_scans:]
#         df_ids = df["id"].values
        
#         new_data = new_data.drop(df_ids, axis=0)
#         count = count + 1
    
#     return new_data

# cc = trim_scans(train_absorbance_pca, 28)
test_absorbance_pca
train_absorbance_pca<jupyter_output><empty_output><jupyter_text>- when we assign folds to every id- we create a list to hold every id in the training folds- loop through the list, get the data for every id- convert to array and append to image_list<jupyter_code>target = train_absorbance_pca[["hdl", "hgb", "ldl"]]
train_absorbance_pca["fold"] = -1
from sklearn.model_selection import GroupKFold

group_k_fold = GroupKFold(n_splits=5)
# group_k_fold.get_n_splits(train, target, groups)

groups = train.donation_id

for fold, (train_idx, val_idx) in enumerate(group_k_fold.split(train_absorbance_pca, target, groups=groups)):
    print(f"FOLD:{fold} ------- TRAIN_IDX:{train_idx} ---------- VAL_IDX:{val_idx} ")
    print("")
    print("")
    train_absorbance_pca.loc[val_idx, "fold"] = fold
    target.loc[val_idx, "fold"] = fold
target_29160_rows = con_tar_onehot( train_absorbance_pca[["hdl", "hgb", "ldl"]] )

target_29160_rows["donation_id"] = train.donation_id
def get_training_data_image(data):
    '''
    This function would take care of returning a dataframe without the [donation, hdl, hgb etc] columns
    the dataframe returned would have the shape of [batch_size, height, width]
    '''
    images = []
    target = []
    
    # get list of donatiion ids
    data_ids = list( data["donation"].unique() )

    for i in data_ids:
        df = pca_patients_df[i]

        df = df.drop(["donation", "hdl", "hgb", "ldl", "humidity", "temperature", "std", "id"], axis=1)
        df = np.array(df)
        images.append(df)

        t = target_29160_rows[target_29160_rows["donation_id"] == i].iloc[0].drop("donation_id")

        target.append(t)
    
    images = np.array(images)
    images = np.reshape(images, [images.shape[0] , images.shape[1], images.shape[2], 1 ])
    return ( images, np.array(target) )


def get_test_data_image(data):
    '''
    This function transforms the test data into images and gets rid of unnecessary columns.
    '''
    images = []
    
    # get list of donatiion ids
    data_ids = list( data["donation_id"].unique() )
    print(len(data_ids))

    for i in data_ids:
        df = pca_patients_test_df[i]

        df = df.drop(["donation_id", "Reading_ID", "id", "temperature", "humidity"], axis=1)
        df = np.array(df)
        images.append(df)
    
    images = np.array(images)
    images = np.reshape(images, [images.shape[0] , images.shape[1], images.shape[2], 1 ])
    return images



class MultioutputAccuracy(keras.metrics.Metric):
    def __init__(self, **kwargs):
        super(MultioutputAccuracy, self).__init__(**kwargs)
        self.accuracy = tf.Variable(0, dtype=tf.float64)

    def update_state(self, y_true, y_pred, sample_weight=None):
        new_y_pred = tf.round(y_pred)

        acm = new_y_pred == tf.cast(y_true, tf.float32)
        print(acm)
        acm = tf.cast(acm, tf.int64)


        # print(f"y_true.shape: {y_true.shape}")
        # print(f"y_pred.shape: {y_pred.shape}")

        acm = tf.reduce_sum(acm, axis=1) / 3

        acm = tf.reduce_mean(acm)
        print(acm)
        self.accuracy.assign_add(acm)
    
    def result(self):
        return self.accuracy
model = keras.models.Sequential([

    keras.layers.Input(shape=[60, 28, 1]),
    keras.layers.Conv2D(64, 3, padding="valid", activation="relu"),
    keras.layers.Conv2D(124, 3, padding="valid", activation="relu"),
    keras.layers.Flatten(),
    keras.layers.Dense(100, activation="relu"),
    keras.layers.Dropout(0.4),
    keras.layers.Dense(64, activation="relu"),
    keras.layers.Dropout(0.5),
    keras.layers.Dense(9, activation="sigmoid"),

])

optimizer = keras.optimizers.Adam(learning_rate=0.01)
model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics="accuracy")
# Validating get_training_data_image function.    
    
vv, bx = get_training_data_image( train_absorbance_pca[train_absorbance_pca["fold"] == 1 ])

print(vv.shape)
print(bx.shape)

# train_absorbance_pca.columns<jupyter_output>(97, 60, 28, 1)
(97, 9)<jupyter_text>Continue fixing the training algorithm<jupyter_code>val_accuracy_score = MultioutputAccuracy()
train_accuracy_score = MultioutputAccuracy()


val_accuracy = []
train_accuracy = []

for fold in range(5):
    print(f"VALIDATING ON FOLD:{fold} ----- TRAINING ON OTHER FOLDS")
    print(" ")

    train_X = train_absorbance_pca[train_absorbance_pca["fold"] != fold]

    (train_X, train_Y) = get_training_data_image(train_X)
    train_X_ , train_Y_ = train_X, train_Y


    val_X = train_absorbance_pca[ train_absorbance_pca["fold"] == fold ]
    
    (val_X_, val_Y_) = get_training_data_image(val_X)
    val_X, val_Y = val_X_, val_Y_

    # ----

    model.fit(train_X_, train_Y_, validation_data=(val_X_, val_Y_) , epochs=10, shuffle=True)
    

    pred_y = model.predict(val_X)
    pred_y_t = model.predict(train_X) 

    acc = val_accuracy_score(val_Y, pred_y)
    acc_t = train_accuracy_score(train_Y, pred_y_t)
    
    val_accuracy.append(acc)
    train_accuracy.append(acc_t)

    val_accuracy_score.reset_states()
    train_accuracy_score.reset_states()

    print("")
    print("")
    print("")<jupyter_output>VALIDATING ON FOLD:0 ----- TRAINING ON OTHER FOLDS
 
Epoch 1/10
13/13 [==============================] - 5s 358ms/step - loss: 6.7463 - accuracy: 0.0954 - val_loss: 5.9860 - val_accuracy: 0.0000e+00
Epoch 2/10
13/13 [==============================] - 4s 331ms/step - loss: 5.9028 - accuracy: 0.0773 - val_loss: 5.8887 - val_accuracy: 0.0000e+00
Epoch 3/10
13/13 [==============================] - 4s 326ms/step - loss: 5.8264 - accuracy: 0.0180 - val_loss: 5.9191 - val_accuracy: 0.0000e+00
Epoch 4/10
13/13 [==============================] - 4s 323ms/step - loss: 5.8746 - accuracy: 0.0077 - val_loss: 5.8968 - val_accuracy: 0.0000e+00
Epoch 5/10
13/13 [==============================] - 5s 357ms/step - loss: 5.7675 - accuracy: 0.0026 - val_loss: 5.8855 - val_accuracy: 0.0000e+00
Epoch 6/10
13/13 [==============================] - 4s 344ms/step - loss: 5.7732 - accuracy: 0.0000e+00 - val_loss: 5.8922 - val_accuracy: 0.0000e+00
Epoch 7/10
13/13 [==============================] - 4s 333ms/step -[...]<jupyter_text>with categorical cross_entropy we have avg accuracy of 2.25with binary cross_entropy we have avg accuracy of<jupyter_code>print(np.mean(train_accuracy) )

print( np.mean(val_accuracy) )<jupyter_output>2.212452848523396
2.212013465179886<jupyter_text>Let's run on test data<jupyter_code>test_absorbance_pca.head(3)
test_absorbance_pca.columns
idx = test_absorbance_pca.Reading_ID.values
id = test_absorbance_pca.donation_id.values
test_absorbance_pca.shape
test_images = get_test_data_image(test_absorbance_pca)
test_predictions = model.predict(test_images)
ff = (model.predict(test_images) > 0.5).astype("int32")
ff
test_predictions.shape
# hdl_high	hdl_low	hdl_ok	hgb_high	hgb_low	hgb_ok	ldl_high	ldl_low	ldl_ok
 
test_predictions = pd.DataFrame(ff, columns=["hdl_high", "hdl_low", "hdl_ok", "hgb_high", "hgb_low", "hgb_ok", "ldl_high", "ldl_low", "ldl_ok"])

test_predictions
test_predictions = undummify(test_predictions)

test_predictions
def modify_test_predictions(test_prediction):
    new_df = pd.DataFrame(columns=["hdl", "hgb", "ldl"])
    for i in range(61):

        for j in range(60):
            new_df = new_df.append( pd.DataFrame(test_prediction.iloc[i] ).T )
    return new_df


test_predictions = modify_test_predictions(test_predictions)
test_predictions
test_predictions["Reading_ID"] = id

test_predictions = test_predictions.rename(columns={"hdl":"hdl_cholesterol_human" , "hgb":"hemoglobin(hgb)_human", "ldl":"cholesterol_ldl_human"})

test_predictions = test_predictions.reset_index(drop=True)

test_predictions
def transform_c_hdl(row):
    return str(row["Reading_ID"]) + "_hdl_cholesterol_human" + "-" +  row["hdl_cholesterol_human"]

hdl_rows = pd.DataFrame(test_predictions.apply(transform_c_hdl, axis=1))

def transform_hemo(row):
    return str(row["Reading_ID"]) + "_hemoglobin(hgb)_human" +  "-" + row["hemoglobin(hgb)_human"]

hemo_rows = pd.DataFrame(test_predictions.apply(transform_hemo, axis=1))

def transform_c_ldl(row):
    return str(row["Reading_ID"]) + "_cholesterol_ldl_human" +  "-" + row["cholesterol_ldl_human"]

ldl_rows = pd.DataFrame(test_predictions.apply(transform_c_ldl, axis=1))

test_predictions = pd.concat([hdl_rows, hemo_rows, ldl_rows]).reset_index(drop=True)
test_predictions["target"] = test_predictions[0].apply(lambda x: x.split("-")[1])
test_predictions[0] = test_predictions[0].apply(lambda x: x.split("-")[0])
test_predictions = test_predictions.rename(columns={0:"Reading_ID"})
test_predictions.to_csv("submissions//01.csv", index=False)
test_predictions[test_predictions["Reading_ID"] == "ID_1961_cholesterol_ldl_human"]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14998964159933706, 'fraction_numerical': 0.03102340998549824, 'mean_word_length': 5.073922617175213, 'pattern_counts': {'":': 11, '<': 58, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '389664', 'license': 'MIT', 'n_tokens_mistral': 7845, 'n_tokens_neox': 7229, 'n_words': 1306, 'path': 'PB_0.7027___VAL_ACC_2.212___TRAIN_2.2124.ipynb', 'repo_name': 'blackhat-coder/Blood-Spectroscopy', 'size': '19308'}