<jupyter_start><jupyter_text>Imports<jupyter_code>import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
import imblearn
from imblearn.over_sampling import SMOTE<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>first_counter_OHE = pd.read_csv("../data/processed/first_encounter_OHE.csv")
first_counter_OHE.sample(5)
first_counter_OHE.columns.tolist()<jupyter_output><empty_output><jupyter_text>Examine the class label imbalance<jupyter_code>neg, pos = np.bincount(first_counter_OHE['readmitted_rescaled'])
total = neg + pos
print('Examples:\n    Total: {}\n    Positive: {} ({:.2f}% of total)\n'.format(
    total, pos, 100 * pos / total))<jupyter_output>Examples:
    Total: 70434
    Positive: 6293 (8.93% of total)<jupyter_text>Split Data<jupyter_code># Use a utility from sklearn to split and shuffle our dataset.
train_df, test_df = train_test_split(first_counter_OHE, test_size=0.2)
train_df, val_df = train_test_split(train_df, test_size=0.2)
print(test_df.shape)
print(train_df.shape)
print(val_df.shape)<jupyter_output>(14087, 382)
(45077, 382)
(11270, 382)<jupyter_text>Distribution of Data<jupyter_code>neg, pos = np.bincount(train_df['readmitted_rescaled'])
total = neg + pos
print('Train Examples:\n    Total: {}\n    Positive: {} ({:.2f}% of total)\n'.format(
    total, pos, 100 * pos / total))

neg, pos = np.bincount(val_df['readmitted_rescaled'])
total = neg + pos
print('Validation Examples:\n    Total: {}\n    Positive: {} ({:.2f}% of total)\n'.format(
    total, pos, 100 * pos / total))


neg, pos = np.bincount(test_df['readmitted_rescaled'])
total = neg + pos
print('Test Examples:\n    Total: {}\n    Positive: {} ({:.2f}% of total)\n'.format(
    total, pos, 100 * pos / total))
val_df.to_csv("../data/processed/validation.csv", index=False)
test_df.to_csv("../data/processed/test.csv", index=False)<jupyter_output><empty_output><jupyter_text>Upsampling-SMOTE<jupyter_code>y_train = train_df["readmitted_rescaled"]
X_train = train_df.drop("readmitted_rescaled", axis=1)
# Instantiate SMOTE and fit
os = SMOTE(random_state=0)
columns = X_train.columns

data_X_smote, data_y_smote = os.fit_sample(X_train, y_train)
data_X_smote = pd.DataFrame(data=data_X_smote, columns=columns)
data_y_smote = pd.DataFrame(data=data_y_smote, columns=['readmitted_rescaled'])

# Check the numbers of our data
print("Length of oversampled data is ", len(data_X_smote))
print("Number of no readmissions in oversampled data",
      len(data_y_smote[data_y_smote['readmitted_rescaled'] == 0]))
print("Number of readmissions", len(
    data_y_smote[data_y_smote['readmitted_rescaled'] == 1]))
print("Proportion of no readmissions data in oversampled data is ", len(
    data_y_smote[data_y_smote['readmitted_rescaled'] == 0])/len(data_y_smote))
print("Proportion of readmissions data in oversampled data is ", len(
    data_y_smote[data_y_smote['readmitted_rescaled'] == 1])/len(data_y_smote))
SMOTE_df = data_X_smote.join(data_y_smote)
SMOTE_df.head(5)

SMOTE_df.to_csv("../data/processed/SMOTE_df.csv", index=False)<jupyter_output><empty_output><jupyter_text>Downsampling<jupyter_code>pos_samples = train_df[train_df['readmitted_rescaled'] == 1]
neg_samples = train_df[train_df['readmitted_rescaled'] == 0]
neg_samples = neg_samples.sample(n=len(pos_samples))

DS_train_df = pd.concat([pos_samples, neg_samples])
DS_train_df.shape
# Check the numbers of our data
print("Length of downsampled data is ", len(DS_train_df))
print("Number of no readmissions in oversampled data",
      len(DS_train_df[DS_train_df['readmitted_rescaled'] == 0]))
print("Number of readmissions", len(
    DS_train_df[DS_train_df['readmitted_rescaled'] == 1]))
print("Proportion of no readmissions data in oversampled data is ", len(
    DS_train_df[DS_train_df['readmitted_rescaled'] == 0])/len(DS_train_df))
print("Proportion of readmissions data in oversampled data is ", len(
    DS_train_df[DS_train_df['readmitted_rescaled'] == 1])/len(DS_train_df))
DS_train_df.to_csv("../data/processed/DS_train_df.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1363858961802155, 'fraction_numerical': 0.017140058765915768, 'mean_word_length': 6.886100386100386, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '344801', 'license': 'MIT', 'n_tokens_mistral': 1618, 'n_tokens_neox': 1514, 'n_words': 279, 'path': 'notebooks/3.0-RA-SMOTE-Downsampling-Class_weights.ipynb', 'repo_name': 'RajeshArasada/Imbalanced_Data_Classification', 'size': '4084'}