<jupyter_start><jupyter_text>1. Load the dataset<jupyter_code>dataset = pd.read_csv(HOME_PATH + FILENAME)
dataset['Age'] = np.round(dataset['Age']).astype('int64')
dataset['Height'] = np.round(dataset['Height'],2)
dataset['Weight'] = np.round(dataset['Weight'],2)
dataset<jupyter_output><empty_output><jupyter_text>2. Analyze data<jupyter_code>categorical_cols = (dataset.select_dtypes(include=['object'])).columns.tolist()
categorical_cols
#dimensions of the dataset
dataset.shape
#data types of each attribute
dataset.dtypes
dataset['FCVC'].values
#peak of the data
dataset.head(20)
#summarize the distribution of each attribute
set_option('precision', 2)
dataset.describe()<jupyter_output><empty_output><jupyter_text>3. Data visualization<jupyter_code>for col in dataset.columns :
    # Multiple box plots on one Axes
    data = dataset[col]
    if col in categorical_cols :
        data = data.astype("category").cat.codes
    fig, ax = plt.subplots()
    ax.boxplot(data)
    ax.set_title(col)
for col in dataset.columns :
    # Multiple box plots on one Axes
    data = dataset[col]
    if col in categorical_cols :
        data = data.astype("category").cat.codes
    fig, ax = plt.subplots()
    ax.hist(data, density=False, histtype='bar')
    ax.set_title(col)
#Correlation matrix
set_option('precision', 2)
pyplot.figure(figsize=(20,10))
cors = abs(dataset.corr(method='pearson'))
seaborn.heatmap(cors, mask=np.triu(np.ones_like(cors, dtype=bool)), vmin=0, vmax=1, cmap='Blues', annot=True)
pyplot.show()<jupyter_output><empty_output><jupyter_text>4. Edit data<jupyter_code>for col in dataset.columns :
    if not dataset[col].isnull().values.any() :
        print(col, ':', 'NO NaN values')
    else :
        print(col, ':', 'NaN values finded')
        print('Number of NaN values: ', dataset[col].isnull().sum())
#quick look at the breakdown of class values
for col in categorical_cols :
    dataset[col] = dataset[col].astype('category')
    print('###########################')
    print(dataset.groupby(col).size())<jupyter_output>###########################
Gender
Female    1043
Male      1068
dtype: int64
###########################
family_history_with_overweight
no      385
yes    1726
dtype: int64
###########################
FAVC
no      245
yes    1866
dtype: int64
###########################
CAEC
Always          53
Frequently     242
Sometimes     1765
no              51
dtype: int64
###########################
SMOKE
no     2067
yes      44
dtype: int64
###########################
SCC
no     2015
yes      96
dtype: int64
###########################
CALC
Always           1
Frequently      70
Sometimes     1401
no             639
dtype: int64
###########################
MTRANS
Automobile                457
Bike                        7
Motorbike                  11
Public_Transportation    1580
Walking                    56
dtype: int64
###########################
Obesity_level
Insufficient_Weight    272
Normal_Weight          287
Obesity_Type_I         351
Obesity_Type_II        297
Obesity_Type_III       3[...]<jupyter_text>5. Data split (train and test)<jupyter_code>from sklearn.model_selection import train_test_split
#Split data indixes in train and test
idx_train, idx_test = train_test_split(dataset.index.tolist(), train_size=0.8, random_state=42, shuffle=True)
print('Train data length: ', len(idx_train))
print('Test data length: ', len(idx_test))
print('Total data length: ', len(idx_train) + len(idx_test))
#Select train data and save locally
diabetes_train_data = dataset.loc[idx_train]
diabetes_train_data.to_csv(HOME_PATH + 'TRAIN DATASETS/A_Diabetes_Data_Real_Train.csv', index=False)
#Select test data and save locally
diabetes_test_data = dataset.loc[idx_test]
diabetes_test_data.to_csv(HOME_PATH + 'TEST DATASETS/A_Diabetes_Data_Real_Test.csv', index=False)
print('Train data size: ', diabetes_train_data.shape)
print('Test data length: ', diabetes_test_data.shape)<jupyter_output>Train data size:  (1688, 17)
Test data length:  (423, 17)
{'fraction_non_alphanumeric': 0.171, 'fraction_numerical': 0.0325, 'mean_word_length': 4.450953678474114, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '25724', 'license': 'MIT', 'n_tokens_mistral': 1403, 'n_tokens_neox': 1288, 'n_words': 301, 'path': 'notebooks/Dataset C - Obesity Level Estimation/EDA and Data Split Dataset C - Obesity.ipynb', 'repo_name': 'Vicomtech/STDG-evaluation-metrics', 'size': '4000'}