<jupyter_start><jupyter_text>Exploratory Data Analysis: Celeba Dataset---  Import libraries<jupyter_code>import sys
sys.path.append("..")
import subprocess
from pathlib import Path

from torch.utils.data import Dataset, DataLoader
from skimage.io import imread
import pandas as pd
import matplotlib.pyplot as plt

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Download data<jupyter_code>from disenn.datasets.celeba_dataset import CelebA
CelebA._download("../celeba")<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>def show_image(img_id: str, title: str = None, data_path: Path = Path("data/celeba")):
    img_path = data_path / "img_align_celeba" / "img_align_celeba" / img_id
    img = imread(img_path)
    fig, ax = plt.subplots()
    ax.imshow(img)
    ax.set_xticks([])
    ax.set_yticks([]) 
    ax.set_title(title)
show_image("000001.jpg")
df_attr = pd.read_csv(data_path / "list_attr_celeba.csv", index_col='image_id')
df_attr.replace(to_replace=-1, value=0, inplace=True)
df_attr.shape
df_attr.info()
img_id = "000600.jpg"
y1 = df_attr.loc[img_id, 'Male']
y2 = df_attr.loc[img_id, 'Young']
title = f"Young:{y2} Male:{y1}"
show_image(img_id, title)<jupyter_output><empty_output><jupyter_text>Data Distribution  Male Faces<jupyter_code>df_attr.hist(column='Male', bins=2)
num_female = df_attr['Male'].value_counts()[0]
num_male = df_attr['Male'].value_counts()[1]
print(f"Male: {num_male / (num_male+num_female) * 100:.3f}%")<jupyter_output>Male: 41.675%<jupyter_text>Possible Targets: Young, Attractive<jupyter_code>df_attr.hist(column=['Young', 'Attractive'], bins=2, density=True)<jupyter_output><empty_output><jupyter_text>Attribute Correlation<jupyter_code>attr_correlations = df_attr.corr(method='pearson')
fig = plt.figure(figsize=(25, 20))
plt.matshow(attr_correlations, fignum=fig.number)
plt.xticks(range(df_attr.shape[1]), df_attr.columns, fontsize=14, rotation=90)
plt.yticks(range(df_attr.shape[1]), df_attr.columns, fontsize=14)
cb = plt.colorbar()
cb.ax.tick_params(labelsize=14)<jupyter_output><empty_output><jupyter_text>What facial attributes correlate with Attractiveness?<jupyter_code>attr_correlations['Attractive'].plot.bar(figsize=(15,5))<jupyter_output><empty_output><jupyter_text>What facial attributes correlate with Gender?<jupyter_code>attr_correlations['Male'].plot.bar(figsize=(15,5))<jupyter_output><empty_output><jupyter_text>What facial attributes correlate with Age?<jupyter_code>attr_correlations['Young'].plot.bar(figsize=(15,5))<jupyter_output><empty_output><jupyter_text>Train Test Split<jupyter_code>df_partition = pd.read_csv(data_path / 'list_eval_partition.csv', index_col='image_id')
df_partition['partition'].value_counts()
target = 'Male'
final_df = df_partition.join(df_attr[target], how='inner')
final_df.head()
train_df = final_df[final_df['partition'] == 0]
val_df = final_df[final_df['partition'] == 1]
test_df = final_df[final_df['partition'] == 2]
train_df.shape[0]
train_df.iloc[2].name
train_df.hist(column=target, bins=2, density=True)
val_df.hist(column=target, bins=2, density=True)<jupyter_output><empty_output><jupyter_text>**Target Distribution**: The target distribution across train-validation splits are the same. We will not balance this out, assuming this as the true data distribution.  Toy Dataset for Experiments  CelebA Dataset API Test<jupyter_code>import sys
sys.path.append('..')

%load_ext autoreload
%autoreload 2
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt
from disenn.datasets.celeba_dataset import CelebA
dataset = CelebA(split='train', data_path='data/celeba')
img, label = dataset[10]
img.max(), img.min()
plt.imshow(img.numpy().transpose(1,2,0))
dl = DataLoader(dataset, batch_size=10, shuffle=False)
x,y = next(iter(dl))
x.shape
plt.imshow(x[0].numpy().transpose(1,2,0))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14285714285714285, 'fraction_numerical': 0.01896103896103896, 'mean_word_length': 8.675879396984925, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '169216', 'license': 'MIT', 'n_tokens_mistral': 1521, 'n_tokens_neox': 1428, 'n_words': 231, 'path': 'notebooks/01-EDA-Celeba.ipynb', 'repo_name': 'AmanDaVinci/DiSENN', 'size': '3850'}