<jupyter_start><jupyter_text>**To do***IterativeImputer*-> models each feature with missing values as a function of other features, and uses that estimate for imputation<jupyter_code>len(np.unique(RCSED_V2.KAPERMAG3ERR_ukidss))-2967341
col_for_nan = anomaly2.sort_values(by='total', ascending=False)[1:149].column_name.to_list()
col_anomaly = anomaly2.sort_values(by='total', ascending=False)[1:149].anomaly_value.to_list()
for col, anomaly in zip(col_for_nan, col_anomaly):
    RCSED_V2[col].replace(anomaly, np.nan)
data_0 = RCSED_V2.fillna(0)
data_0
data_0 = data_0.replace([np.inf, -np.inf], np.nan).fillna(0)<jupyter_output><empty_output><jupyter_text>Labeled part of data<jupyter_code>GrID.head()
len(GrID)
sdss_indx = GrID[-GrID.iGrID.isna()].index.to_list()
len(sdss_indx)
data_0.ind
x = data_0[data_0.ind.isin(sdss_indx)]
len(x)
y = GrID.iloc[sdss_indx]
s = pd.Series(y.iGrID).value_counts()
one_sized = y[y.iGrID.isin(s[s == 1].index)].index.to_list()
y_without_one = y[~y.iGrID.isin(s[s == 1].index)]
len(y_without_one)
x_without_one = x[~x.ind.isin(one_sized)]
x_without_one.head()<jupyter_output><empty_output><jupyter_text>VAE<jupyter_code>from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
x = scaler.fit_transform(x_without_one)
display(x)
x.shape
Y = np.array(y_without_one).reshape(-1)
Y.shape<jupyter_output><empty_output><jupyter_text>-----<jupyter_code>class VAE(nn.Module):
    def __init__(self, x_dim, h_dim1, h_dim2, z_dim):
        super(VAE, self).__init__()
        
        self.fc1 = nn.Linear(x_dim, h_dim1)
        self.fc2 = nn.Linear(h_dim1, h_dim2)
        self.fc31 = nn.Linear(h_dim2, z_dim)
        self.fc32 = nn.Linear(h_dim2, z_dim)

        self.fc4 = nn.Linear(z_dim, h_dim2)
        self.fc5 = nn.Linear(h_dim2, h_dim1)
        self.fc6 = nn.Linear(h_dim1, x_dim)
        
    def encoder(self, x):
        h = F.relu(self.fc1(x))
        h = F.relu(self.fc2(h))
        return self.fc31(h), self.fc32(h)
    
    def sampling(self, mu, log_var):
        std = torch.exp(0.5*log_var)
        eps = torch.randn_like(std)
        return eps.mul(std).add_(mu)
        
    def decoder(self, z):
        h = F.relu(self.fc4(z))
        h = F.relu(self.fc5(h))
        return torch.sigmoid(self.fc6(h)) 
    
    def forward(self, x):
        mu, log_var = self.encoder(x.view(-1, x.shape[1]))
        z = self.sampling(mu, log_var)
        return self.decoder(z), mu, log_var
vae = VAE(x_dim=x.shape[1], h_dim1= int(x.shape[1]/2), h_dim2=int(x.shape[1]/4), z_dim=1)


optimizer = optim.Adam(vae.parameters())
train_loader = torch.utils.data.DataLoader(dataset=x, batch_size=int(len(x)/1000), shuffle=True)#, num_workers=20)

def loss_function(recon_x, x, mu, log_var):
    criterion = nn.MSELoss().cuda()
    BCE = criterion(recon_x, x.view(-1, x.shape[1]))
    KLD = -0.5 * torch.sum(1 + log_var - mu.pow(2) - log_var.exp())
    return BCE + KLD

def train(epoch):
    vae.train()
    train_loss = 0
    for batch_idx in range(400):
        #data = torch.tensor(x.sample(n=1000).astype(np.float32).values)
        data = torch.tensor(x[np.random.choice(range(x.shape[0]), 1000)].astype(np.float32))
        optimizer.zero_grad()
        
        recon_batch, mu, log_var = vae(data)
        loss = loss_function(recon_batch, data, mu, log_var)
        
        loss.backward()
        train_loss += loss.item()
        optimizer.step()
        
        if batch_idx % 100 == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.item() / len(data)))
    print('====> Epoch: {} Average loss: {:.4f}'.format(epoch, train_loss / len(train_loader.dataset)))
from tqdm.notebook import tqdm

for epoch in tqdm(range(1, 25)):
    train(epoch)
mu, log_var = vae.encoder(torch.tensor(x.astype(np.float32)))
embed = vae.sampling(mu, log_var).detach().numpy()
hdbScan = hdbscan.hdbscan_.HDBSCAN(min_cluster_size=2, min_samples=10, 
                                   allow_single_cluster=False, core_dist_n_jobs=20).fit(embed)
m2 = homogeneity_completeness_v_measure(Y,hdbScan.labels_)
m2
pd.Series(hdbScan.labels_).value_counts(ascending=True).tail(15)
pd.Series(Y).value_counts(ascending=True).tail(15)
z_dim = 1
input_size = x.shape[1]
n_samples = x.shape[0]

h_dim1 = int(x.shape[1]/2)
h_dim2 = int(x.shape[1]/4)
n_samples
class AVAE(nn.Module):
    def __init__(self):
        super(AVAE, self).__init__()
        
        self.gen_l1 = torch.nn.Linear(z_dim, h_dim2)
        self.gen_l2 = torch.nn.Linear(h_dim2, h_dim1)
        self.gen_l3 = torch.nn.Linear(h_dim1, input_size)
        
        self.enc_l1 = torch.nn.Linear(input_size+z_dim, h_dim1)
        self.enc_l2 = torch.nn.Linear(h_dim1, h_dim2)
        self.enc_l3 = torch.nn.Linear(h_dim2, z_dim)
        
        self.disc_l1 = torch.nn.Linear(input_size+z_dim, h_dim1)
        self.disc_l2 = torch.nn.Linear(h_dim1, h_dim2)
        self.disc_l3 = torch.nn.Linear(h_dim2, 1)
    
    def sample_prior(self, s):
        if self.training:
            m = torch.zeros((s.data.shape[0], z_dim))
            std = torch.ones((s.data.shape[0], z_dim))
            d = Variable(torch.normal(m,std))
        else:
            d = Variable(torch.zeros((s.data.shape[0], z_dim)))
            
        return d
        
    def discriminator(self, x,z):
        i = torch.cat((x, z), dim=1)
        h = F.relu(self.disc_l1(i))
        h = F.relu(self.disc_l2(h))
        return self.disc_l3(h)  
        
    def sample_posterior(self, x):
        i = torch.cat((x, self.sample_prior(x)), dim=1)
        h = F.relu(self.enc_l1(i))
        h = F.relu(self.enc_l2(h))
        return self.enc_l3(h)
    
    def decoder(self, z):
        i = F.relu(self.gen_l1(z))
        h = F.relu(self.gen_l2(i))
        return torch.sigmoid(self.gen_l3(h))
        
    
    def forward(self, x):
        mu, log_var = self.encoder(x.view(-1, x.shape[1]))
        z = self.sampling(mu, log_var)
        return self.decoder(z), mu, log_var

    def forward(self, x):
        z_p = self.sample_prior(x)
        
        z_q = self.sample_posterior(x)
        
        log_d_prior = self.discriminator(x, z_p)
        log_d_posterior = self.discriminator(x, z_q)
        
        disc_loss = torch.mean(
            nn.functional.binary_cross_entropy_with_logits(
            log_d_posterior, torch.ones_like(log_d_posterior)
        )
        + nn.functional.binary_cross_entropy_with_logits(
            log_d_prior, torch.zeros_like(log_d_prior))
        )
        
        x_recon = self.decoder(z_q)
        recon_liklihood = -nn.functional.binary_cross_entropy(
                                                x_recon, x)*x.data.shape[0]
        
        gen_loss = torch.mean(log_d_posterior)-torch.mean(recon_liklihood)
        
        return disc_loss, gen_loss
model = AVAE()
disc_params = []
gen_params = []
for name, param in model.named_parameters():
    
    if 'disc' in name:
        
        disc_params.append(param)
    else:
        gen_params.append(param)
disc_optimizer = torch.optim.Adam(disc_params, lr=1e-3)
gen_optimizer = torch.optim.Adam(gen_params, lr=1e-3)
x.shape
def train_AVAE(epoch, batches_per_epoch = 501, log_interval=500):
    model.train()
    
    ind = np.arange(x.shape[0])
    for i in range(batches_per_epoch):
        data = torch.from_numpy(x[np.random.choice(ind, size=batch_size)])
        data = Variable(data, requires_grad=False).float()
        
        
        discrim_loss, gen_loss= model(data)
        
        gen_optimizer.zero_grad()
        gen_loss.backward(retain_graph=True)
        gen_optimizer.step()
        
        disc_optimizer.zero_grad()
        discrim_loss.backward(retain_graph=True)
        disc_optimizer.step()
        if (i % log_interval == 0) and (epoch % 1 ==0):
            #Print progress
            print('Train Epoch: {} [{}/{}]\tLoss: {:.6f}\tLoss: {:.6f}'.format(
                epoch, i * batch_size, batch_size*batches_per_epoch,
                discrim_loss.item() / len(data), gen_loss.item() / len(data)))

    print('====> Epoch: {} done!'.format(
          epoch))
%%time
for epoch in range(1, 15):
    train_AVAE(epoch)
data = Variable(torch.from_numpy(x), requires_grad=False)

model.train()
zs = model.sample_posterior(data.float()).detach().numpy()


#import matplotlib.pyplot as plt
#%matplotlib inline

#plt.scatter(zs[:,0], zs[:, 1], c=Y)
#check the shape
zs.shape
x.shape
embed.shape<jupyter_output><empty_output><jupyter_text>TODO: 1) grid search for min_samples & min_cluster_size2) optimize AVAE + graphics<jupyter_code>hdbScan2 = hdbscan.hdbscan_.HDBSCAN(min_cluster_size=2, min_samples=9, 
                                   allow_single_cluster=False, core_dist_n_jobs=20).fit(zs)
len(hdbScan2.labels_)
m4 = homogeneity_completeness_v_measure(Y,hdbScan2.labels_) 
m4
len(np.unique(hdbScan2.labels_))
len(np.unique(Y)) #true number of clusters
#9
pd.Series(hdbScan2.labels_).value_counts(ascending=True).tail(15)
ls = pd.DataFrame(hdbScan2.labels_)
ind = ls[ls[0] == -1].index.to_list()
pd.Series(Y).value_counts(ascending=True).tail(15)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14599805047113615, 'fraction_numerical': 0.023719267843604463, 'mean_word_length': 2.951219512195122, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '206755', 'license': 'MIT', 'n_tokens_mistral': 3727, 'n_tokens_neox': 3595, 'n_words': 543, 'path': 'Galaxies_AVAE.ipynb', 'repo_name': 'ketrint/galaxymass', 'size': '9233'}