<jupyter_start><jupyter_text>Regression for redshift z<jupyter_code>DATA.head()
Z = DATA['z_sdss']
imputer = SimpleImputer(missing_values=np.nan, strategy='median')
x = imputer.fit_transform(DATA[col_name_corr])
x.shape
# DATA.columns
size = x.shape[0]

full_sdss_indx = list(Z[~Z.isna()].index)

X = x[full_sdss_indx,:]
Y = Z[full_sdss_indx].to_numpy()
X.shape
from sklearn.preprocessing import StandardScaler
X_scaled = StandardScaler().fit_transform(X)

train_X, val_X, train_y, val_y = train_test_split(X_scaled,Y, test_size = 0.3, random_state = 14)
# Create an experiment
# api_key could be found on https://www.comet.ml
experiment = Experiment(api_key='<KEY>',
                        project_name="galaxy", workspace='azarodnyuk')
NN_model = Sequential()

# The Input Layer :
NN_model.add(Dense(int(train_X.shape[1]), kernel_initializer='normal',input_dim = train_X.shape[1], activation='relu'))

# The Hidden Layers :
NN_model.add(Dense(int(train_X.shape[1]/2), kernel_initializer='normal',activation='relu'))
NN_model.add(Dense(int(train_X.shape[1]/4), kernel_initializer='normal',activation='relu'))
NN_model.add(Dense(int(train_X.shape[1]/8), kernel_initializer='normal',activation='relu'))
# NN_model.add(Dense(int(train_X.shape[1]/16), kernel_initializer='normal',activation='relu'))

# The Output Layer :
NN_model.add(Dense(1, kernel_initializer='normal',activation='linear'))

# Compile the network :
NN_model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mean_absolute_error']) #'mean_absolute_error'
NN_model.summary()
checkpoint_name = 'Weights-{epoch:03d}--{val_loss:.5f}.hdf5' 
checkpoint = ModelCheckpoint(checkpoint_name, monitor='val_loss', verbose = 1, save_best_only = True, mode ='auto')
callbacks_list = [checkpoint]
NN_model.fit(train_X, train_y, epochs=15, batch_size=100, validation_split = 0.2, callbacks=callbacks_list,
             workers=20, use_multiprocessing=True)
predictions = NN_model.predict(val_X)
sklearn.metrics.r2_score(val_y, predictions)  #0.9998067818472489      0.6159740382704414<jupyter_output><empty_output><jupyter_text>Reconstruct the redshift<jupyter_code>nan_z_indx = list(Z[Z.isna()].index)
X_reconstruct = x[nan_z_indx,:]
z_reconstructed = NN_model.predict(X_reconstruct)
z_reconstructed.flatten()
len(nan_z_indx)
Z[nan_z_indx] = z_reconstructed.flatten()
Z.isna().sum()
Z.name = 'z_regres'
Z.to_csv('z_reconstructed.csv', index = False)
pd.read_csv('z_reconstructed.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15, 'fraction_numerical': 0.02661290322580645, 'mean_word_length': 7.924460431654676, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '397051', 'license': 'MIT', 'n_tokens_mistral': 996, 'n_tokens_neox': 944, 'n_words': 129, 'path': 'clustering/z_regression_rcsed_v2_8.ipynb', 'repo_name': 'Azarodnyuk/galaxymass', 'size': '2500'}