<jupyter_start><jupyter_text>Import Data<jupyter_code>data = Datasets.get_preprocessed_datasets("RAW","PCA")
raw = data["RAW"]
pca = data["PCA"]
# choose random 6 subsets
np.random.seed(42)

randomset = list()
for c in np.random.choice(45, size=6):
    randomset.append(os.listdir("data/PCA")[c])

randomset.append("6,9")
randomset<jupyter_output><empty_output><jupyter_text>First find sweet spot of AE Epochs<jupyter_code>for subsetname in randomset:
    for epochs in range(1,30,2):
        print(subsetname, epochs)
        x_train = raw[subsetname]["x_train"]
        x_test = raw[subsetname]["x_test"]

        #autoencoder
        x_train_ae,x_test_ae,hist= pp.Autoencoder(x_train=x_train, x_test=x_test, outputsize=4,epochs=epochs)
        h.store(x_train_ae,"data/ae_epochs/epochs={}/{}".format(epochs,subsetname),"x_train_ae")
        h.store(x_test_ae,"data/ae_epochs/epochs={}/{}".format(epochs,subsetname),"x_test_ae")
        h.store(hist,"data/ae_epochs/epochs={}/{}".format(epochs,subsetname),"hist")
raw = Datasets.get_preprocessed_datasets("RAW")["RAW"]
for epochs_folder in sorted(os.listdir("data/ae_epochs"),key=lambda x: int(x.split("=")[-1])):


    for subsetname in os.listdir("data/ae_epochs/{}".format(epochs_folder)):

        print(epochs_folder, subsetname)
        x_train = np.load("data/ae_epochs/{}/{}/x_train_ae.npy".format(epochs_folder,subsetname))
        x_test = np.load("data/ae_epochs/{}/{}/x_test_ae.npy".format(epochs_folder,subsetname))

        tf.compat.v1.reset_default_graph()
        with tf.device("/gpu:0"):
              
            model  = tf.keras.models.Sequential([
                tf.keras.layers.Dense(1, activation='sigmoid', input_shape=(4,))])


            model.compile(optimizer='adam',loss="binary_crossentropy", metrics=['accuracy'])

            hist = model.fit(
                x=x_train,
                y=raw[subsetname]["y_train_binary"],
                epochs=10,
                batch_size = 32,
                verbose=0,
                validation_data=(x_test,raw[subsetname]["y_test_binary"])).history

        weights = model.trainable_weights[0].numpy()
        bias = model.trainable_weights[1].numpy()

        h.store(weights,"measure/ae_epochs/{}/{}".format(epochs_folder,subsetname),"weights")
        h.store(bias,"measure/ae_epochs/{}/{}".format(epochs_folder,subsetname),"bias")
        h.store(hist,"measure/ae_epochs/{}/{}".format(epochs_folder,subsetname),"hist")

        del model<jupyter_output>epochs=1 4,7
epochs=1 1,4
epochs=1 3,6
epochs=1 6,8
epochs=1 6,9
epochs=1 1,9
epochs=1 1,6
epochs=3 4,7
epochs=3 1,4
epochs=3 3,6
epochs=3 6,8
epochs=3 6,9
epochs=3 1,9
epochs=3 1,6
epochs=5 4,7
epochs=5 1,4
epochs=5 3,6
epochs=5 6,8
epochs=5 6,9
epochs=5 1,9
epochs=5 1,6
epochs=7 4,7
epochs=7 1,4
epochs=7 3,6
epochs=7 6,8
epochs=7 6,9
epochs=7 1,9
epochs=7 1,6
epochs=9 4,7
epochs=9 1,4
epochs=9 3,6
epochs=9 6,8
epochs=9 6,9
epochs=9 1,9
epochs=9 1,6
epochs=11 4,7
epochs=11 1,4
epochs=11 3,6
epochs=11 6,8
epochs=11 6,9
epochs=11 1,9
epochs=11 1,6
epochs=13 4,7
epochs=13 1,4
epochs=13 3,6
epochs=13 6,8
epochs=13 6,9
epochs=13 1,9
epochs=13 1,6
epochs=15 4,7
epochs=15 1,4
epochs=15 3,6
epochs=15 6,8
epochs=15 6,9
epochs=15 1,9
epochs=15 1,6
epochs=17 4,7
epochs=17 1,4
epochs=17 3,6
epochs=17 6,8
epochs=17 6,9
epochs=17 1,9
epochs=17 1,6
epochs=19 4,7
epochs=19 1,4
epochs=19 3,6
epochs=19 6,8
epochs=19 6,9
epochs=19 1,9
epochs=19 1,6
epochs=21 4,7
epochs=21 1,4
epochs=21 3,6
epochs=21 6,8[...]<jupyter_text>Store different numbers of Autoencoder Latentdim and PCA Components<jupyter_code>for subsetname in randomset:
    for latentdim in range(1,12,1):

        print(subsetname,latentdim)
        x_train = raw[subsetname]["x_train"]
        x_test = raw[subsetname]["x_test"]

        #apply preprocessings
        #pca
        x_train_pca,x_test_pca = pp.PCA(x_train=x_train, x_test=x_test, outputsize=latentdim)

        h.store(x_train_pca,"data/latentdimvergleich/pca/latentdim={}/{}".format(latentdim,subsetname),"x_train_pca")
        h.store(x_test_pca,"data/latentdimvergleich/pca/latentdim={}/{}".format(latentdim,subsetname),"x_test_pca")

        #autoencoder
        x_train_ae,x_test_ae,hist= pp.Autoencoder(x_train=x_train, x_test=x_test, outputsize=latentdim,epochs=6)
        h.store(x_train_ae,"data/latentdimvergleich/ae/latentdim={}/{}".format(latentdim,subsetname),"x_train_ae")
        h.store(x_test_ae,"data/latentdimvergleich/ae/latentdim={}/{}".format(latentdim,subsetname),"x_test_ae")
        h.store(hist,"data/latentdimvergleich/ae/latentdim={}/{}".format(latentdim,subsetname),"hist")
    
for subsetname in randomset:
    for latentdim in range(1,12,1):

        print(subsetname,latentdim)
        x_train = raw[subsetname]["x_train"]
        x_test = raw[subsetname]["x_test"]
        #autoencoder
        x_train_ae,x_test_ae,hist= pp.Autoencoder(x_train=x_train, x_test=x_test, outputsize=latentdim,epochs=7)
        h.store(x_train_ae,"data/latentdimvergleich/ae/epochs={}/latentdim={}/{}".format(7,latentdim,subsetname),"x_train_ae")
        h.store(x_test_ae,"data/latentdimvergleich/ae/epochs={}/latentdim={}/{}".format(7,latentdim,subsetname),"x_test_ae")
        h.store(hist,"data/latentdimvergleich/ae/epochs={}/latentdim={}/{}".format(7,latentdim,subsetname),"hist")<jupyter_output>1,9 1
1,9 2
1,9 3
1,9 4
1,9 5
1,9 6
1,9 7
1,9 8
1,9 9
1,9 10
1,9 11
6,8 1
6,8 2
6,8 3
6,8 4
6,8 5
6,8 6
6,8 7
6,8 8
6,8 9
6,8 10
6,8 11
1,4 1
1,4 2
1,4 3
1,4 4
1,4 5
1,4 6
1,4 7
1,4 8
1,4 9
1,4 10
1,4 11
1,6 1
1,6 2
1,6 3
1,6 4
1,6 5
1,6 6
1,6 7
1,6 8
1,6 9
1,6 10
1,6 11
4,7 1
4,7 2
4,7 3
4,7 4
4,7 5
4,7 6
4,7 7
4,7 8
4,7 9
4,7 10
4,7 11
3,6 1
3,6 2
3,6 3
3,6 4
3,6 5
3,6 6
3,6 7
3,6 8
3,6 9
3,6 10
3,6 11
6,9 1
6,9 2
6,9 3
6,9 4
6,9 5
6,9 6
6,9 7
6,9 8
6,9 9
6,9 10
6,9 11<jupyter_text>Now run neural Network<jupyter_code>for preprocessing in os.listdir("data/latentdimvergleich(7)"):
    for latentdim in os.listdir("data/latentdimvergleich(7)/{}".format(preprocessing)):
        latentdimnum = int(latentdim.split("=")[-1])
        for subset in os.listdir("data/latentdimvergleich(7)/{}/{}".format(preprocessing,latentdim)):

            print(preprocessing,latentdim,subset)
        
            x_train = np.load("data/latentdimvergleich(7)/{}/{}/{}/x_train_{}.npy".format(preprocessing,latentdim,subset,preprocessing))
            x_test = np.load("data/latentdimvergleich(7)/{}/{}/{}/x_test_{}.npy".format(preprocessing,latentdim,subset,preprocessing))

            tf.compat.v1.reset_default_graph()
            with tf.device("/gpu:0"):

               if latentdimnum > 1:
                    model = tf.keras.models.Sequential([
                         tf.keras.layers.Dense(round(latentdimnum/2), activation='relu', input_shape=(latentdimnum,)),
                         tf.keras.layers.Dense(1, activation='sigmoid', input_shape=(latentdimnum,))])

               else:
                    model  = tf.keras.models.Sequential([
                         tf.keras.layers.Dense(1, activation='sigmoid', input_shape=(latentdimnum,))])
    
               model.compile(optimizer='adam',loss="binary_crossentropy", metrics=['accuracy'])

               hist = model.fit(
                    x=x_train,
                    y=raw[subset]["y_train_binary"],
                    epochs=10,
                    batch_size = 32,
                    verbose=0,
                    validation_data=(x_test,raw[subset]["y_test_binary"])).history
            
            h.store(hist,"measure/latentdimvergleich(7)_bigger/{}/{}/{}".format(preprocessing,latentdim,subset),"hist")

            weights = model.trainable_weights[0].numpy()
            bias = model.trainable_weights[1].numpy()

            h.store(weights,"measure/latentdimvergleich(7)_bigger/{}/{}/{}".format(preprocessing,latentdim,subset),"weights")
            h.store(bias,"measure/latentdimvergleich(7)_bigger/{}/{}/{}".format(preprocessing,latentdim,subset),"bias")

            del model

for latentdim in os.listdir("data/latentdimvergleich/ae/epochs=7/"):
    latentdimnum = int(latentdim.split("=")[-1])
    for subset in os.listdir("data/latentdimvergleich/ae/epochs=7/{}".format(latentdim)):

        print(latentdim,subset)
    
        x_train = np.load("data/latentdimvergleich/ae/epochs=7/{}/{}/x_train_ae.npy".format(latentdim,subset))
        x_test = np.load("data/latentdimvergleich/ae/epochs=7/{}/{}/x_test_ae.npy".format(latentdim,subset))

        tf.compat.v1.reset_default_graph()
        with tf.device("/gpu:0"):

                if latentdimnum > 1:
                    model  = tf.keras.models.Sequential([
                        tf.keras.layers.Dense(round(latentdimnum/2), activation='relu', input_shape=(latentdimnum,)),
                        tf.keras.layers.Dense(1, activation='sigmoid', input_shape=(latentdimnum,))])
                
                model  = tf.keras.models.Sequential([
                    tf.keras.layers.Dense(1, activation='sigmoid', input_shape=(latentdimnum,))])

                model.compile(optimizer='adam',loss="binary_crossentropy", metrics=['accuracy'])

                hist = model.fit(
                    x=x_train,
                    y=raw[subset]["y_train_binary"],
                    epochs=10,
                    batch_size = 32,
                    verbose=0,
                    validation_data=(x_test,raw[subset]["y_test_binary"])).history
        
        h.store(hist,"measure/latentdimvergleich/ae/epochs=7/{}/{}".format(latentdim,subset),"hist")

        weights = model.trainable_weights[0].numpy()
        bias = model.trainable_weights[1].numpy()

        h.store(weights,"measure/latentdimvergleich/ae/epochs=7/{}/{}".format(latentdim,subset),"weights")
        h.store(bias,"measure/latentdimvergleich/ae/epochs=7/{}/{}".format(latentdim,subset),"bias")

        del model
round(3/2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15736348036741699, 'fraction_numerical': 0.059452912082366006, 'mean_word_length': 3.2651743435213088, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '389223', 'license': 'MIT', 'n_tokens_mistral': 4159, 'n_tokens_neox': 3967, 'n_words': 569, 'path': 'praktikum/ae_epochen.ipynb', 'repo_name': 'FelixBieswanger/masKIT', 'size': '9907'}