<jupyter_start><jupyter_text>Imports<jupyter_code>#packages
import numpy
import tensorflow as tf
from tensorflow.core.example import example_pb2

#utils
import os
import random
import pickle
import struct
import time
from generators import *

#keras
import keras
from keras.preprocessing import text, sequence
from keras.preprocessing.text import Tokenizer
from keras.models import Model, Sequential
from keras.models import load_model
from keras.layers import Dense, Dropout, Activation, Concatenate, Dot, Embedding, LSTM, Conv1D, MaxPooling1D, Input, Lambda
    #callbacks
from keras.callbacks import TensorBoard, ModelCheckpoint, Callback

with open('stats-4.pickle', 'rb') as handle: stats = pickle.load(handle)
print(stats)
def get_f1(clean_data, pseudo_data, generator_data, uniform_data):
    tpr_data = []
    tnr_data = []
    for i in range(len(clean_data)):
        TP = clean_data[i] + 1e-10
        TN = pseudo_data[i] + uniform_data[i] + 1e-10
        FN = 1 - clean_data[i] + 1e-10
        FP = 2 - pseudo_data[i] - uniform_data[i] + 1e-10
        tpr = TP/(TP+FN)
        tnr = TN/(TN+FP)
        
        tpr_data.append(tpr)
        tnr_data.append(tnr)
    return tpr_data, tnr_data

model_name = 'tw_on_pseudorandom'
clean_data = []
pseudo_data = []
generator_data = []
uniform_data = []

thresholds_data = []

for threshold in sorted(stats[model_name].keys()):
    thresholds_data.append(threshold)
    for data_type in stats[model_name][threshold].keys():
        if data_type == 'on_clean':
            clean_data.append(stats[model_name][threshold][data_type])
        if data_type == 'on_pseudo':
            pseudo_data.append(stats[model_name][threshold][data_type])
        if data_type == 'on_generator':
            generator_data.append(stats[model_name][threshold][data_type])
        elif data_type == 'on_uniform':
            uniform_data.append(stats[model_name][threshold][data_type])

#get prec, rec, f1
tpr_data_twd, tnr_data_twd = get_f1(clean_data, pseudo_data, generator_data, uniform_data)
results = numpy.concatenate((numpy.array(thresholds_data)[:,numpy.newaxis], 1-numpy.array(tnr_data_twd)[:,numpy.newaxis], 1-numpy.array(tpr_data_twd)[:,numpy.newaxis] ), axis=1)
print(results)<jupyter_output>[[1.00000000e-01 1.72136147e-01 1.05327369e-02]
 [3.00000000e-01 1.22475624e-01 3.19463612e-02]
 [5.00000000e-01 9.79282752e-02 5.75383306e-02]
 [6.00000000e-01 8.60463083e-02 7.41643310e-02]
 [7.00000000e-01 7.43819625e-02 9.54039098e-02]
 [7.50000000e-01 6.82886467e-02 1.09505594e-01]
 [8.00000000e-01 6.08461015e-02 1.27002060e-01]
 [8.10000000e-01 5.89310602e-02 1.31963790e-01]
 [8.20000000e-01 5.67548760e-02 1.38144136e-01]
 [8.30000000e-01 5.54056392e-02 1.43976331e-01]
 [8.40000000e-01 5.36211692e-02 1.51201248e-01]
 [8.50000000e-01 5.15320339e-02 1.58339143e-01]
 [8.60000000e-01 4.96169925e-02 1.66434527e-01]
 [8.70000000e-01 4.80066166e-02 1.74965203e-01]
 [8.80000000e-01 4.59174775e-02 1.86977744e-01]
 [8.90000000e-01 4.40894850e-02 1.99425459e-01]
 [9.00000000e-01 4.16956842e-02 2.12047339e-01]
 [9.10000000e-01 3.91713083e-02 2.25278556e-01]
 [9.20000000e-01 3.66904587e-02 2.44690120e-01]
 [9.30000000e-01 3.32956128e-02 2.65233278e-01]
 [9.40000000e-01 3.02924794e-02 2.983983[...]<jupyter_text>CPU usage<jupyter_code>os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"   # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = ""<jupyter_output><empty_output><jupyter_text>Global parameters<jupyter_code># Embedding
max_features = 400000
maxlen_text = 400
maxlen_summ = 80
embedding_size = 100 #128

# Convolution
kernel_size = 5
filters = 64
pool_size = 4

# LSTM
lstm_output_size = 70

# Training
batch_size = 32
epochs = 5
use_multiprocessing = True
workers = 4
shuffle = False

model_path_prefix = '/home/donald/documents/MT/implementation-and-experiments/'
#model_path_prefix = '/home/oala/Documents/MT/implementation-experiments/'
#data_path_prefix = '/mnt/disks/500gb/experimental-data-mini/experimental-data-mini/'
#data_path_prefix = '/media/oala/4TB/experimental-data/'
data_path_prefix = '/mnt/disks/500gb/experimental-data-mini/'
#get preprocessing data
processing_dir = '/mnt/disks/500gb/stats-and-meta-data/400000/'
#processing_dir = '/media/oala/4TB/experimental-data/stats-and-meta-data/400000/'
with open(processing_dir+'tokenizer.pickle', 'rb') as handle: tokenizer = pickle.load(handle)
embedding_matrix = numpy.load(processing_dir+'embedding_matrix.npy')
#stats
maxi = numpy.load(processing_dir+'training-stats-all/maxi.npy')
mini = numpy.load(processing_dir+'training-stats-all/mini.npy')
sample_info = (numpy.random.uniform, mini,maxi)
list_of_outputs = ['output-scoring/baseline/', 'output-scoring/pg/', 'output-scoring/pgc/','output-scoring/sumgan/']
model_path = model_path_prefix + 'exciting-crazy/experiments/tw-on-pseudorandom/1/best.h5'
threshold = 0.85
model = load_model(model_path)

for output_suffix in list_of_outputs:
    #eval on clean test
    data_dir = data_path_prefix + output_suffix
    with open(data_dir+'partition.pickle', 'rb') as handle: partition = pickle.load(handle)
    with open(data_dir+'labels.pickle', 'rb') as handle: labels = pickle.load(handle)
    
    #batch generator parameters
    params = {'dim': [(maxlen_text,embedding_size),(maxlen_summ,embedding_size)],
              'batch_size': batch_size,
              'shuffle': shuffle,
             'tokenizer':tokenizer,
             'embedding_matrix':embedding_matrix,
             'maxlen_text':maxlen_text,
             'maxlen_summ':maxlen_summ,
             'data_dir':data_dir,
             'sample_info':sample_info}
    #generators
    test_generator = ContAllGenerator(partition['test'], labels, **params)
    # Train model on dataset
    #out = model.evaluate_generator(generator=test_generator,
                        #use_multiprocessing=use_multiprocessing,
                        #workers=workers)

    preds = model.predict_generator(generator=test_generator,
                        use_multiprocessing=use_multiprocessing,
                        workers=workers)
    preds[preds<threshold] = 0
    preds[preds!=0] = 1
    
    print(output_suffix+': '+str(numpy.mean(preds)))<jupyter_output>output-scoring/baseline/: 0.5782556
output-scoring/pg/: 0.48798746
output-scoring/pgc/: 0.3918872
output-scoring/sumgan/: 0.45656338
{'fraction_non_alphanumeric': 0.12844472600570161, 'fraction_numerical': 0.12796959138422553, 'mean_word_length': 4.125811688311688, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 21, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '296182', 'license': 'MIT', 'n_tokens_mistral': 2911, 'n_tokens_neox': 2372, 'n_words': 376, 'path': 'experiments/eval-script/.ipynb_checkpoints/eval_template_output-scoring-v2-checkpoint.ipynb', 'repo_name': 'peglegpete/gen-seq-noise', 'size': '6314'}