<jupyter_start><jupyter_text>1. Generation of a dataset 1.a. Obtain dataset with true labels<jupyter_code>import sklearn.datasets as datasets

n_classes = 6
classes = list(range(n_classes))
n_samples = 20000
n_features = 80
n_redundant = 0
n_clusters_per_class = 2
n_informative = n_features
X, y = datasets.make_classification(n_samples=n_samples, n_features=n_features,
                           n_classes=n_classes, random_state=random_state,
                           n_redundant=n_redundant,
                           n_informative=n_informative,
                           n_clusters_per_class=n_clusters_per_class)
n_samples = X.shape[0]
n_features = X.shape[1]

Y = label_binarize(y, range(n_classes))
plt.scatter(X[:500,0], X[:500,1], c=y[:500], cmap=cmap)
print('n_samples = {}'.format(n_samples))
print('n_features = {}'.format(n_features))

print('X.shape = {}'.format(X.shape))
print('y.shape = {}'.format(y.shape))<jupyter_output>n_samples = 20000
n_features = 80
X.shape = (20000, 80)
y.shape = (20000,)<jupyter_text>1.b. Divide into training, validation and test- Validation and test will always have only true labels, while the training may have weak labels as well- $S_{train} = \{S_{wt-train}, S_{w-train}\} = [\{(x_i, b_i, y_i), i = 1,...,n\} X x Z x C, \{(x_i, b_i), i = 1,...,n\} \in X x Z\}]$- $S_{val} = \{(x_i, y_i), i = 1,...,n\} \in X x C$- $S_{test} = \{(x_i, y_i), i = 1,...,n\} \in X x C$<jupyter_code>#train_val_test_proportions = numpy.array([0.5, 0.2, 0.3])
print('Original proportions for the 3 partitions (train, validation and test)')
print(train_val_test_proportions)
# Ensure that all proportions sum to 1
train_val_test_proportions /= train_val_test_proportions.sum()
print('Proportions where to split')
train_val_test_proportions = numpy.cumsum(train_val_test_proportions)
print(train_val_test_proportions)
print('Indices where to split (from a total of {} samples)'.format(X.shape[0]))
indices = (train_val_test_proportions*X.shape[0]).astype(int)[:-1]
print(indices)

# # Divide into training, validation and test
X_train, X_val, X_test = numpy.array_split(X, indices)
Y_train, Y_val, Y_test = numpy.array_split(Y, indices)
y_train, y_val, y_test = numpy.array_split(y, indices)

print('Final sizes')
print('Training samples = {}'.format(X_train.shape[0]))
print('Validation samples = {}'.format(X_val.shape[0]))
print('Test samples = {}'.format(X_test.shape[0]))<jupyter_output>Original proportions for the 3 partitions (train, validation and test)
[0.5 0.7 1. ]
Proportions where to split
[0.22727273 0.54545455 1.        ]
Indices where to split (from a total of 20000 samples)
[ 4545 10909]
Final sizes
Training samples = 4545
Validation samples = 6364
Test samples = 9091<jupyter_text>1.c. Generate weakening processes- This will generate weak labels given the specified mixing process.- It will also show 3 plots with the true labels, weak labels and the corresponding rows of the mixing matrix M.- In all the mixing processes we remove the unlabeled option as this can be seen as the all labels (if we assume that every samples belongs to one class)<jupyter_code>#M_method_list = ['odd_even', 'random_weak', 'noisy', 'random_noise', 'IPL', 'quasi_IPL']
#alpha = 0.1
#beta = 1 - alpha

M_list = []
for i, key in enumerate(M_method_list):
    M_list.append(computeM(n_classes, alpha=1.0, beta=beta, method=key, seed=random_state,
                           unsupervised=False))
    for s in range(n_classes):
        if s not in M_supersets[i]:
            M_list[-1][M_list[-1][:,s] != 0] = numpy.zeros(n_classes)
    M_list[-1] /= M_list[-1].sum(axis=0)
    M_list[-1] = numpy.nan_to_num(M_list[-1])

    print('\nMixing matrix for set {} of type {} and shape = {}\n{}'.format(
        i, key, M_list[-1].shape, numpy.round(M_list[-1], decimals=2)))
    
    fig = plt.figure(figsize=(3, 3)) 
    ax = fig.add_subplot(1,1,1)
    ax.set_title('True M')
    cax = ax.imshow(M_list[-1], interpolation='nearest', aspect='auto')<jupyter_output>/home/maikel/git/uob/RecyclingLabels/venv/lib/python3.6/site-packages/ipykernel_launcher.py:12: RuntimeWarning: invalid value encountered in true_divide
  if sys.path[0] == '':<jupyter_text>1.d. Divide training into weak portions- Currently every weak partition is of the same size- We will assume that a proportion of each weak set has been annotated with the true labels<jupyter_code>#w_wt_drop_proportions = numpy.array([0.1, 0.1]) # for weak, for true [the rest to drop]
cut_indices = (w_wt_drop_proportions.cumsum()*X_train.shape[0]).astype(int)
print('Indices for the cuts = {}'.format(cut_indices))

X_w_train, X_wt_train, _ = numpy.array_split(X_train, cut_indices)
y_w_train, y_wt_train, _ = numpy.array_split(y_train, cut_indices)
Y_w_train, Y_wt_train, _ = numpy.array_split(Y_train, cut_indices)
print('Portion with only weak labels = {}'.format(X_w_train.shape[0]))
print('Portion with weak and true labels = {}'.format(X_wt_train.shape[0]))

X_w_train_list = []
y_w_train_list = []
Y_w_train_list = []
for s in M_supersets:
    indices = numpy.isin(y_w_train, s)
    X_w_train_list.append(X_w_train[indices])
    y_w_train_list.append(y_w_train[indices])
    Y_w_train_list.append(Y_w_train[indices])

Z_w_train_list = []
z_w_train_list = []

print('## Portion with only weak labels ##')
for i, M in enumerate(M_list):
    print('Generating weak labels for set {} with mixing process {}'.format(i, M_method_list[i]))
    z_w_train_list.append(generateWeak(y_w_train_list[i], M))
    Z_w_train_list.append(binarizeWeakLabels(z_w_train_list[i], n_classes))
    
    print('Total shape = {}'.format(z_w_train_list[-1].shape))
    print('Sample of z labels\n{}'.format(z_w_train_list[-1][:3]))
    print('Sample of Z labels\n{}'.format(Z_w_train_list[-1][:3]))
    
X_wt_train_list = []
y_wt_train_list = []
Y_wt_train_list = []
for s in M_supersets:
    indices = numpy.isin(y_wt_train, s)
    X_wt_train_list.append(X_wt_train[indices])
    y_wt_train_list.append(y_wt_train[indices])
    Y_wt_train_list.append(Y_wt_train[indices])
    
Z_wt_train_list = []
z_wt_train_list = []

print('## Portion with both weak and true labels ##')
for i, M in enumerate(M_list):
    print('Generating weak labels for set {} with mixing process {}'.format(i, M_method_list[i]))
    z_wt_train_list.append(generateWeak(y_wt_train_list[i], M))
    Z_wt_train_list.append(binarizeWeakLabels(z_wt_train_list[i], n_classes))
    
    print('Total shape = {}'.format(z_wt_train_list[-1].shape))
    print('Sample of z labels\n{}'.format(z_wt_train_list[-1][:3]))
    print('Sample of Z labels\n{}'.format(Z_wt_train_list[-1][:3]))
from experiments.visualizations import plot_multilabel_scatter

fig = plt.figure(figsize=(6, len(z_wt_train_list)*3))
j = 1
for i in range(len(Z_wt_train_list)):
    X_sample = X_wt_train_list[i][:50].reshape((50, -1))
    ax = fig.add_subplot(len(Z_wt_train_list), 2, j)
    _ = plot_multilabel_scatter(X_sample, Z_wt_train_list[i][:100], fig=fig,
                                ax=ax, title='Weak labels set {}'.format(i), cmap=cmap)
    ax.set_ylabel('M {}'.format(M_method_list[i]))
    ax = fig.add_subplot(len(Z_wt_train_list), 2, j+1)
    _ = plot_multilabel_scatter(X_sample, Y_wt_train_list[i][:100], fig=fig,
                                ax=ax, title='True labels set {}'.format(i), cmap=cmap)
    j += 2
fig.tight_layout()<jupyter_output><empty_output><jupyter_text>Define a common model<jupyter_code>from keras.callbacks import EarlyStopping, Callback
from keras import regularizers

def log_loss(y_true, y_pred):
    y_pred = K.clip(y_pred, K.epsilon(), 1.0-K.epsilon())
    out = -y_true*K.log(y_pred)
    return K.mean(out, axis=-1)

#max_epochs = 1000

# Callback to show performance per epoch in the same line
class EpochCallback(Callback):
    def on_epoch_end(self, epoch, logs={}):
        print('\rEpoch {}, val_loss = {:.2e}, val_acc = {:.2f}'.format(epoch, logs['val_loss'], logs['val_acc']), end=' ')

# Callback for early stopping
epoch_callback = EpochCallback()
early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=int(max_epochs/20), 
                               verbose=2, mode='auto', baseline=None,
                               restore_best_weights=True)

def make_model(loss, l2=0.0):
    # Careful that it is ussing global variables for the input and output shapes
    numpy.random.seed(0)
    model = keras.models.Sequential()
    model.add(keras.layers.Dense(Y.shape[1], input_shape=X[0].shape,
                                 kernel_regularizer=regularizers.l2(l2),
                                 activation='softmax'))
    model.compile(optimizer='adam', loss=loss, metrics=['ce', 'mse', 'acc'])
    return model

# Keyword arguments for the fit function
fit_kwargs = dict(validation_data=(X_val, Y_val), epochs=max_epochs, verbose=0,
                  callbacks=[early_stopping, epoch_callback], shuffle=True)

# Save the final model for each method
final_models = {}<jupyter_output><empty_output><jupyter_text>Fully supervised (upperbound)Train with all true labels<jupyter_code>x_axis = range(sum([aux.shape[0] for aux in y_w_train_list]))
fig = plt.figure()
ax1 = fig.add_subplot(1, 2, 1)
ax1.scatter(x_axis, numpy.concatenate(y_w_train_list))
ax2 = fig.add_subplot(1, 2, 2)
ax2.scatter(x_axis, numpy.concatenate(z_w_train_list))
train_method = 'Supervised'

# In this dataset the best l2 parameter is 0.0
l2_list = numpy.array([0.0, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0, 1e1])
l2_list = numpy.array([0.0001])

model_supervised_list = []
val_losses = numpy.zeros_like(l2_list)
for i, l2 in enumerate(l2_list):
    print('Evaluating l2 regularization = {}'.format(l2))
    model = make_model(log_loss, l2=l2)
    history = model.fit(numpy.concatenate((*X_w_train_list, *X_wt_train_list)),
                        numpy.concatenate((*Y_w_train_list, *Y_wt_train_list)),
                        **fit_kwargs)

    plot_history(history, model, X_test, y_test)
    model_supervised_list.append(model)
    best_epoch = numpy.argmin(model.history.history['val_loss'])
    val_losses[i] = model.history.history['val_loss'][best_epoch]
    plt.show()

best_supervised = numpy.argmin(val_losses)
final_models[train_method] = model_supervised_list[best_supervised]
l2 = l2_list[best_supervised]
print('Best l2 = {}'.format(l2))

fig = plt.figure()
ax = fig.add_subplot(111)
ax.semilogx(l2_list, val_losses, 'o-')
ax.scatter(l2, val_losses[best_supervised], color='gold',
           edgecolor='black', marker='*', s=150, zorder=3)<jupyter_output>Evaluating l2 regularization = 0.0001
Epoch 102, val_loss = 1.29e-01, val_acc = 0.72 Restoring model weights from the end of the best epoch
Epoch 103, val_loss = 1.29e-01, val_acc = 0.73 Epoch 00104: early stopping<jupyter_text>Our method with EM and original MTrain EM with all weak labels<jupyter_code>def EM_log_loss(y_true, y_pred):
    y_pred = K.clip(y_pred, K.epsilon(), 1.0-K.epsilon())
    Q = y_true * y_pred
    Z_em_train = Q / K.sum(Q, axis=-1, keepdims=True)
    out = -K.stop_gradient(Z_em_train)*K.log(y_pred)
    return K.mean(out, axis=-1)

model = make_model(EM_log_loss, l2=l2)

M_true_list = []
n_samples_train = X_w_train.shape[0] + X_wt_train.shape[0]
# Add weak samples
for i, M in enumerate(M_list):
    q = (X_w_train_list[i].shape[0]/n_samples_train)
    M_true_list.append(M * q)
    print('q_{} weak = {:.3f}'.format(i, q))
# Add true samples
M_supervised = computeM(n_classes, method='supervised')
for i, M in enumerate(M_list):
    q = (X_wt_train_list[i].shape[0]/n_samples_train)
    M_true_list.append(M_supervised * q)
    print('q_{} true = {:.3f}'.format(i, q))
    
M_true = numpy.concatenate(M_true_list)
last_index = 0
Z_train_index_list = []
V_train_list = []
# Add weak samples
for i in range(len(M_method_list)):
    Z_train_index_list.append(last_index + weak_to_index(Z_w_train_list[i], method=M_method_list[i]))
    last_index += len(M_list[i])
    V_train_list.append(M_true[Z_train_index_list[-1]])
# Add true samples
for i in range(len(M_method_list)):
    Z_train_index_list.append(last_index + weak_to_index(Y_wt_train_list[i], method='supervised'))
    last_index += n_classes
    V_train_list.append(M_true[Z_train_index_list[-1]])

history = model.fit(numpy.concatenate((*X_w_train_list, *X_wt_train_list)),
                    numpy.concatenate(V_train_list),
                    **fit_kwargs)
    
plot_history(history, model, X_test, y_test)

final_models['EM original M'] = model<jupyter_output>q_0 weak = 0.218
q_1 weak = 0.224
q_2 weak = 0.225
q_0 true = 0.126
q_1 true = 0.114
q_2 true = 0.093
Epoch 102, val_loss = 1.34e-01, val_acc = 0.72 Restoring model weights from the end of the best epoch
Epoch 103, val_loss = 1.34e-01, val_acc = 0.73 Epoch 00104: early stopping<jupyter_text>Our method with EM and estimated M<jupyter_code>from wlc.WLweakener import estimate_M
model = make_model(EM_log_loss, l2=l2)

M_estimated_list = []
n_samples_train = X_w_train.shape[0] + X_wt_train.shape[0]
# Add weak samples
for i in range(len(M_list)):
    M = estimate_M(Z_wt_train_list[i], Y_wt_train_list[i],
                   range(n_classes), reg='Partial', Z_reg=Z_w_train_list[i], alpha=1)
    s = M_supersets[i]
    for j in range(n_classes):
        if j not in s:
            M[:,j] = 0
    q = (X_w_train_list[i].shape[0]/n_samples_train)
    M_estimated_list.append(M * q)
    print('q_{} weak = {:.3f}'.format(i, q))
# Add true samples
M_supervised = computeM(n_classes, method='supervised')
for i in range(len(M_list)):
    q = (X_wt_train_list[i].shape[0]/n_samples_train)
    M_estimated_list.append(M_supervised * q)
    print('q_{} true = {:.3f}'.format(i, q))
    
M_estimated = numpy.concatenate(M_estimated_list)
last_index = 0
Z_train_index_list = []
V_train_list = []
# Add weak samples
for i in range(len(M_method_list)):
    Z_train_index_list.append(last_index + weak_to_index(Z_w_train_list[i], method='random_weak'))
    last_index += 2**n_classes
    V_train_list.append(M_estimated[Z_train_index_list[-1]])
# Add true samples
for i in range(len(M_method_list)):
    Z_train_index_list.append(last_index + weak_to_index(Y_wt_train_list[i], method='supervised'))
    last_index += n_classes
    V_train_list.append(M_estimated[Z_train_index_list[-1]])

history = model.fit(numpy.concatenate((*X_w_train_list, *X_wt_train_list)),
                    numpy.concatenate(V_train_list),
                    **fit_kwargs)
    
plot_history(history, model, X_test, y_test)

final_models['EM estimated M'] = model
M_true_list[0].sum(axis=0)
for i, (m1, m2) in enumerate(zip(M_true_list, M_estimated_list)):
    fig = plt.figure(figsize=(10, 5)) 
    ax = fig.add_subplot(1,3,1)
    ax.set_title('True M')
    cax = ax.imshow(m1, interpolation='nearest', aspect='auto')
    fig.colorbar(cax, orientation="horizontal")
    ax = fig.add_subplot(1,3,2)
    ax.set_title('Estimated M')
    cax = ax.imshow(m2, interpolation='nearest', aspect='auto')
    fig.colorbar(cax, orientation="horizontal")
    if m1.shape == m2.shape:
        mse = numpy.power(m1 - m2, 2).sum()
        ax = fig.add_subplot(1,3,3)
        ax.set_title('MSE = {:.2f}'.format(mse))
        cax = ax.imshow(numpy.power(m1 - m2, 2), interpolation='nearest', aspect='auto')
        fig.colorbar(cax, orientation="horizontal")<jupyter_output><empty_output><jupyter_text>Weak (lowerbound)<jupyter_code>model = make_model(log_loss, l2=l2)

history = model.fit(numpy.concatenate((*X_w_train_list, *X_wt_train_list)),
                    numpy.concatenate((*Z_w_train_list, *Y_wt_train_list)),
                    **fit_kwargs)

plot_history(history, model, X_test, y_test)

final_models['Weak'] = model<jupyter_output>Epoch 129, val_loss = 1.34e-01, val_acc = 0.71 Restoring model weights from the end of the best epoch
Epoch 130, val_loss = 1.33e-01, val_acc = 0.71 Epoch 00131: early stopping<jupyter_text>Optimistic Superset Loss<jupyter_code>def OSL_log_loss(y_true, y_pred):
    # Careful, I had to use a global variable here for the number of classes
    # for some reason I can not use y_osl.shape[-1] in the reshape function
    y_pred = K.clip(y_pred, K.epsilon(), 1.0-K.epsilon())
    y_osl = y_true * y_pred
    y_osl_max = K.max(y_osl, axis=-1)
    y_osl_max = K.repeat_elements(y_osl_max, n_classes, 0)
    y_osl_max = K.reshape(y_osl_max, (-1, n_classes))
    y_osl = K.cast(K.equal(y_osl, y_osl_max), y_pred.dtype)
    y_osl = y_osl / K.sum(y_osl, axis=-1, keepdims=True)
    out = -K.stop_gradient(y_osl) * K.log(y_pred)
    return K.mean(out, axis=-1)

model = make_model(OSL_log_loss, l2=l2)

history = model.fit(numpy.concatenate((*X_w_train_list, *X_wt_train_list)),
                    numpy.concatenate((*Z_w_train_list, *Y_wt_train_list)),
                    **fit_kwargs)
    
plot_history(history, model, X_test, y_test)

final_models['OSL'] = model<jupyter_output>Epoch 102, val_loss = 1.30e-01, val_acc = 0.72 Restoring model weights from the end of the best epoch
Epoch 103, val_loss = 1.30e-01, val_acc = 0.72 Epoch 00104: early stopping<jupyter_text>Convex Loss for Partial Labels def CLPL_log_loss(y_true, y_pred):     to implement    return K.mean(out, axis=-1)model = make_model(CLPL_log_loss, l2=l2)history = model.fit(numpy.concatenate((*X_w_train_list, *X_wt_train_list)),                    numpy.concatenate((*Z_w_train_list, *Y_wt_train_list)),                    **fit_kwargs)    plot_history(history, model, X_test, y_test)final_models['OSL'] = model<jupyter_code>plt.figure(figsize=(15, 4))
lowest_acc = 1.0
highest_acc = 0.0
test_acc_dict = {}
for i, (key, model) in enumerate(sorted(final_models.items())):
    lw = (len(final_models)*2 - i)/5
    p = plt.plot(model.history.history['val_acc'], '-', lw=lw, label='Val. ' + key)
    test_acc = numpy.mean(model.predict_classes(X_test) == y_test)
    print('{} : {}'.format(key, test_acc))
    plt.axhline(y=test_acc, color=p[0].get_color(), lw=lw, linestyle='--')
    lowest_acc = test_acc if test_acc < lowest_acc else lowest_acc
    highest_acc = test_acc if test_acc > highest_acc else highest_acc
    test_acc_dict[key] = test_acc
plt.title('Validation accuracy (dashed for test set)')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
range_acc = highest_acc - lowest_acc
plt.ylim([lowest_acc-range_acc*0.2, highest_acc+range_acc*0.2])
plt.legend()<jupyter_output>EM estimated M : 0.7123333333333334
EM original M : 0.717
OSL : 0.7143333333333334
Supervised : 0.7178333333333333
Weak : 0.7078333333333333<jupyter_text>Save results and aggregate- The following saves all the results of this experiment in a csv file- And the next cell loads all the results with similar format and aggregates them in a final plot<jupyter_code>export_dictionary = dict(
    dataset_name=dataset_name,
    random_state=random_state,
    train_prop=train_val_test_proportions[0],
    val_prop=train_val_test_proportions[1],
    test_prop=train_val_test_proportions[2],
    weak_prop=w_wt_drop_proportions[0],
    weak_true_prop=w_wt_drop_proportions[1],
    n_samples=n_samples,
    n_samples_train=n_samples_train,
    M_method_list='"' + ",".join(M_method_list) + '"',
    models='"' + ",".join(final_models.keys()) + '"',
    alpha=alpha,
    beta=beta,
    max_epochs=max_epochs,
)

import datetime

unique_file = 'Example_14_{}_a{}_r{:03.0f}_{}'.format(dataset_name, alpha*100, random_state,
                                              datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S'))

export_dictionary = {**export_dictionary, **test_acc_dict}
csv_text ="\n".join([k+','+str(v) for k,v in export_dictionary.items()])
#You can store this CSV string variable to file as below
with open(unique_file + "_summary.csv", "w") as file:
    file.write(csv_text)
import os
import glob
import pandas

cmap = plt.cm.get_cmap('tab20')

from cycler import cycler
default_cycler = (cycler(color=['darkred', 'forestgreen', 'darkblue', 'violet', 'darkorange', 'saddlebrown']) +
                  cycler(linestyle=['-', '--', '-.', '-', '--', '-.']) + 
                  cycler(marker=['o', 'v', 'x', '*', '+', '.']) +
                  cycler(lw=[2, 1.8, 1.6, 1.4, 1.2, 1]))

plt.rcParams['figure.figsize'] = (3, 2)
plt.rcParams["figure.dpi"] = 100
plt.rc('lines', linewidth=1)
plt.rc('axes', prop_cycle=default_cycler)

files_list = glob.glob("./Example_14*summary.csv")
print('List of files to aggregate')
print(files_list)

list_ = []

for file_ in files_list:
    df = pandas.read_csv(file_,index_col=0, header=None, quotechar='"').T
    list_.append(df)

df = pandas.concat(list_, axis = 0, ignore_index = True)
df = df[df['dataset_name'] == dataset_name]
del df['dataset_name']
df_grouped = df.groupby(['beta', 'M_method_list'])
for name, df_ in df_grouped:
    print(name)
    n_iterations = len(df_['random_state'].unique())
    columns = df_['models'].iloc[0].split(',')
    columns.append('n_samples_train')
    df_ = df_[columns]
    df_ = df_.apply(pandas.to_numeric)
    df_.index = df_['n_samples_train']
    del df_['n_samples_train']
    df_.sort_index(inplace=True)
    df_ = df_.groupby(df_.index).mean()
    fig = plt.figure(figsize=(5, 4))
    ax = fig.add_subplot(111)
    for column in sorted(df_.columns):
        ax.plot(df_.index, df_[column], label=column)
    ax.set_title('dataset {}, beta = {}'.format(dataset_name, name[0]))
    ax.set_ylabel('Mean acc. (#it {})'.format(n_iterations))
    ax.set_xlabel('Number of weak samples')
    ax.legend()
    fig.tight_layout()
    fig.savefig(os.path.join('Example_14_{}_a{:03.0f}.svg'.format(dataset_name,
                                                         float(name[0])*100)))<jupyter_output>List of files to aggregate
['./Example_14_make_classification_a100.0_r000_2019_03_29_17_24_19_summary.csv', './Example_14_make_classification_a90.0_r000_2019_04_01_17_06_22_summary.csv', './Example_14_make_classification_a90.0_r000_2019_04_01_16_23_28_summary.csv', './Example_14_make_classification_a100.0_r000_2019_03_29_18_02_04_summary.csv', './Example_14_make_classification_a90.0_r000_2019_04_01_17_07_01_summary.csv', './Example_14_make_classification_a90.0_r000_2019_04_01_16_19_52_summary.csv', './Example_14_make_classification_a90.0_r000_2019_04_01_16_12_26_summary.csv', './Example_14_make_classification_a100.0_r000_2019_03_29_17_19_14_summary.csv']
('0.1', 'IPL,IPL,IPL')
('0.8', 'IPL,IPL,IPL')
{'fraction_non_alphanumeric': 0.14051921266197373, 'fraction_numerical': 0.03564542886607183, 'mean_word_length': 4.401792201501574, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '533322', 'license': 'MIT', 'n_tokens_mistral': 8939, 'n_tokens_neox': 8410, 'n_words': 1635, 'path': 'notebooks/Example_14_full_em_osl_make_classification_multiple_ipl.ipynb', 'repo_name': 'perellonieto/RecyclingLabels', 'size': '22303'}