<jupyter_start><jupyter_text>Model EvaluationFor model performance assessment we want to obtain the distribution of the model quality over 20 independent runs of the training procedure.* [The experiment details](The-experiment-details)* [Results](Results) The experiment detalisTrain and test dataset creation<jupyter_code>import os
import sys
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf

sys.path.append('../../..')

from seismicpro.batchflow import Pipeline, B, V, F, C
from seismicpro.batchflow.models.tf import UNet
from seismicpro.batchflow.research import Research, RP
from seismicpro.src import (SeismicDataset, FieldIndex, TraceIndex,
                            draw_histogram)

plt.style.use('ggplot')
path_raw = '/data/H1_WZ/NA/DN02A_LIFT_AMPSCAL.sgy'
path_lift = '/data/H1_WZ/NA/DN02B_SHOTS_LIFT1.sgy'

index = (FieldIndex(name='raw', extra_headers=['offset'], path=path_raw)
         .merge(FieldIndex(name='lift', path=path_lift)))

train_index = index.create_subset(index.indices[:5])
train_set = SeismicDataset(TraceIndex(train_index))
test_set = SeismicDataset(TraceIndex(index.create_subset(index.indices[20:21])))<jupyter_output><empty_output><jupyter_text>Define model config<jupyter_code>model_config = {
    'initial_block/inputs': 'x',
    'inputs': dict(x={'shape': (3000, 1)}, 
                   y={'name':'targets', 'shape': (3000, 1)}),
    'body/filters': [16, 32, 64, 128, 256],
    'body/encoder': dict(layout='caca', kernel_size=7, activation=tf.nn.elu),
    'body/downsample': dict(layout='pd', pool_size=2, pool_strides=2, dropout_rate=0.05),
    'body/decoder': dict(layout='caca', kernel_size=7, activation=tf.nn.elu),
    'body/upsample': dict(layout='tad', kernel_size=7, strides=2,
                          dropout_rate=0.05, activation=tf.nn.elu),
    'loss': 'l1',
    'optimizer': 'Adam'
}<jupyter_output><empty_output><jupyter_text>Define train and test pipelines<jupyter_code>def make_data(batch, **kwagrs):
    return {'x': np.expand_dims(np.vstack(batch.raw), -1),
            'y': np.expand_dims(np.vstack(batch.lift), -1)}

B_SIZE = 16
train_pipeline = (Pipeline()
                  .load(components=('raw', 'lift'), fmt='segy', tslice=np.arange(3000))
                  .init_variable('loss', [])
                  .init_model(mode='dynamic', name='unet',
                              model_class=UNet, config=model_config)
                  .train_model('unet', make_data=make_data,
                               fetches='loss', save_to=V('loss', mode='w'))
                  .run_later(B_SIZE, n_epochs=None, drop_last=True, shuffle=True)
                 ) << train_set

test_pipeline = (Pipeline()
                    .import_model('unet', C('import_from')) 
                    .init_variable('res', [])
                    .init_variable('raw', [])
                    .init_variable('lift', [])
                    .load(components=('raw', 'lift'), tslice=np.arange(3000), fmt='segy')
                    .update(V('raw', mode='a'), B('raw'))
                    .update(V('lift', mode='a'), B('lift'))
                    .predict_model('unet', fetches='predictions', make_data=make_data,
                                   save_to=V('res', mode='a'))
                    .run_later(B_SIZE, n_epochs=1, drop_last=True, shuffle=True)
                   ) << test_set<jupyter_output><empty_output><jupyter_text>Let's define the functions for calculating the metrics<jupyter_code>def get_l1(pipeline):
    """ Calculate l1 norm."""
    res = np.squeeze(np.vstack(pipeline.v("res")), axis=-1)
    lift = np.vstack(np.concatenate(pipeline.v("lift")))
    return np.mean(np.abs(res - lift))<jupyter_output><empty_output><jupyter_text>Create a research object<jupyter_code>research = (Research()
            .init_domain(n_reps=20)
            .add_pipeline(train_pipeline, variables='loss', name='train')
            .add_pipeline(test_pipeline, name='test_ppl', execute=5,
                         run=True, import_from=RP('train'))
            .add_callable(get_l1, returns='l1', name='test', 
                          execute=5, pipeline=RP('test_ppl'))
)<jupyter_output><empty_output><jupyter_text>Run 20 independent training and test procedures<jupyter_code>research.run(n_iters=500, name='reserach_estimation', workers=4,
             devices=[0, 1, 2, 3], bar=True)<jupyter_output>Research reserach_estimation is starting...<jupyter_text>ResultsHistogram and a median value of the test metrics<jupyter_code>df = research.load_results(use_alias=True).df
draw_histogram(df, 'test/l1', 100)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14728850325379608, 'fraction_numerical': 0.01843817787418655, 'mean_word_length': 3.3541076487252126, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '361053', 'license': 'Apache-2.0', 'n_tokens_mistral': 1596, 'n_tokens_neox': 1498, 'n_words': 262, 'path': 'models/Ground-roll_attenuation/Unet_1D_model/model_estimation.ipynb', 'repo_name': 'pplotn/SeismicPro', 'size': '4610'}