<jupyter_start><jupyter_text>Metrics for cubesThis notebooks computes and shows various metrics for each cube.Pseudocode of this notebook looks like: ```pythonfor each cube:    mkdir    infer geometry    if SHOW, log to std.out        for each metric:        compute metric        save png        if SAVE_POINT_CLOUDS, save point cloud        if SHOW, draw in std.out  ```  There are following parameters: * `paths` controls which cubes are used*  `metrics` allows to choose computed metrics* `script_dir` is a location to save metric maps* `local_kwargs`, `support_kwargs` define exact parameters of metric computation* `ADD_PREFIX` stores parameters of metric computation in the name of saved metric map* `SAVE_POINT_CLOUD` stores obtained values next to the metric map* `SHOW` controls whether images are displayed in the notebook<jupyter_code>import os
import sys
from glob import glob
from tqdm.auto import tqdm
from copy import copy

sys.path.append('..')
from seismiqb import SeismicGeometry, GeometryMetrics, plot_image, save_point_cloud
# Cubes to evaluate
paths = sorted(glob('/data/seismic_data/seismic_interpretation/CUBE_*/amp*.hdf5'))
[print(path) for path in paths]

# Metrics to compute
# metrics = GeometryMetrics.AVAILABLE_METRICS
metrics = ['support_hellinger', 'support_corrs',]
# Directory for all the files to be saved in
# For each cube, a separate dir is created inside
script_dir = 'cube_metrics'
try: os.mkdir(script_dir)
except FileExistsError: pass


# Parameters for `local` metrics like `local_correlation`
local_kwargs = {
    'agg': None,
    'kernel_size': 3,
    'reduce_func': 'mean',
}

# Parameters for `support` metrics like `support_hellinger`
support_kwargs = {
    'agg': 'mean',
    'supports': 20,
}

# Parameters of drawing
plot_kwargs = {
    'cmap': 'Reds',
    'figsize': (15, 15),
}


# Whether file names contain info about parameters of metric computation
ADD_PREFIX = True

# Whether to save point clouds of metrics right next to the images
SAVE_POINT_CLOUDS = False

# Whether to log progress/draw images to std.out
SHOW = True
%%time
for path_cube in tqdm(paths, ncols=1100):
    
    geometry = SeismicGeometry(path_cube)
    gm = GeometryMetrics(geometry)
    
    cube_dir = '/'.join((script_dir, geometry.short_name))
    try: os.mkdir(cube_dir)
    except FileExistsError: pass
    
    if SHOW:
        print('▆'*130); print('▆'*130);
        print(geometry)
    
    computed_metrics = []
    for metric_name in tqdm(metrics, ncols=600):        
        save_path = '/'.join((cube_dir, metric_name))
        kwargs = copy(local_kwargs) if metric_name.startswith('local') else copy(support_kwargs)

        if ADD_PREFIX:
            save_path += '@'
            save_path += '|'.join([':'.join([key,str(value)])
                                   for key, value in kwargs.items()])
        
        metric = gm.evaluate(metric_name, **kwargs,
                             plot=True, show_plot=SHOW, plot_kwargs=plot_kwargs,
                             savepath=save_path + '.png')
        computed_metrics.append(metric)
        
        if SAVE_POINT_CLOUDS:
            save_point_cloud(metric, save_path, geometry)

            
    save_path = '/'.join((cube_dir, 'quality_map'))
    if ADD_PREFIX:
        save_path += '@'
        save_path += '|'.join([':'.join([key,str(value)])
                               for key, value in kwargs.items()])
    quality_map = gm.evaluate('quality_map', quantiles=[0.1], agg=None,
                              computed_metrics=computed_metrics,
                              plot=True, show_plot=SHOW, plot_kwargs=plot_kwargs,
                              savepath=save_path + '.png')
    plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10066934404283802, 'fraction_numerical': 0.006157965194109772, 'mean_word_length': 2.9787007454739083, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '655859', 'license': 'Apache-2.0', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1102, 'n_words': 328, 'path': 'datasets/03_Cube_metrics.ipynb', 'repo_name': 'gazprom-neft/seismiqb', 'size': '3735'}