<commit_before>import pytest
import abc
import os
from glob import glob
import xarray as xr
import numpy as np
from climt import HeldSuarez, DataArray

cache_folder = os.path.join(
    os.path.dirname(os.path.realpath(__file__)), 'cached_component_output')


def cache_dictionary(dictionary, filename):
    dataset = xr.Dataset(dictionary)
    dataset.to_netcdf(filename, engine='scipy')


def load_dictionary(filename):
    dataset = xr.open_dataset(filename)
    return dict(dataset.data_vars)


class ComponentBase(object):

    @abc.abstractmethod
    def get_input_state(self):
        pass

    @abc.abstractmethod
    def get_component_instance(self):
        pass

    def get_cached_output(self):
        cache_filename_list = sorted(glob(
            os.path.join(
                cache_folder,
                '{}-*.cache'.format(
                    self.__class__.__name__))))
        if len(cache_filename_list) > 0:
            return_list = []
            for filename in cache_filename_list:
                return_list.append(load_dictionary(filename))
            return return_list
        else:
            return None

    def cache_output(self, output):
        for i in range(len(output)):
            cache_filename = os.path.join(
                cache_folder, '{}-{}.cache'.format(self.__class__.__name__, i))
            cache_dictionary(output[i], cache_filename)

    def compare_outputs(self, current, cached):
        for i in range(len(current)):
            for key in current[i].keys():
                assert np.all(current[i][key] == cached[i][key])<commit_msg>Updated compare_output for component testing<commit_after>import pytest
import abc
import os
from glob import glob
import xarray as xr
import numpy as np
from climt import HeldSuarez, DataArray

cache_folder = os.path.join(
    os.path.dirname(os.path.realpath(__file__)), 'cached_component_output')


def cache_dictionary(dictionary, filename):
    dataset = xr.Dataset(dictionary)
    dataset.to_netcdf(filename, engine='scipy')


def load_dictionary(filename):
    dataset = xr.open_dataset(filename, engine='scipy')
    return dict(dataset.data_vars)


class ComponentBase(object):

    @abc.abstractmethod
    def get_input_state(self):
        pass

    @abc.abstractmethod
    def get_component_instance(self):
        pass

    def get_cached_output(self):
        cache_filename_list = sorted(glob(
            os.path.join(
                cache_folder,
                '{}-*.cache'.format(
                    self.__class__.__name__))))
        if len(cache_filename_list) > 0:
            return_list = []
            for filename in cache_filename_list:
                return_list.append(load_dictionary(filename))
            return return_list
        else:
            return None

    def cache_output(self, output):
        for i in range(len(output)):
            cache_filename = os.path.join(
                cache_folder, '{}-{}.cache'.format(self.__class__.__name__, i))
            cache_dictionary(output[i], cache_filename)

    def compare_outputs(self, current, cached):
        for i in range(len(current)):
            for key in current[i].keys():
                assert np.all(current[i][key].values == cached[i][key].values)
                for attr in current[i][key].attrs:
                    assert current[i][key].attrs[attr] == cached[i][key].attrs[attr]
                for attr in cached[i][key].attrs:
                    assert attr in current[i][key].attrs
                assert current[i][key].dims == cached[i][key].dims
{'fraction_non_alphanumeric': 0.104008971124194, 'fraction_numerical': 0.000560695262125035, 'mean_word_length': 2.487781036168133, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5737452', 'n_tokens_mistral': 1045, 'n_tokens_neox': 1001, 'n_words': 224}