<commit_before>import hashlib

from datapackage_pipelines.utilities.extended_json import json

from ..errors import SpecError
from .dependency_resolver import resolve_dependencies


class HashCalculator(object):

    def __init__(self):
        self.all_pipeline_ids = {}

    def calculate_hash(self, spec):

        cache_hash = None
        if spec.pipeline_id in self.all_pipeline_ids:
            message = 'Duplicate key {0} in {1}' \
                .format(spec.pipeline_id, spec.abspath)
            spec.errors.append(SpecError('Duplicate Pipeline Id', message))

        else:
            cache_hash = resolve_dependencies(spec, self.all_pipeline_ids)
            if len(spec.errors) > 0:
                return cache_hash

            for step in spec.pipeline_details['pipeline']:
                m = hashlib.md5()
                m.update(cache_hash.encode('ascii'))
                with open(step['executor'], 'rb') as f:
                    m.update(f.read())
                m.update(json.dumps(step, ensure_ascii=True, sort_keys=True)
                         .encode('ascii'))
                cache_hash = m.hexdigest()
                step['_cache_hash'] = cache_hash

            self.all_pipeline_ids[spec.pipeline_id] = spec

        spec.cache_hash = cache_hash<commit_msg>Fix error in error log<commit_after>import hashlib

from datapackage_pipelines.utilities.extended_json import json

from ..errors import SpecError
from .dependency_resolver import resolve_dependencies


class HashCalculator(object):

    def __init__(self):
        self.all_pipeline_ids = {}

    def calculate_hash(self, spec):

        cache_hash = None
        if spec.pipeline_id in self.all_pipeline_ids:
            message = 'Duplicate key {0} in {1}' \
                .format(spec.pipeline_id, spec.path)
            spec.errors.append(SpecError('Duplicate Pipeline Id', message))

        else:
            cache_hash = resolve_dependencies(spec, self.all_pipeline_ids)
            if len(spec.errors) > 0:
                return cache_hash

            for step in spec.pipeline_details['pipeline']:
                m = hashlib.md5()
                m.update(cache_hash.encode('ascii'))
                with open(step['executor'], 'rb') as f:
                    m.update(f.read())
                m.update(json.dumps(step, ensure_ascii=True, sort_keys=True)
                         .encode('ascii'))
                cache_hash = m.hexdigest()
                step['_cache_hash'] = cache_hash

            self.all_pipeline_ids[spec.pipeline_id] = spec

        spec.cache_hash = cache_hash
{'fraction_non_alphanumeric': 0.09615384615384616, 'fraction_numerical': 0.003076923076923077, 'mean_word_length': 2.0927467300832343, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4039378', 'n_tokens_mistral': 771, 'n_tokens_neox': 728, 'n_words': 151}