<commit_before>import operator
import six
import logging
import os
import glob
import hashlib

# noinspection PyUnresolvedReferences
from six.moves import reduce
from avro.schema import UnionSchema, Schema, ArraySchema

if six.PY2:
    from avro.io import validate
else:
    from avro.io import Validate as validate

from rabix.common.util import sec_files_naming_conv, wrap_in_list, to_abspath
from rabix.expressions.evaluator import ExpressionEvaluator

log = logging.getLogger(__name__)


def intersect_dicts(d1, d2):
    return {k: v for k, v in six.iteritems(d1) if v == d2.get(k)}


def meta(path, inputs, eval, outputBinding):
    meta, result = outputBinding.get('metadata', {}), {}
    inherit = meta.pop('__inherit__', None)
    if inherit:
        src = inputs.get(inherit)
        if isinstance(src, list):
            result = reduce(intersect_dicts, [x.meta for x in src]) \
                if len(src) > 1 else src[0].meta
        else:
            result = src.meta
    result.update(**meta)
    for k, v in six.iteritems(result):
        result[k] = eval.resolve(v, context={"class": "File", "path": path})
    return result


def secondary_files(p, outputBinding, evaluator):
    secondaryFiles = []
    secFiles = wrap_in_list(evaluator.resolve(outputBinding.get('secondaryFiles', [])))
    for s in secFiles:
        path = sec_files_naming_conv(p, s)
        secondaryFiles.append({'path': to_abspath(path)})
    return secondaryFiles


class InputAdapter(object):
    def __init__(self, value, evaluator, schema, input_binding=None, key=''):
        self.evaluator = evaluator
        self.schema = schema
        self.adapter = input_binding or (
            isinstance(self.schema, Schema) and
            self.schema.props.get('inputBinding')
        )
        self.has_adapter = self.adapter is not None
        self.adapter = self.adapter or {}
        self.key = key
        if isinstance(self.schema, UnionSchema):
            for opt in self.schema.schemas:
                if validate(opt, value):
                    self.schema = opt
                    break
        self.value = evaluator.resolve(value)

    __str__ = lambda self: six.text_type(self.value)
    __repr__ = lambda self: 'InputAdapter(%s)' % self
    position = property(lambda self: (self.adapter.get('position', 0), self.key))<commit_msg>Evaluating expressions in inputBindings<commit_after>import operator
import six
import logging
import os
import glob
import hashlib

# noinspection PyUnresolvedReferences
from six.moves import reduce
from avro.schema import UnionSchema, Schema, ArraySchema

if six.PY2:
    from avro.io import validate
else:
    from avro.io import Validate as validate

from rabix.common.util import sec_files_naming_conv, wrap_in_list, to_abspath
from rabix.expressions.evaluator import ExpressionEvaluator

log = logging.getLogger(__name__)


def intersect_dicts(d1, d2):
    return {k: v for k, v in six.iteritems(d1) if v == d2.get(k)}


def meta(path, inputs, eval, outputBinding):
    meta, result = outputBinding.get('metadata', {}), {}
    inherit = meta.pop('__inherit__', None)
    if inherit:
        src = inputs.get(inherit)
        if isinstance(src, list):
            result = reduce(intersect_dicts, [x.meta for x in src]) \
                if len(src) > 1 else src[0].meta
        else:
            result = src.meta
    result.update(**meta)
    for k, v in six.iteritems(result):
        result[k] = eval.resolve(v, context={"class": "File", "path": path})
    return result


def secondary_files(p, outputBinding, evaluator):
    secondaryFiles = []
    secFiles = wrap_in_list(evaluator.resolve(outputBinding.get('secondaryFiles', [])))
    for s in secFiles:
        path = sec_files_naming_conv(p, s)
        secondaryFiles.append({'path': to_abspath(path)})
    return secondaryFiles


class InputAdapter(object):
    def __init__(self, value, evaluator, schema, input_binding=None, key=''):
        self.evaluator = evaluator
        self.schema = schema
        self.adapter = input_binding or (
            isinstance(self.schema, Schema) and
            self.schema.props.get('inputBinding')
        )
        self.has_adapter = self.adapter is not None
        self.adapter = self.adapter or {}
        self.key = key
        if isinstance(self.schema, UnionSchema):
            for opt in self.schema.schemas:
                if validate(opt, value):
                    self.schema = opt
                    break
        expr = self.adapter.get('valueFrom')
        json = value.to_dict() if hasattr(value, 'to_dict') else value
        self.value = evaluator.resolve(expr, json) if expr else value

    __str__ = lambda self: six.text_type(self.value)
    __repr__ = lambda self: 'InputAdapter(%s)' % self
    position = property(lambda self: (self.adapter.get('position', 0), self.key))
{'fraction_non_alphanumeric': 0.10627321502269914, 'fraction_numerical': 0.0033016921172100704, 'mean_word_length': 2.9057211925866238, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '312261', 'n_tokens_mistral': 1454, 'n_tokens_neox': 1430, 'n_words': 392}