<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import collections
import numbers
import sys

from onnx.onnx_pb2 import \
    AttributeProto, TensorProto, NodeProto, GraphProto, IR_VERSION
import onnx.onnx_cpp2py_export as C

def make_node(
        op_type, inputs, outputs,
        name=None, **kwargs):
    node = NodeProto()
    node.op_type = op_type
    node.input.extend(inputs)
    node.output.extend(outputs)
    if name:
        node.name = name
    if kwargs:
        node.attribute.extend(
            make_attribute(key, value)
            for key, value in kwargs.items())
    return node


def make_graph(nodes, name, inputs, outputs, initializer=[]):
    graph = GraphProto()
    # Touch graph.ir_version so it is stored as the version from which it is
    # generated.
    graph.ir_version = IR_VERSION
    graph.node.extend(nodes)
    graph.name = name
    graph.input.extend(inputs)
    graph.output.extend(outputs)
    graph.initializer.extend(initializer)
    return graph


def make_tensor(name, data_type, dims, vals, raw=False):
    '''
    Make a TensorProto with specified arguments.  If raw is False, this
    function will choose the corresponding proto field to store the
    values based on data_type. If raw is True, use "raw_data" proto
    field to store the values, and values should be of type bytes in
    this case.
    '''
    tensor = TensorProto()
    tensor.data_type = data_type
    tensor.name = name

    if data_type == TensorProto.STRING:
        assert not raw, "Can not use raw_data to store string type"
        tensor.string_data.extend(vals)
    elif data_type in [TensorProto.UINT8,
                       TensorProto.INT8,
                       TensorProto.UINT16,
                       TensorProto.INT16,
                       TensorProto.INT32,
                       TensorProto.FLOAT16,
                       TensorProto.BOOL,
                       TensorProto.FLOAT]:
        if raw:
            tensor.raw_data = vals
        else:
            if data_type == TensorProto.FLOAT:
                tensor.float_data.extend(vals)
            elif data_type == TensorProto.INT64:
                tensor.int64_data.extend(vals)
            else:
                tensor.int32_data.extend(vals)
    else:
        raise RuntimeError('Unrecognized data_type: {}'.format(data_type))
    tensor.dims.extend(dims)
    return tensor


def _to_bytes_or_false(val):
    """An internal graph to convert the input to a bytes or to False.

    The criteria for conversion is as follows and should be python 2 and 3
    compatible:
    - If val is py2 str or py3 bytes: return bytes
    - If val is py2 unicode or py3 str: return val.decode('ascii')
    - Otherwise, return False
    """
    if isinstance(val, bytes):
        return val
    else:
        try:
            return val.encode('ascii')
        except AttributeError:
            return False


def make_attribute(key, value):
    """Makes an AttributeProto based on the value type."""
    attr = AttributeProto()
    attr.name = key

    is_iterable = isinstance(value, collections.Iterable)
    bytes_or_false = _to_bytes_or_false(value)
    # First, singular cases
    # float
    if isinstance(value, float):
        attr.f = value
    # integer
    elif isinstance(value, numbers.Integral):
        attr.i = value
    # string
    elif bytes_or_false:
        attr.s = bytes_or_false
    elif isinstance(value, TensorProto):
        attr.t.CopyFrom(value)
    elif isinstance(value, GraphProto):
        attr.g.CopyFrom(value)
    # third, iterable cases
    elif is_iterable:
        byte_array = [_to_bytes_or_false(v) for v in value]
        if all(isinstance(v, float) for v in value):
            attr.floats.extend(value)
        elif all(isinstance(v, numbers.Integral) for v in value):
            attr.ints.extend(value)
        elif all(byte_array):
            attr.strings.extend(byte_array)
        elif all(isinstance(v, TensorProto) for v in value):
            attr.tensors.extend(value)
        elif all(isinstance(v, GraphProto) for v in value):
            attr.graphs.extend(value)
        else:
            raise ValueError(
                "You passed in an iterable attribute but I cannot figure out "
                "its applicable type.")
    else:
        raise ValueError(
            'Value "{}" is not valid attribute data type.'.format(value))
    return attr


def is_attribute_legal(attr):
    """Checks if an AttributeProto is legal.

    Inputs:
        arg: an AttributeProto object.
    Returns:
        bool.
    """
    if not isinstance(attr, AttributeProto):
        raise RuntimeError("You cannot pass an object that is not AttributeProto.")
    return C.is_attribute_legal(attr.SerializeToString())


def _sanitize_str(s):
    if isinstance(s, text_type):
        sanitized = s
    elif isinstance(s, binary_type):
        sanitized = s.decode('ascii', errors='ignore')
    else:
        sanitized = str(s)
    if len(sanitized) < 64:
        return sanitized
    else:
        return sanitized[:64] + '...<+len=%d>' % (len(sanitized) - 64)


def printable_attribute(attr):
    content = []
    content.append(attr.name)
    content.append("=")
    if attr.HasField("f"):
        content.append(str(attr.f))
    elif attr.HasField("i"):
        content.append('{}'.format(attr.i))
    elif attr.HasField("s"):
        content.append(_sanitize_str(attr.s))
    elif attr.HasField("t"):
        content.append("<Tensor>")
    elif attr.floats:
        content.append(str(list(attr.floats)))
    elif attr.ints:
        content.append('[' + ', '.join(map(lambda i: '{}'.format(i), attr.ints)) + ']')
    elif attr.strings:
        content.append(str(list(map(_sanitize_str, attr.strings))))
    elif attr.tensors:
        content.append("[<Tensor>, ...]")
    else:
        content.append("<Unknown>")
    return ' '.join(content)


def printable_node(node, prefix=''):
    content = []
    if len(node.output):<commit_msg>Make printable_graph more deterministic across Python versions.<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import collections
import numbers
import sys

from onnx.onnx_pb2 import \
    AttributeProto, TensorProto, NodeProto, GraphProto, IR_VERSION
import onnx.onnx_cpp2py_export as C

def make_node(
        op_type, inputs, outputs,
        name=None, **kwargs):
    node = NodeProto()
    node.op_type = op_type
    node.input.extend(inputs)
    node.output.extend(outputs)
    if name:
        node.name = name
    if kwargs:
        node.attribute.extend(
            make_attribute(key, value)
            for key, value in kwargs.items())
    return node


def make_graph(nodes, name, inputs, outputs, initializer=[]):
    graph = GraphProto()
    # Touch graph.ir_version so it is stored as the version from which it is
    # generated.
    graph.ir_version = IR_VERSION
    graph.node.extend(nodes)
    graph.name = name
    graph.input.extend(inputs)
    graph.output.extend(outputs)
    graph.initializer.extend(initializer)
    return graph


def make_tensor(name, data_type, dims, vals, raw=False):
    '''
    Make a TensorProto with specified arguments.  If raw is False, this
    function will choose the corresponding proto field to store the
    values based on data_type. If raw is True, use "raw_data" proto
    field to store the values, and values should be of type bytes in
    this case.
    '''
    tensor = TensorProto()
    tensor.data_type = data_type
    tensor.name = name

    if data_type == TensorProto.STRING:
        assert not raw, "Can not use raw_data to store string type"
        tensor.string_data.extend(vals)
    elif data_type in [TensorProto.UINT8,
                       TensorProto.INT8,
                       TensorProto.UINT16,
                       TensorProto.INT16,
                       TensorProto.INT32,
                       TensorProto.FLOAT16,
                       TensorProto.BOOL,
                       TensorProto.FLOAT]:
        if raw:
            tensor.raw_data = vals
        else:
            if data_type == TensorProto.FLOAT:
                tensor.float_data.extend(vals)
            elif data_type == TensorProto.INT64:
                tensor.int64_data.extend(vals)
            else:
                tensor.int32_data.extend(vals)
    else:
        raise RuntimeError('Unrecognized data_type: {}'.format(data_type))
    tensor.dims.extend(dims)
    return tensor


def _to_bytes_or_false(val):
    """An internal graph to convert the input to a bytes or to False.

    The criteria for conversion is as follows and should be python 2 and 3
    compatible:
    - If val is py2 str or py3 bytes: return bytes
    - If val is py2 unicode or py3 str: return val.decode('ascii')
    - Otherwise, return False
    """
    if isinstance(val, bytes):
        return val
    else:
        try:
            return val.encode('ascii')
        except AttributeError:
            return False


def make_attribute(key, value):
    """Makes an AttributeProto based on the value type."""
    attr = AttributeProto()
    attr.name = key

    is_iterable = isinstance(value, collections.Iterable)
    bytes_or_false = _to_bytes_or_false(value)
    # First, singular cases
    # float
    if isinstance(value, float):
        attr.f = value
    # integer
    elif isinstance(value, numbers.Integral):
        attr.i = value
    # string
    elif bytes_or_false:
        attr.s = bytes_or_false
    elif isinstance(value, TensorProto):
        attr.t.CopyFrom(value)
    elif isinstance(value, GraphProto):
        attr.g.CopyFrom(value)
    # third, iterable cases
    elif is_iterable:
        byte_array = [_to_bytes_or_false(v) for v in value]
        if all(isinstance(v, float) for v in value):
            attr.floats.extend(value)
        elif all(isinstance(v, numbers.Integral) for v in value):
            attr.ints.extend(value)
        elif all(byte_array):
            attr.strings.extend(byte_array)
        elif all(isinstance(v, TensorProto) for v in value):
            attr.tensors.extend(value)
        elif all(isinstance(v, GraphProto) for v in value):
            attr.graphs.extend(value)
        else:
            raise ValueError(
                "You passed in an iterable attribute but I cannot figure out "
                "its applicable type.")
    else:
        raise ValueError(
            'Value "{}" is not valid attribute data type.'.format(value))
    return attr


def is_attribute_legal(attr):
    """Checks if an AttributeProto is legal.

    Inputs:
        arg: an AttributeProto object.
    Returns:
        bool.
    """
    if not isinstance(attr, AttributeProto):
        raise RuntimeError("You cannot pass an object that is not AttributeProto.")
    return C.is_attribute_legal(attr.SerializeToString())


def _sanitize_str(s):
    if isinstance(s, text_type):
        sanitized = s
    elif isinstance(s, binary_type):
        sanitized = s.decode('ascii', errors='ignore')
    else:
        sanitized = str(s)
    if len(sanitized) < 64:
        return sanitized
    else:
        return sanitized[:64] + '...<+len=%d>' % (len(sanitized) - 64)


def printable_attribute(attr):
    content = []
    content.append(attr.name)
    content.append("=")
    def str_float(f):
        # NB: Different Python versions print different numbers of trailing
        # decimals, specifying this explicitly keeps it consistent for all
        # versions
        return '{:.15g}'.format(f)
    def str_int(i):
        # NB: In Python 2, longs will repr() as '2L', which is ugly and
        # unnecessary.  Explicitly format it to keep it consistent.
        return '{:d}'.format(i)
    def str_str(s):
        return repr(s)
    def str_list(str_elem, xs):
        return '[' + ', '.join(map(str_elem, xs)) + ']'
    if attr.HasField("f"):
        content.append(str_float(attr.f))
    elif attr.HasField("i"):
        content.append(str_int(attr.i))
    elif attr.HasField("s"):
        # TODO: Bit nervous about Python 2 / Python 3 determinism implications
        content.append(repr(_sanitize_str(attr.s)))
    elif attr.HasField("t"):
        content.append("<Tensor>")
    elif attr.floats:
        content.append(str_list(str_float, attr.floats))
    elif attr.ints:
        content.append(str_list(str_int, attr.ints))
    elif attr.strings:
        # TODO: Bit nervous about Python 2 / Python 3 determinism implications
        content.append(str(list(map(_sanitize_str, attr.strings))))
    elif attr.tensors:
        content.append("[<Tensor>, ...]")
    else:
        content.append("<Unknown>")
    return ' '.join(content)


def printable_node(node, prefix=''):
    content = []
    if len(node.output):
{'fraction_non_alphanumeric': 0.09672676623075137, 'fraction_numerical': 0.005261936082952875, 'mean_word_length': 2.395691014188124, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '567440', 'n_tokens_mistral': 3811, 'n_tokens_neox': 3715, 'n_words': 1073}