<commit_before>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

from . import core
import numpy as np
import os
import six
from six.moves import zip, range, xrange
import multiprocessing
import warnings

from .framework import Variable, default_main_program, _current_expected_place
from .framework import _cpu_num, _cuda_ids
__all__ = ['DataFeeder']


def convert_dtype(dtype):
    if isinstance(dtype, core.VarDesc.VarType):
        if dtype == core.VarDesc.VarType.BOOL:
            return 'bool'
        elif dtype == core.VarDesc.VarType.FP16:
            return 'float16'
        elif dtype == core.VarDesc.VarType.FP32:
            return 'float32'
        elif dtype == core.VarDesc.VarType.FP64:
            return 'float64'
        elif dtype == core.VarDesc.VarType.INT8:
            return 'int8'
        elif dtype == core.VarDesc.VarType.INT16:
            return 'int16'
        elif dtype == core.VarDesc.VarType.INT32:
            return 'int32'
        elif dtype == core.VarDesc.VarType.INT64:
            return 'int64'
        elif dtype == core.VarDesc.VarType.UINT8:
            return 'uint8'
    elif isinstance(dtype, type):
        if dtype in [
                np.bool, np.float16, np.float32, np.float64, np.int8, np.int16,
                np.int32, np.int64, np.uint8
        ]:
            return dtype.__name__
    else:
        if dtype in [
                'bool', 'float16', 'float32', 'float64', 'int8', 'int16',
                'int32', 'int64', 'uint8', u'bool', u'float16', u'float32',
                u'float64', u'int8', u'int16', u'int32', u'int64', u'uint8'
        ]:
            # this code is a little bit dangerous, since error could happen
            # when casting no-asci code to str in python2.
            # but since the set itself is limited, so currently, it is good.
            # however, jointly supporting python2 and python3, (as well as python4 maybe)
            # may still be a long-lasting problem.
            return str(dtype)

    raise ValueError(
        "dtype must be any of [bool, float16, float32, float64, int8, int16, "
        "int32, int64, uint8]")


def check_variable_and_dtype(input,
                             input_name,
                             expected_dtype,
                             op_name,
                             extra_message=''):
    check_type(input, input_name, Variable, op_name, extra_message)
    check_dtype(input.dtype, input_name, expected_dtype, op_name, extra_message)


def check_type(input, input_name, expected_type, op_name, extra_message=''):
    if not isinstance(input, expected_type):
        raise TypeError(
            "The type of '%s' in %s must be %s, but received %s. %s" %
            (input_name, op_name, expected_type, type(input), extra_message))


def check_dtype(input_dtype,
                input_name,
                expected_dtype,
                op_name,
                extra_message=''):
    if convert_dtype(input_dtype) in ['float16']:
        warnings.warn(
            "The data type of '%s' in %s only support float16 in GPU now. %s" %
            (input_name, op_name, extra_message))
    if convert_dtype(input_dtype) not in expected_dtype:
        raise TypeError(
            "The data type of '%s' in %s must be %s, but received %s. %s" %
            (input_name, op_name, expected_dtype, convert_dtype(input_dtype),
             extra_message))
<commit_msg>skip type/dtype check in dygraph mode, test=develop (#22915)<commit_after>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

from . import core
import numpy as np
import os
import six
from six.moves import zip, range, xrange
import multiprocessing
import warnings

from .framework import Variable, default_main_program, _current_expected_place, in_dygraph_mode
from .framework import _cpu_num, _cuda_ids
__all__ = ['DataFeeder']


def convert_dtype(dtype):
    if isinstance(dtype, core.VarDesc.VarType):
        if dtype == core.VarDesc.VarType.BOOL:
            return 'bool'
        elif dtype == core.VarDesc.VarType.FP16:
            return 'float16'
        elif dtype == core.VarDesc.VarType.FP32:
            return 'float32'
        elif dtype == core.VarDesc.VarType.FP64:
            return 'float64'
        elif dtype == core.VarDesc.VarType.INT8:
            return 'int8'
        elif dtype == core.VarDesc.VarType.INT16:
            return 'int16'
        elif dtype == core.VarDesc.VarType.INT32:
            return 'int32'
        elif dtype == core.VarDesc.VarType.INT64:
            return 'int64'
        elif dtype == core.VarDesc.VarType.UINT8:
            return 'uint8'
    elif isinstance(dtype, type):
        if dtype in [
                np.bool, np.float16, np.float32, np.float64, np.int8, np.int16,
                np.int32, np.int64, np.uint8
        ]:
            return dtype.__name__
    else:
        if dtype in [
                'bool', 'float16', 'float32', 'float64', 'int8', 'int16',
                'int32', 'int64', 'uint8', u'bool', u'float16', u'float32',
                u'float64', u'int8', u'int16', u'int32', u'int64', u'uint8'
        ]:
            # this code is a little bit dangerous, since error could happen
            # when casting no-asci code to str in python2.
            # but since the set itself is limited, so currently, it is good.
            # however, jointly supporting python2 and python3, (as well as python4 maybe)
            # may still be a long-lasting problem.
            return str(dtype)

    raise ValueError(
        "dtype must be any of [bool, float16, float32, float64, int8, int16, "
        "int32, int64, uint8]")


def check_variable_and_dtype(input,
                             input_name,
                             expected_dtype,
                             op_name,
                             extra_message=''):
    check_type(input, input_name, Variable, op_name, extra_message)
    check_dtype(input.dtype, input_name, expected_dtype, op_name, extra_message)


def check_type(input, input_name, expected_type, op_name, extra_message=''):
    # NOTE [ Why skip dynamic graph check ]:
    # 1. If the input type / dtype of a layer is wrong, it will be reported
    # directly on that line. User can easily print the relevant information
    # on which line. It is easier to debug, so there is no need to check
    # in dynamic graph mode.
    # 2. Performance considerations. Because these checks are executed at
    # each step in dynamic graph mode, it will bring a heavy performance burden.
    if in_dygraph_mode():
        return
    if not isinstance(input, expected_type):
        raise TypeError(
            "The type of '%s' in %s must be %s, but received %s. %s" %
            (input_name, op_name, expected_type, type(input), extra_message))


def check_dtype(input_dtype,
                input_name,
                expected_dtype,
                op_name,
                extra_message=''):
    # See NOTE [ Why skip dynamic graph check ]
    if in_dygraph_mode():
        return
    if convert_dtype(input_dtype) in ['float16']:
        warnings.warn(
            "The data type of '%s' in %s only support float16 in GPU now. %s" %
            (input_name, op_name, extra_message))
    if convert_dtype(input_dtype) not in expected_dtype:
        raise TypeError(
            "The data type of '%s' in %s must be %s, but received %s. %s" %
            (input_name, op_name, expected_dtype, convert_dtype(input_dtype),
             extra_message))

{'fraction_non_alphanumeric': 0.09012430939226519, 'fraction_numerical': 0.02382596685082873, 'mean_word_length': 2.330394787274818, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5430775', 'n_tokens_mistral': 2591, 'n_tokens_neox': 2507, 'n_words': 854}