<commit_before>#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
from .... import core
from ....framework import IrGraph
from ....framework import _get_paddle_place

__all__ = ['Quant2Int8MkldnnPass']

OpRole = core.op_proto_and_checker_maker.OpRole


class Quant2Int8MkldnnPass(object):
    """
    Transform a quant model IrGraph into MKL-DNN supported INT8 IrGraph.
    The pass consists of the following transformations:
        1. gather scale values from fake quantize/dequantize operators,
        2. extract FP32 inference model graph from the quant graph, i.e.
            a.  remove fake quantize/dequantize operators,
            b.  dequantize conv2d and mul's weights,
        3. optimize the FP32 graph using standard FP32 optimization fuses
            (e.g. `conv2d`+`bn` -> `conv2d`),
        4. quantize the optimized FP32 graph using standard INT8v2 quantization
            passes (`cpu_quantize_pass`, `cpu_quantize_squash_pass`).
    """

    def __init__(self,
                 _ops_to_quantize,
                 _op_ids_to_skip=None,
                 _scope=None,
                 _place=None,
                 _core=None,
                 _debug=False):
        self._scope = _scope
        self._place = _get_paddle_place(_place)
        self._core = _core
        self._debug = _debug
        self._fake_quantize_types = [
            'fake_quantize_moving_average_abs_max',
            'fake_quantize_range_abs_max',
        ]
        self._fake_dequantize_types = [
            'fake_dequantize_max_abs', 'fake_channel_wise_dequantize_max_abs'
        ]
        self._fake_quantize_dequantize_types = [
            'fake_quantize_dequantize_abs_max',
            'fake_quantize_dequantize_moving_average_abs_max',
            'fake_channel_wise_quantize_dequantize_abs_max'
        ]
        self._ops_to_quantize = _ops_to_quantize
        self._op_ids_to_skip = _op_ids_to_skip if _op_ids_to_skip is not None else set(
            [-1])
        self._scale_immutable_ops = [
            'transpose2', 'reshape2', 'pool2d', 'scale'
        ]
        self._conv_ops = ['conv2d', 'depthwise_conv2d']
        self._pool_ops = ['pool2d']
        self._mul_ops = ['mul']
        self._fc_ops = ['fc']
        self._relu_ops = ['relu', 'relu6']
        self._matmul_ops = ['matmul']
        self._gru_ops = ['fusion_gru', 'multi_gru']
        self._weight_thresholds = {}
        # Collect the Input and Output sclaes from Fake quant models
        self._var_quant_scales = {}
        self._max_range = {}
        self._s8_max = 127
        self._pass_idx = 0
        self._pass_group = 'int8'

    def apply(self, graph):
        assert isinstance(graph,
                          IrGraph), 'graph must be the instance of IrGraph.'

        self._reset_pass_idx_and_group('int8')
        graph = self._label_skip_quantized_op(graph)
        graph = self._gather_weight_thresholds_from_fake(graph)
        graph = self._gather_output_scales_from_attr(graph)
        graph = self._gather_input_scales_from_fake(graph)
        graph = self._remove_fake_ops(graph)
        graph = self._dequantize_weights(graph)
        graph = self._optimize_fp32_graph(graph)
        graph = self._compute_weight_scales(graph)
        graph = self._update_relu_output_scales(graph)
        graph = self._propagate_scales(graph)
        graph = self._quantize_fp32_graph(graph)
        graph = self._final_optimizations(graph)
        graph = self._cleanup(graph)
        return graph

    def prepare_and_optimize_fp32(self, graph):
        assert isinstance(graph,
                          IrGraph), 'graph must be the instance of IrGraph.'

        self._reset_pass_idx_and_group('fp32')
        graph = self._optimize_fp32_graph(graph)
        graph = self._final_optimizations(graph)
        graph = self._cleanup(graph)
        return graph

    def _reset_pass_idx_and_group(self, group):
        self._pass_idx = 0
        self._pass_group = group

    def _convert_scale2tensor(self, scale):
        tensor = core.LoDTensor()
        tensor.set(scale, core.CPUPlace())
        return tensor

    def _is_quantizing_all_ops(self):
        return len(self._ops_to_quantize) == 0

    def _is_any_of_op_types_in_graph(self, op_types, graph):
        return any(op.name() in op_types for op in graph.all_op_nodes())

    def _is_any_of_op_types_quantized(self, op_types, graph):
        return self._is_any_of_op_types_in_graph(
            op_types, graph) and (self._is_quantizing_all_ops() or
                                  any(op_type in self._ops_to_quantize
                                      for op_type in op_types))

    def _is_conv_quantized(self, graph):
        return self._is_any_of_op_types_quantized(self._conv_ops, graph)

    def _is_fc_quantized(self, graph):
        return self._is_any_of_op_types_quantized(self._fc_ops, graph)

    def _label_skip_quantized_op(self, graph):
        """
        For some ops(conv2d, depthwise_conv2d, mul, matml), find and label
        the skip quantized ops. cpu_quantize_placement_pass will use the
        label to identify it.
        For static models, the skip quantized ops have `skip_quant` attr.
        Therefore, it only needs to find and label the skip quantized ops for
        dygraph models, in which the quantized ops don't have `quantization_type`
        attr.
        """
        target_ops = self._conv_ops + self._mul_ops + self._matmul_ops
        for op_node in graph.all_op_nodes():
            if op_node.name() in target_ops and \
               not op_node.op().has_attr("quantization_type"):
                is_quantized_op = True
                for var_node in op_node.inputs:
                    for front_op_node in var_node.inputs:
                        if "fake_quantize_dequantize_" not in front_op_node.name(
                        ):
                            is_quantized_op = False
                if not is_quantized_op:
                    op_node.op()._set_attr("skip_quant", True)
        return graph

    def _gather_input_scales_from_fake(self, graph):
        def _add_scale_for_vars(var_names, use_unsigned_int, lod_tensor):
            scales = self._var_quant_scales
            for var_name in var_names:
                scales[var_name] = (use_unsigned_int, lod_tensor)

        # fake_quantize_dequantize_abs_max doesn't have scale value
        fake_ops = ['fake_quantize_dequantize_moving_average_abs_max']
        fake_ops.extend(self._fake_quantize_types)
<commit_msg>Fix bug of supporting channelwise dygraph quantized model, test=develop (#30531)<commit_after>#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
from .... import core
from ....framework import IrGraph
from ....framework import _get_paddle_place

__all__ = ['Quant2Int8MkldnnPass']

OpRole = core.op_proto_and_checker_maker.OpRole


class Quant2Int8MkldnnPass(object):
    """
    Transform a quant model IrGraph into MKL-DNN supported INT8 IrGraph.
    The pass consists of the following transformations:
        1. gather scale values from fake quantize/dequantize operators,
        2. extract FP32 inference model graph from the quant graph, i.e.
            a.  remove fake quantize/dequantize operators,
            b.  dequantize conv2d and mul's weights,
        3. optimize the FP32 graph using standard FP32 optimization fuses
            (e.g. `conv2d`+`bn` -> `conv2d`),
        4. quantize the optimized FP32 graph using standard INT8v2 quantization
            passes (`cpu_quantize_pass`, `cpu_quantize_squash_pass`).
    """

    def __init__(self,
                 _ops_to_quantize,
                 _op_ids_to_skip=None,
                 _scope=None,
                 _place=None,
                 _core=None,
                 _debug=False):
        self._scope = _scope
        self._place = _get_paddle_place(_place)
        self._core = _core
        self._debug = _debug
        self._fake_quantize_types = [
            'fake_quantize_moving_average_abs_max',
            'fake_quantize_range_abs_max',
        ]
        self._fake_dequantize_types = [
            'fake_dequantize_max_abs', 'fake_channel_wise_dequantize_max_abs'
        ]
        self._fake_quantize_dequantize_types = [
            'fake_quantize_dequantize_abs_max',
            'fake_quantize_dequantize_moving_average_abs_max',
            'fake_channel_wise_quantize_dequantize_abs_max'
        ]
        self._ops_to_quantize = _ops_to_quantize
        self._op_ids_to_skip = _op_ids_to_skip if _op_ids_to_skip is not None else set(
            [-1])
        self._scale_immutable_ops = [
            'transpose2', 'reshape2', 'pool2d', 'scale'
        ]
        self._conv_ops = ['conv2d', 'depthwise_conv2d']
        self._pool_ops = ['pool2d']
        self._mul_ops = ['mul']
        self._fc_ops = ['fc']
        self._relu_ops = ['relu', 'relu6']
        self._matmul_ops = ['matmul']
        self._gru_ops = ['fusion_gru', 'multi_gru']
        self._weight_thresholds = {}
        # Collect the Input and Output sclaes from Fake quant models
        self._var_quant_scales = {}
        self._max_range = {}
        self._s8_max = 127
        self._pass_idx = 0
        self._pass_group = 'int8'

    def apply(self, graph):
        assert isinstance(graph,
                          IrGraph), 'graph must be the instance of IrGraph.'

        self._reset_pass_idx_and_group('int8')
        graph = self._label_skip_quantized_op(graph)
        graph = self._gather_weight_thresholds_from_fake(graph)
        graph = self._gather_output_scales_from_attr(graph)
        graph = self._gather_input_scales_from_fake(graph)
        graph = self._remove_fake_ops(graph)
        graph = self._dequantize_weights(graph)
        graph = self._optimize_fp32_graph(graph)
        graph = self._compute_weight_scales(graph)
        graph = self._update_relu_output_scales(graph)
        graph = self._propagate_scales(graph)
        graph = self._quantize_fp32_graph(graph)
        graph = self._final_optimizations(graph)
        graph = self._cleanup(graph)
        return graph

    def prepare_and_optimize_fp32(self, graph):
        assert isinstance(graph,
                          IrGraph), 'graph must be the instance of IrGraph.'

        self._reset_pass_idx_and_group('fp32')
        graph = self._optimize_fp32_graph(graph)
        graph = self._final_optimizations(graph)
        graph = self._cleanup(graph)
        return graph

    def _reset_pass_idx_and_group(self, group):
        self._pass_idx = 0
        self._pass_group = group

    def _convert_scale2tensor(self, scale):
        tensor = core.LoDTensor()
        tensor.set(scale, core.CPUPlace())
        return tensor

    def _is_quantizing_all_ops(self):
        return len(self._ops_to_quantize) == 0

    def _is_any_of_op_types_in_graph(self, op_types, graph):
        return any(op.name() in op_types for op in graph.all_op_nodes())

    def _is_any_of_op_types_quantized(self, op_types, graph):
        return self._is_any_of_op_types_in_graph(
            op_types, graph) and (self._is_quantizing_all_ops() or
                                  any(op_type in self._ops_to_quantize
                                      for op_type in op_types))

    def _is_conv_quantized(self, graph):
        return self._is_any_of_op_types_quantized(self._conv_ops, graph)

    def _is_fc_quantized(self, graph):
        return self._is_any_of_op_types_quantized(self._fc_ops, graph)

    def _label_skip_quantized_op(self, graph):
        """
        For some ops(conv2d, depthwise_conv2d, mul, matml), find and label
        the skip quantized ops. cpu_quantize_placement_pass will use the
        label to identify it.
        For static models, the skip quantized ops have `skip_quant` attr.
        Therefore, it only needs to find and label the skip quantized ops for
        dygraph models, in which the quantized ops don't have `quantization_type`
        attr.
        """
        target_ops = self._conv_ops + self._mul_ops + self._matmul_ops
        for op_node in graph.all_op_nodes():
            if op_node.name() in target_ops and \
               not op_node.op().has_attr("quantization_type"):
                is_quantized_op = True
                for var_node in op_node.inputs:
                    for front_op_node in var_node.inputs:
                        if "quantize_dequantize" not in front_op_node.name():
                            is_quantized_op = False
                if not is_quantized_op:
                    op_node.op()._set_attr("skip_quant", True)
        return graph

    def _gather_input_scales_from_fake(self, graph):
        def _add_scale_for_vars(var_names, use_unsigned_int, lod_tensor):
            scales = self._var_quant_scales
            for var_name in var_names:
                scales[var_name] = (use_unsigned_int, lod_tensor)

        # fake_quantize_dequantize_abs_max doesn't have scale value
        fake_ops = ['fake_quantize_dequantize_moving_average_abs_max']
        fake_ops.extend(self._fake_quantize_types)

{'fraction_non_alphanumeric': 0.07648466084998592, 'fraction_numerical': 0.008795384182381086, 'mean_word_length': 2.4589924555853004, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1374924', 'n_tokens_mistral': 4406, 'n_tokens_neox': 4246, 'n_words': 1025}