<commit_before># encoding: utf-8

"""
.. codeauthor:: <NAME> <<EMAIL>>
"""

from __future__ import absolute_import
from __future__ import unicode_literals

from six.moves import zip

from ._common import (
    DEFAULT_INF_VALUE,
    DEFAULT_NAN_VALUE,
    DEFAULT_STRICT_TYPE_MAPPING,
    default_bool_converter,
    default_datetime_converter,
)
from ._data_property import (
    DataProperty,
    ColumnDataProperty
)
from ._function import is_empty_sequence


class MissmatchProcessing(object):
    EXCEPTION = 1 << 1
    TRIM = 1 << 2
    EXTEND = 1 << 3


class PropertyExtractor(object):

    def __init__(self):
        self.header_list = []
        self.data_matrix = []
        self.min_padding_len = 0
        self.none_value = None
        self.inf_value = DEFAULT_INF_VALUE
        self.nan_value = DEFAULT_NAN_VALUE
        self.bool_converter = default_bool_converter
        self.datetime_converter = default_datetime_converter
        self.datetime_format_str = "%Y-%m-%dT%H:%M:%S%z"
        self.is_strict_type_mapping = dict(DEFAULT_STRICT_TYPE_MAPPING)
        self.east_asian_ambiguous_width = 1

        self.mismatch_processing = MissmatchProcessing.TRIM

    def extract_data_property_matrix(self):
        return [
            self.__extract_data_property_list(data_list)
            for data_list in self.data_matrix
        ]

    def extract_column_property_list(self):
        column_prop_list = self.__extract_header_data_prop_list()

        try:
            data_prop_matrix = self.extract_data_property_matrix()
        except TypeError:
            return column_prop_list

        for col_idx, col_prop_list in enumerate(zip(*data_prop_matrix)):
            try:
                column_prop_list[col_idx]
            except IndexError:
                if self.mismatch_processing == MissmatchProcessing.EXCEPTION:
                    raise ValueError(
                        "column not found: col-size={}, col-index={}".format(
                            len(column_prop_list), col_idx))

                if any([
                    self.mismatch_processing == MissmatchProcessing.EXTEND,
                    all([
                        self.mismatch_processing == MissmatchProcessing.TRIM,
                        is_empty_sequence(self.header_list),
                    ])
                ]):
                    column_prop_list.append(ColumnDataProperty(
                        min_padding_len=self.min_padding_len,
                        datetime_format_str=self.datetime_format_str,
                        east_asian_ambiguous_width=self.east_asian_ambiguous_width
                    ))
                elif self.mismatch_processing == MissmatchProcessing.TRIM:
                    # ignore columns that longer than header column
                    continue

            for prop in col_prop_list:
                column_prop_list[col_idx].update_body(prop)

        return column_prop_list

    def __extract_header_data_prop_list(self):
        header_prop_list = self.__extract_data_property_list(self.header_list)
        column_prop_list = []

        for header_prop in header_prop_list:
            column_prop = ColumnDataProperty(
                min_padding_len=self.min_padding_len,
                datetime_format_str=self.datetime_format_str,
                east_asian_ambiguous_width=self.east_asian_ambiguous_width
            )
            column_prop.update_header(header_prop)
            column_prop_list.append(column_prop)

        return column_prop_list

    def __extract_data_property_list(self, data_list):
        if is_empty_sequence(data_list):
            return []

        return [
            DataProperty(
                data,
                none_value=self.none_value,
                inf_value=self.inf_value,
                nan_value=self.nan_value,
                bool_converter=self.bool_converter,
                datetime_converter=self.datetime_converter,
                datetime_format_str=self.datetime_format_str,
                is_strict_type_mapping=self.is_strict_type_mapping,
                east_asian_ambiguous_width=self.east_asian_ambiguous_width
            )
            for data in data_list
        ]<commit_msg>Change a private method to a public method<commit_after># encoding: utf-8

"""
.. codeauthor:: <NAME> <<EMAIL>>
"""

from __future__ import absolute_import
from __future__ import unicode_literals

from six.moves import zip

from ._common import (
    DEFAULT_INF_VALUE,
    DEFAULT_NAN_VALUE,
    DEFAULT_STRICT_TYPE_MAPPING,
    default_bool_converter,
    default_datetime_converter,
)
from ._data_property import (
    DataProperty,
    ColumnDataProperty
)
from ._function import is_empty_sequence


class MissmatchProcessing(object):
    EXCEPTION = 1 << 1
    TRIM = 1 << 2
    EXTEND = 1 << 3


class PropertyExtractor(object):

    def __init__(self):
        self.header_list = []
        self.data_matrix = []
        self.min_padding_len = 0
        self.none_value = None
        self.inf_value = DEFAULT_INF_VALUE
        self.nan_value = DEFAULT_NAN_VALUE
        self.bool_converter = default_bool_converter
        self.datetime_converter = default_datetime_converter
        self.datetime_format_str = "%Y-%m-%dT%H:%M:%S%z"
        self.is_strict_type_mapping = dict(DEFAULT_STRICT_TYPE_MAPPING)
        self.east_asian_ambiguous_width = 1

        self.mismatch_processing = MissmatchProcessing.TRIM

    def extract_data_property_matrix(self):
        return [
            self.extract_data_property_list(data_list)
            for data_list in self.data_matrix
        ]

    def extract_column_property_list(self):
        column_prop_list = self.__extract_header_data_prop_list()

        try:
            data_prop_matrix = self.extract_data_property_matrix()
        except TypeError:
            return column_prop_list

        for col_idx, col_prop_list in enumerate(zip(*data_prop_matrix)):
            try:
                column_prop_list[col_idx]
            except IndexError:
                if self.mismatch_processing == MissmatchProcessing.EXCEPTION:
                    raise ValueError(
                        "column not found: col-size={}, col-index={}".format(
                            len(column_prop_list), col_idx))

                if any([
                    self.mismatch_processing == MissmatchProcessing.EXTEND,
                    all([
                        self.mismatch_processing == MissmatchProcessing.TRIM,
                        is_empty_sequence(self.header_list),
                    ])
                ]):
                    column_prop_list.append(ColumnDataProperty(
                        min_padding_len=self.min_padding_len,
                        datetime_format_str=self.datetime_format_str,
                        east_asian_ambiguous_width=self.east_asian_ambiguous_width
                    ))
                elif self.mismatch_processing == MissmatchProcessing.TRIM:
                    # ignore columns that longer than header column
                    continue

            for prop in col_prop_list:
                column_prop_list[col_idx].update_body(prop)

        return column_prop_list

    def __extract_header_data_prop_list(self):
        header_prop_list = self.extract_data_property_list(self.header_list)
        column_prop_list = []

        for header_prop in header_prop_list:
            column_prop = ColumnDataProperty(
                min_padding_len=self.min_padding_len,
                datetime_format_str=self.datetime_format_str,
                east_asian_ambiguous_width=self.east_asian_ambiguous_width
            )
            column_prop.update_header(header_prop)
            column_prop_list.append(column_prop)

        return column_prop_list

    def extract_data_property_list(self, data_list):
        if is_empty_sequence(data_list):
            return []

        return [
            DataProperty(
                data,
                none_value=self.none_value,
                inf_value=self.inf_value,
                nan_value=self.nan_value,
                bool_converter=self.bool_converter,
                datetime_converter=self.datetime_converter,
                datetime_format_str=self.datetime_format_str,
                is_strict_type_mapping=self.is_strict_type_mapping,
                east_asian_ambiguous_width=self.east_asian_ambiguous_width
            )
            for data in data_list
        ]
{'fraction_non_alphanumeric': 0.06368675551362189, 'fraction_numerical': 0.002122891850454063, 'mean_word_length': 2.072463768115942, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2345995', 'n_tokens_mistral': 2437, 'n_tokens_neox': 2291, 'n_words': 369}