<commit_before># Copyright 2014 Cloudera Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from six import StringIO
import pandas as pd


def parse_metadata(descr_table):
    parser = MetadataParser(descr_table)
    return parser.parse()


def _noop(tup):
    return None


def _item_converter(i):
    def _get_item(converter=None):
        def _converter(tup):
            result = tup[i]
            if converter is not None:
                result = converter(result)
            return result

        return _converter

    return _get_item

_get_type = _item_converter(1)
_get_comment = _item_converter(2)


def _try_timestamp(x):
    try:
        return pd.Timestamp(x)
    except (ValueError, TypeError):
        return x


def _try_unix_timestamp(x):
    try:
        return pd.Timestamp.fromtimestamp(int(x))
    except (ValueError, TypeError):
        return x


def _try_boolean(x):
    try:
        x = x.lower()
        if x in ('true', 'yes'):
            return True
        elif x in ('false', 'no'):
            return False
        return x
    except (ValueError, TypeError):
        return x


def _try_int(x):
    try:
        return int(x)
    except (ValueError, TypeError):
        return x


class MetadataParser(object):

    """<commit_msg>BUG: work around pandas issue 12622. Close #787<commit_after># Copyright 2014 Cloudera Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from six import StringIO
import pandas as pd


def parse_metadata(descr_table):
    parser = MetadataParser(descr_table)
    return parser.parse()


def _noop(tup):
    return None


def _item_converter(i):
    def _get_item(converter=None):
        def _converter(tup):
            result = tup[i]
            if converter is not None:
                result = converter(result)
            return result

        return _converter

    return _get_item

_get_type = _item_converter(1)
_get_comment = _item_converter(2)


def _try_timestamp(x):
    try:
        ts = pd.Timestamp(x)
        return ts.to_pydatetime()
    except (ValueError, TypeError):
        return x


def _try_unix_timestamp(x):
    try:
        ts = pd.Timestamp.fromtimestamp(int(x))
        return ts.to_pydatetime()
    except (ValueError, TypeError):
        return x


def _try_boolean(x):
    try:
        x = x.lower()
        if x in ('true', 'yes'):
            return True
        elif x in ('false', 'no'):
            return False
        return x
    except (ValueError, TypeError):
        return x


def _try_int(x):
    try:
        return int(x)
    except (ValueError, TypeError):
        return x


class MetadataParser(object):

    """
{'fraction_non_alphanumeric': 0.08292011019283746, 'fraction_numerical': 0.007713498622589532, 'mean_word_length': 2.678824721377913, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2096136', 'n_tokens_mistral': 1099, 'n_tokens_neox': 1050, 'n_words': 379}