<commit_before>from __future__ import unicode_literals

import re
import string

from datetime import date
from datetime import datetime
from datetime import time
from enum import Enum
from typing import Any
from typing import Dict
from typing import Generator
from typing import List
from typing import Optional
from typing import Union


from ._compat import PY2
from ._compat import decode
from ._compat import long
from ._compat import unicode
from ._utils import escape_string

if PY2:
    from functools32 import lru_cache
else:
    from functools import lru_cache


def item(value, _parent=None):
    from .container import Container

    if isinstance(value, Item):
        return value

    if isinstance(value, bool):
        return Bool(value, Trivia())
    elif isinstance(value, int):
        return Integer(value, Trivia(), str(value))
    elif isinstance(value, float):
        return Float(value, Trivia(), str(value))
    elif isinstance(value, dict):
        val = Table(Container(), Trivia(), False)
        for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])):
            val[k] = item(v, _parent=val)

        return val
    elif isinstance(value, list):
        if value and isinstance(value[0], dict):
            a = AoT([])
        else:
            a = Array([], Trivia())

        for v in value:
            if isinstance(v, dict):
                table = Table(Container(), Trivia(), True)

                for k, _v in sorted(
                    v.items(), key=lambda i: (isinstance(i[1], dict), i[0])
                ):
                    i = item(_v)
                    if isinstance(table, InlineTable):
                        i.trivia.trail = ""

                    table[k] = item(i)

                v = table

            a.append(v)

        return a
    elif isinstance(value, (str, unicode)):
        escaped = escape_string(value)

        return String(StringType.SLB, value, escaped, Trivia())
    elif isinstance(value, datetime):
        return DateTime(value, Trivia(), value.isoformat().replace("+00:00", "Z"))
    elif isinstance(value, date):
        return Date(value, Trivia(), value.isoformat())
    elif isinstance(value, time):
        return Time(value, Trivia(), value.isoformat())

    raise ValueError("Invalid type {}".format(type(value)))


class StringType(Enum):
    # Single Line Basic
    SLB = '"'
    # Multi Line Basic
    MLB = '"""'
    # Single Line Literal
    SLL = "'"
    # Multi Line Literal
    MLL = "'''"

    @property
    @lru_cache(maxsize=None)
    def unit(self):  # type: () -> str
        return self.value[0]

    @lru_cache(maxsize=None)
    def is_basic(self):  # type: () -> bool
        return self in {StringType.SLB, StringType.MLB}<commit_msg>Fix string initialization via item() on Python 2.7<commit_after>from __future__ import unicode_literals

import re
import string

from datetime import date
from datetime import datetime
from datetime import time
from enum import Enum
from typing import Any
from typing import Dict
from typing import Generator
from typing import List
from typing import Optional
from typing import Union


from ._compat import PY2
from ._compat import decode
from ._compat import long
from ._compat import unicode
from ._utils import escape_string

if PY2:
    from functools32 import lru_cache
else:
    from functools import lru_cache


def item(value, _parent=None):
    from .container import Container

    if isinstance(value, Item):
        return value

    if isinstance(value, bool):
        return Bool(value, Trivia())
    elif isinstance(value, int):
        return Integer(value, Trivia(), str(value))
    elif isinstance(value, float):
        return Float(value, Trivia(), str(value))
    elif isinstance(value, dict):
        val = Table(Container(), Trivia(), False)
        for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])):
            val[k] = item(v, _parent=val)

        return val
    elif isinstance(value, list):
        if value and isinstance(value[0], dict):
            a = AoT([])
        else:
            a = Array([], Trivia())

        for v in value:
            if isinstance(v, dict):
                table = Table(Container(), Trivia(), True)

                for k, _v in sorted(
                    v.items(), key=lambda i: (isinstance(i[1], dict), i[0])
                ):
                    i = item(_v)
                    if isinstance(table, InlineTable):
                        i.trivia.trail = ""

                    table[k] = item(i)

                v = table

            a.append(v)

        return a
    elif isinstance(value, (str, unicode)):
        escaped = escape_string(value)

        return String(StringType.SLB, decode(value), escaped, Trivia())
    elif isinstance(value, datetime):
        return DateTime(value, Trivia(), value.isoformat().replace("+00:00", "Z"))
    elif isinstance(value, date):
        return Date(value, Trivia(), value.isoformat())
    elif isinstance(value, time):
        return Time(value, Trivia(), value.isoformat())

    raise ValueError("Invalid type {}".format(type(value)))


class StringType(Enum):
    # Single Line Basic
    SLB = '"'
    # Multi Line Basic
    MLB = '"""'
    # Single Line Literal
    SLL = "'"
    # Multi Line Literal
    MLL = "'''"

    @property
    @lru_cache(maxsize=None)
    def unit(self):  # type: () -> str
        return self.value[0]

    @lru_cache(maxsize=None)
    def is_basic(self):  # type: () -> bool
        return self in {StringType.SLB, StringType.MLB}
{'fraction_non_alphanumeric': 0.11266344259358768, 'fraction_numerical': 0.0053734551316496505, 'mean_word_length': 2.4278698588090855, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2922717', 'n_tokens_mistral': 1660, 'n_tokens_neox': 1597, 'n_words': 516}