<commit_before># -*- coding: utf-8 -*-
import six
from ._collections import PVLGroup, Units
from ._strings import needs_quotes, quote_string


class PVLEncoder(object):
    begin_group = b'BEGIN_GROUP'
    end_group = b'END_GROUP'
    begin_object = b'BEGIN_OBJECT'
    end_object = b'END_OBJECT'
    end_statement = b'END'
    null = b'NULL'
    true = b'TRUE'
    false = b'FALSE'
    assignment = b' = '
    indentation = b'  '
    newline = b'\n'

    def indent(self, level, stream):
        stream.write(level * self.indentation)

    def encode(self, module, stream):
        self.encode_block(module, 0, stream)
        stream.write(self.end_statement)

    def encode_block(self, block, level, stream):
        for key, value in six.iteritems(block):
            self.encode_statement(key, value, level, stream)

    def encode_statement(self, key, value, level, stream):
        if isinstance(value, PVLGroup):
            return self.encode_group(key, value, level, stream)

        if isinstance(value, dict):
            return self.encode_object(key, value, level, stream)

        self.encode_assignment(key, value, level, stream)

    def encode_group(self, key, value, level, stream):
        self.encode_group_begin(key, value, level, stream)
        self.encode_block(value, level + 1, stream)
        self.encode_group_end(key, value, level, stream)

    def encode_group_begin(self, key, value, level, stream):
        self.encode_raw_assignment(
            key=self.begin_group,
            value=key.encode('utf-8'),
            level=level,
            stream=stream
        )

    def encode_group_end(self, key, value, level, stream):
        self.encode_raw_assignment(
            key=self.end_group,
            value=key.encode('utf-8'),
            level=level,
            stream=stream
        )

    def encode_object(self, key, value, level, stream):
        self.encode_object_begin(key, value, level, stream)
        self.encode_block(value, level + 1, stream)
        self.encode_object_end(key, value, level, stream)

    def encode_object_begin(self, key, value, level, stream):
        self.encode_raw_assignment(
            key=self.begin_object,
            value=key.encode('utf-8'),
            level=level,
            stream=stream
        )

    def encode_object_end(self, key, value, level, stream):
        self.encode_raw_assignment(
            key=self.end_object,
            value=key.encode('utf-8'),
            level=level,
            stream=stream
        )

    def encode_assignment(self, key, value, level, stream):
        self.encode_raw_assignment(
            key=key.encode('utf-8'),
            value=self.encode_value(value),
            level=level,
            stream=stream
        )

    def encode_raw_assignment(self, key, value, level, stream):
        self.indent(level, stream)
        stream.write(key)
        stream.write(self.assignment)
        stream.write(value)
        stream.write(self.newline)

    def encode_value(self, value):
        if isinstance(value, Units):
            units = self.encode_units(value.units)
            value = self.encode_simple_value(value.value)
            return value + b' ' + units
        return self.encode_simple_value(value)

    def encode_simple_value(self, value):
        if isinstance(value, six.string_types):
            return self.encode_string(value)

        if isinstance(value, (int, float)):
            return self.encode_number(value)

        if value is None:
            return self.encode_null(value)

        if isinstance(value, bool):
            return self.encode_bool(value)

        if isinstance(value, list):
            return self.encode_list(value)

        if isinstance(value, set):
            return self.encode_set(value)

        return self.default(value)

    def encode_units(self, value):
        return b'<' + value.encode('utf-8') + b'>'

    def encode_null(self, value):
        return self.null

    def encode_number(self, value):
        return str(value).encode('utf-8')

    def encode_string(self, value):
        if needs_quotes(value):
            return quote_string(value)
        return value.encode('utf-8')

    def encode_bool(self, value):
        if value:
            return self.true
        return self.false

    def encode_sequence(self, values):
        return b', '.join([self.encode_value(v) for v in values])

    def encode_list(self, value):
        return b'(' + self.encode_sequence(value) + b')'<commit_msg>Use repr to serialize numbers (str truncates the digits).<commit_after># -*- coding: utf-8 -*-
import six
from ._collections import PVLGroup, Units
from ._strings import needs_quotes, quote_string


class PVLEncoder(object):
    begin_group = b'BEGIN_GROUP'
    end_group = b'END_GROUP'
    begin_object = b'BEGIN_OBJECT'
    end_object = b'END_OBJECT'
    end_statement = b'END'
    null = b'NULL'
    true = b'TRUE'
    false = b'FALSE'
    assignment = b' = '
    indentation = b'  '
    newline = b'\n'

    def indent(self, level, stream):
        stream.write(level * self.indentation)

    def encode(self, module, stream):
        self.encode_block(module, 0, stream)
        stream.write(self.end_statement)

    def encode_block(self, block, level, stream):
        for key, value in six.iteritems(block):
            self.encode_statement(key, value, level, stream)

    def encode_statement(self, key, value, level, stream):
        if isinstance(value, PVLGroup):
            return self.encode_group(key, value, level, stream)

        if isinstance(value, dict):
            return self.encode_object(key, value, level, stream)

        self.encode_assignment(key, value, level, stream)

    def encode_group(self, key, value, level, stream):
        self.encode_group_begin(key, value, level, stream)
        self.encode_block(value, level + 1, stream)
        self.encode_group_end(key, value, level, stream)

    def encode_group_begin(self, key, value, level, stream):
        self.encode_raw_assignment(
            key=self.begin_group,
            value=key.encode('utf-8'),
            level=level,
            stream=stream
        )

    def encode_group_end(self, key, value, level, stream):
        self.encode_raw_assignment(
            key=self.end_group,
            value=key.encode('utf-8'),
            level=level,
            stream=stream
        )

    def encode_object(self, key, value, level, stream):
        self.encode_object_begin(key, value, level, stream)
        self.encode_block(value, level + 1, stream)
        self.encode_object_end(key, value, level, stream)

    def encode_object_begin(self, key, value, level, stream):
        self.encode_raw_assignment(
            key=self.begin_object,
            value=key.encode('utf-8'),
            level=level,
            stream=stream
        )

    def encode_object_end(self, key, value, level, stream):
        self.encode_raw_assignment(
            key=self.end_object,
            value=key.encode('utf-8'),
            level=level,
            stream=stream
        )

    def encode_assignment(self, key, value, level, stream):
        self.encode_raw_assignment(
            key=key.encode('utf-8'),
            value=self.encode_value(value),
            level=level,
            stream=stream
        )

    def encode_raw_assignment(self, key, value, level, stream):
        self.indent(level, stream)
        stream.write(key)
        stream.write(self.assignment)
        stream.write(value)
        stream.write(self.newline)

    def encode_value(self, value):
        if isinstance(value, Units):
            units = self.encode_units(value.units)
            value = self.encode_simple_value(value.value)
            return value + b' ' + units
        return self.encode_simple_value(value)

    def encode_simple_value(self, value):
        if isinstance(value, six.string_types):
            return self.encode_string(value)

        if isinstance(value, (int, float)):
            return self.encode_number(value)

        if value is None:
            return self.encode_null(value)

        if isinstance(value, bool):
            return self.encode_bool(value)

        if isinstance(value, list):
            return self.encode_list(value)

        if isinstance(value, set):
            return self.encode_set(value)

        return self.default(value)

    def encode_units(self, value):
        return b'<' + value.encode('utf-8') + b'>'

    def encode_null(self, value):
        return self.null

    def encode_number(self, value):
        return repr(value).encode('utf-8')

    def encode_string(self, value):
        if needs_quotes(value):
            return quote_string(value)
        return value.encode('utf-8')

    def encode_bool(self, value):
        if value:
            return self.true
        return self.false

    def encode_sequence(self, values):
        return b', '.join([self.encode_value(v) for v in values])

    def encode_list(self, value):
        return b'(' + self.encode_sequence(value) + b')'
{'fraction_non_alphanumeric': 0.10768213380359308, 'fraction_numerical': 0.0026452110657996255, 'mean_word_length': 2.3607407407407406, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '454027', 'n_tokens_mistral': 2724, 'n_tokens_neox': 2648, 'n_words': 609}