<commit_before># -*- coding: utf-8 -*-
"""
Representation of content for a queue where the values may exceed the native
pipe size.
"""
import os
import sys
import tempfile

from codecs import getreader

from .enum import enum
from .kitchenpatch import getwriter
from kitchen.text.converters import to_bytes

# Py2 uses "file" as the base class for IO
# Used for an isinstance comparison
if sys.version_info[0] == 3:
    from io import IOBase as file


class ContentWrapper(object):
    """
    Representation of content for a queue where the values may exceed the
    native pipe size.
    """
    TYPES = enum(
        'DIRECT',
        'FILE')

    # Need to figure out a way to do this automatically
    THRESHOLD = 2**14  # 2**14 = 16,384

    def __getattr__(self, attr):
        """When necessary pull the 'value' from a buffer file"""
        if attr == "value" \
                and object.__getattribute__(self, "type") == \
                ContentWrapper.TYPES.FILE:
            return self._getValueFromFile()
        else:
            return object.__getattribute__(self, attr)

    def __setattr__(self, attr, val):
        """When necessary, save the 'value' to a buffer file"""
        if attr == "value":
            # Within the threshold size limit
            if len(to_bytes(val)) < ContentWrapper.THRESHOLD:
                object.__setattr__(self, attr, val)

            # Larger than what a queue value can hold<commit_msg>Add more detail to ContentWrapper docstring.<commit_after># -*- coding: utf-8 -*-
"""
Representation of content for a queue where the values may exceed the native
pipe size.
"""
import os
import sys
import tempfile

from codecs import getreader

from .enum import enum
from .kitchenpatch import getwriter
from kitchen.text.converters import to_bytes

# Py2 uses "file" as the base class for IO
# Used for an isinstance comparison
if sys.version_info[0] == 3:
    from io import IOBase as file


class ContentWrapper(object):
    """
    Representation of content for a queue where the values may exceed the
    native pipe size.

    Store data with
        cw = ContentWrapper("My text data")
        cw.value = "Updated text data"

    Access data with
        print("My data: {}".format(cw.value))

    TODO: Manage value updates that cross THRESHOLD
    """
    TYPES = enum(
        'DIRECT',
        'FILE')

    # Need to figure out a way to do this automatically
    THRESHOLD = 2**14  # 2**14 = 16,384

    def __getattr__(self, attr):
        """When necessary pull the 'value' from a buffer file"""
        if attr == "value" \
                and object.__getattribute__(self, "type") == \
                ContentWrapper.TYPES.FILE:
            return self._getValueFromFile()
        else:
            return object.__getattribute__(self, attr)

    def __setattr__(self, attr, val):
        """When necessary, save the 'value' to a buffer file"""
        if attr == "value":
            # Within the threshold size limit
            if len(to_bytes(val)) < ContentWrapper.THRESHOLD:
                object.__setattr__(self, attr, val)

            # Larger than what a queue value can hold
{'fraction_non_alphanumeric': 0.08952380952380952, 'fraction_numerical': 0.009523809523809525, 'mean_word_length': 2.489479512735327, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2339840', 'n_tokens_mistral': 889, 'n_tokens_neox': 854, 'n_words': 344}