<commit_before>""" Writer types.

Writers convert data records to lines of text.

"""
from ._util import field_type

__all__ = ("DelimitedWriter", "FixedWidthWriter")


class _Writer(object):
    """ Abstract base class for all serial data writers.

    Serial data consists of individual records stored as lines of text.

    """
    def __init__(self, stream):
        """ Initialize this object.

        The output stream is any object that implements write() to write a line
        of text.

        """
        self._stream = stream
        self._filters = []
        return

    def filter(self, *callbacks):
        """ Add a filter to this writer or clear all filters.

        A filter is a callable object that accepts a data record as its only
        argument. Based on this record the filter can perform the following
        actions:
        1. Return None to reject the record (it will not be written).
        2. Return the data record as is.
        3. Return a new* record.
        
        *Take care not to modify a mutable argument unless the caller doesn't
         expect write() to be free of side effects.

        """
        if not callbacks:
            self._filters = []
        else:
            self._filters.extend(callbacks)
        return

    def write(self, record):
        """ Write a record to the output stream.
        
        """
        for callback in self._filters:
            record = callback(record)
            if record is None:
                return
        self._put(record)
        return

    def dump(self, records):
        """ Write all records to the output stream.
        
        """
        for record in records:
            self.write(record)
        return

    def _put(self, record):
        """ Write a record to the output stream.
        
        This is called after the record has been passed through all filters.
        
        """
        raise NotImplementedError


class _TabularWriter(_Writer):
    """ Abstract base class for tabular data writers.

    Tabular data is organized fields such that each field occupies the same
    position in each record. One line of text corresponds to a one complete
    record.

    """
    def __init__(self, stream, fields, endl="\n"):
        """ Initialize this object.

        """
        super(_TabularWriter, self).__init__(stream)
        self._fields = [field_type(*args) for args in fields]
        self._endl = endl
        return

    def _put(self, record):
        """ Write a filtered record to the output stream.

        """
        tokens = [field.dtype.encode(record.get(field.name)) for field in
                  self._fields]
        self._stream.write(self._join(tokens) + self._endl)
        return

    def _join(self, tokens):
        """ Join a sequence of tokens into a line of text.

        """
        raise NotImplementedError


class DelimitedWriter(_TabularWriter):
    """ A writer for fields delineated by a delimiter.

    The position of each scalar field is be given as an integer index, and the
    position of an array field is the pair [beg, end).

    """
    def __init__(self, stream, fields, delim, endl="\n"):
        """ Initialize this object.

        At this time there is no escaping of characters in the input records
        that match the delimiter; this may cause issues when trying to parse
        the resulting output.

        """
        super(DelimitedWriter, self).__init__(stream, fields, endl)
        self._delim = delim
        return

    def _join(self, tokens):
        """ Join a sequence of tokens into a line of text.

        """
        pos = 0
        while pos < len(tokens):
            # A token can itself be a sequence of tokens (c.f. ArrayType).
            token = tokens[pos]
            if isinstance(token, basestring):
                pos += 1
            else:
                # A sequence of tokens; expand inline.
                tokens[pos:pos+1] = token
                pos += len(token)
        return self._delim.join(tokens)


class FixedWidthWriter(DelimitedWriter):<commit_msg>refactored _TabularWriter hierarchy<commit_after>""" Writer types.

Writers convert data records to lines of text.

"""
from ._util import field_type

__all__ = ("DelimitedWriter", "FixedWidthWriter")


class _Writer(object):
    """ Abstract base class for all serial data writers.

    Serial data consists of individual records stored as lines of text.

    """
    def __init__(self, stream):
        """ Initialize this object.

        The output stream is any object that implements write() to write a line
        of text.

        """
        self._stream = stream
        self._filters = []
        return

    def filter(self, *callbacks):
        """ Add a filter to this writer or clear all filters.

        A filter is a callable object that accepts a data record as its only
        argument. Based on this record the filter can perform the following
        actions:
        1. Return None to reject the record (it will not be written).
        2. Return the data record as is.
        3. Return a new* record.
        
        *Take care not to modify a mutable argument unless the caller doesn't
         expect write() to be free of side effects.

        """
        if not callbacks:
            self._filters = []
        else:
            self._filters.extend(callbacks)
        return

    def write(self, record):
        """ Write a record to the output stream.
        
        """
        for callback in self._filters:
            record = callback(record)
            if record is None:
                return
        self._put(record)
        return

    def dump(self, records):
        """ Write all records to the output stream.
        
        """
        for record in records:
            self.write(record)
        return

    def _put(self, record):
        """ Write a record to the output stream.
        
        This is called after the record has been passed through all filters.
        
        """
        raise NotImplementedError


class _TabularWriter(_Writer):
    """ Abstract base class for tabular data writers.

    Tabular data is organized fields such that each field occupies the same
    position in each record. One line of text corresponds to a one complete
    record.

    """
    def __init__(self, stream, fields, endl="\n"):
        """ Initialize this object.

        """
        super(_TabularWriter, self).__init__(stream)
        self._fields = [field_type(*args) for args in fields]
        self._endl = endl
        return

    def _put(self, record):
        """ Write a filtered record to the output stream.

        """
        tokens = []
        for field in self._fields:
            token = field.dtype.encode(record.get(field.name))
            if isinstance(token, basestring):
                tokens.append(token)
            else:
                # A sequence of tokens (e.g. an ArrayType); expand inline.
                tokens.extend(token)
        self._stream.write(self._join(tokens) + self._endl)
        return

    def _join(self, tokens):
        """ Join a sequence of tokens into a line of text.

        """
        raise NotImplementedError


class DelimitedWriter(_TabularWriter):
    """ A writer for fields delineated by a delimiter.

    The position of each scalar field is be given as an integer index, and the
    position of an array field is the pair [beg, end).

    """
    def __init__(self, stream, fields, delim, endl="\n"):
        """ Initialize this object.

        At this time there is no escaping of characters in the input records
        that match the delimiter; this may cause issues when trying to parse
        the resulting output.

        """
        super(DelimitedWriter, self).__init__(stream, fields, endl)
        self._delim = delim
        return

    def _join(self, tokens):
        """ Join a sequence of tokens into a line of text.

        """
        return self._delim.join(tokens)


class FixedWidthWriter(DelimitedWriter):
{'fraction_non_alphanumeric': 0.07751457273967506, 'fraction_numerical': 0.0011162098474513209, 'mean_word_length': 1.9636163175303198, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1165110', 'n_tokens_mistral': 2071, 'n_tokens_neox': 1986, 'n_words': 855}