<commit_before># -*- coding: utf-8 -*-
import codecs
import csv
import six
from six.moves import cStringIO


# These classes are copied from http://docs.python.org/2/library/csv.html


class CsvUnicodeWriter(object):
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()

    def cast_to_str(self, obj):
        if isinstance(obj, six.text_type):
            return obj.encode('utf-8')
        elif isinstance(obj, six.byte_type):
            return obj
        elif hasattr(obj, '__unicode__'):
            return six.test_type(obj).encode('utf-8')
        elif hasattr(obj, '__str__'):
            return str(obj)
        else:
            raise TypeError('Expecting unicode, str, or object castable'
                            ' to unicode or string, got: %r' % type(obj))

    def writerow(self, row):
        self.writer.writerow([self.cast_to_str(s) for s in row])
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        self.stream.write(data)
        # empty queue<commit_msg>Use six for strings / unicodes / bytes<commit_after># -*- coding: utf-8 -*-
import codecs
import csv
import six
from six.moves import cStringIO


# These classes are copied from http://docs.python.org/2/library/csv.html


class CsvUnicodeWriter(object):
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()

    def cast_to_str(self, obj):
        if isinstance(obj, six.text_type):
            return obj.encode('utf-8')
        elif isinstance(obj, six.binary_type):
            return obj
        elif hasattr(obj, '__unicode__'):
            return six.test_type(obj).encode('utf-8')
        elif hasattr(obj, '__str__'):
            return str(obj)
        else:
            raise TypeError('Expecting unicode, str, or object castable'
                            ' to unicode or string, got: %r' % type(obj))

    def writerow(self, row):
        self.writer.writerow([self.cast_to_str(s) for s in row])
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        self.stream.write(data)
        # empty queue
{'fraction_non_alphanumeric': 0.11483253588516747, 'fraction_numerical': 0.004465709728867623, 'mean_word_length': 2.2531120331950207, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5702305', 'n_tokens_mistral': 948, 'n_tokens_neox': 919, 'n_words': 270}