<commit_before>"""
Unicode Readers and Writers for use with the stdlib's csv module.

See <http://docs.python.org/library/csv.html> for details.
"""
import csv
import codecs
import cStringIO


class UnicodeRecoder:
    """
    Iterator that reads an encoded stream and reencodes the input to the
    specified encoding
    """
    def __init__(self, f, encoding):
        self.reader = codecs.getreader(encoding)(f)
        self.encoding = encoding

    def __iter__(self):
        return self

    def next(self):
        return self.reader.next().encode(self.encoding)


class UnicodeReader:
    """
    A CSV reader which will iterate over lines in the CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        f = UnicodeRecoder(f, encoding)
        self.encoding = encoding
        self.reader = csv.reader(f, dialect=dialect, **kwds)
        self.line_num = 0

    def next(self):
        row = self.reader.next()
        self.line_num = self.reader.line_num
        return [unicode(s, self.encoding) for s in row]

    def __iter__(self):
        return self


class UnicodeWriter:
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoding = encoding
        self.encoder = codecs.getincrementalencoder(encoding)()

    def writerow(self, row):
        self.writer.writerow([unicode(s).encode(self.encoding) for s in row])
        data = self.queue.getvalue()
        data = data.decode(self.encoding)
        data = self.encoder.encode(data)
        self.stream.write(data)
        self.queue.truncate(0)

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)<commit_msg>Use **kwargs instead of **kwds for consistency<commit_after>"""
Unicode Readers and Writers for use with the stdlib's csv module.

See <http://docs.python.org/library/csv.html> for details.
"""
import csv
import codecs
import cStringIO


class UnicodeRecoder:
    """
    Iterator that reads an encoded stream and reencodes the input to the
    specified encoding
    """
    def __init__(self, f, encoding):
        self.reader = codecs.getreader(encoding)(f)
        self.encoding = encoding

    def __iter__(self):
        return self

    def next(self):
        return self.reader.next().encode(self.encoding)


class UnicodeReader:
    """
    A CSV reader which will iterate over lines in the CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwargs):
        f = UnicodeRecoder(f, encoding)
        self.encoding = encoding
        self.reader = csv.reader(f, dialect=dialect, **kwargs)
        self.line_num = 0

    def next(self):
        row = self.reader.next()
        self.line_num = self.reader.line_num
        return [unicode(s, self.encoding) for s in row]

    def __iter__(self):
        return self


class UnicodeWriter:
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwargs):
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwargs)
        self.stream = f
        self.encoding = encoding
        self.encoder = codecs.getincrementalencoder(encoding)()

    def writerow(self, row):
        self.writer.writerow([unicode(s).encode(self.encoding) for s in row])
        data = self.queue.getvalue()
        data = data.decode(self.encoding)
        data = self.encoder.encode(data)
        self.stream.write(data)
        self.queue.truncate(0)

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)
{'fraction_non_alphanumeric': 0.11141507087789107, 'fraction_numerical': 0.001989554837105198, 'mean_word_length': 2.6136567834681044, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6240240', 'n_tokens_mistral': 1244, 'n_tokens_neox': 1176, 'n_words': 349}