<commit_before># coding: utf-8
from decimal import Decimal


class FixedWidthWriter():

    def __init__(self, fd, fields, line_endings='linux'):
        self.fd = fd
        self.fields = fields
        if line_endings == 'linux':
            self.line_endings = '\n'
        elif line_endings == 'windows':
            self.line_endings = '\r\n'
        else:
            raise ValueError('Only windows or linux line endings supported')

    def writerow(self, rowdict):
        row = []
        for field in self.fields:
            try:
                key, width, options = field
            except ValueError:
                key, width = field
                options = {}
            value = rowdict[key]
            decimal_spaces = options.get('decimal_spaces', 0)
            if decimal_spaces:
                value = unicode(Decimal(value)
                                .quantize(Decimal(10)**-decimal_spaces))
            part = '{0: {1}{2}}' \
                .format(value, options.get('direction', '<'), width)
            row.append(part)
        row = ''.join(row)
        self.fd.write(row + self.line_endings)

    def writerows(self, rowdicts):
        for rowdict in rowdicts:
            self.writerow(rowdict)<commit_msg>Changes the line_endings arg to lineterminators to align with python's own csv module. Defaults to \r\n.<commit_after># coding: utf-8
from decimal import Decimal


class FixedWidthWriter():

    def __init__(self, fd, fields, lineterminator='\r\n'):
        self.fd = fd
        self.fields = fields

    def writerow(self, rowdict):
        row = []
        for field in self.fields:
            try:
                key, width, options = field
            except ValueError:
                key, width = field
                options = {}
            value = rowdict[key]
            decimal_spaces = options.get('decimal_spaces', 0)
            if decimal_spaces:
                value = unicode(Decimal(value)
                                .quantize(Decimal(10)**-decimal_spaces))
            part = '{0: {1}{2}}' \
                .format(value, options.get('direction', '<'), width)
            row.append(part)
        row = ''.join(row)
        self.fd.write(row + self.lineterminator)

    def writerows(self, rowdicts):
        for rowdict in rowdicts:
            self.writerow(rowdict)
{'fraction_non_alphanumeric': 0.11035364294844482, 'fraction_numerical': 0.005965061780997018, 'mean_word_length': 1.7113163972286374, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '897596', 'n_tokens_mistral': 683, 'n_tokens_neox': 643, 'n_words': 165}