<commit_before>import os
import collections
import datetime
import itertools


class Index(object):
    version = 1

    def __init__(self, root_path):
        self.headers = collections.OrderedDict()
        self.files = collections.OrderedDict()

        # Default to version in class; if reading file, this will get overwritten
        self.headers['version'] = Index.version
        self.headers['root_path'] = root_path

    def itercontent(self):
        """Return an iterator over the content,
        converted to strings for writing to disk"""

        header_strings = ("{}: {}\n".format(k, v) for k, v in self.headers.iteritems())
        separator = ("\n",)
        file_strings = ("{} {}\n".format(hash_, path) for path, hash_ in self.files.iteritems())

        return itertools.chain(header_strings, separator, file_strings)

    def add(self, path, hash_):
        self.files[path] = hash_

    def remove(self, path):
        del self.files[path]

    def read(self, path):
        with open(path, 'r') as f:
            self.headers = parse_headers(f)

            if 'version' not in self.headers:
                raise Exception("Invalid file format: no version header")

            if self.headers['version'] != str(Index.version):
                raise Exception("Unsupported file format version: file is {}, parser is {}".format(self.headers['version'], Index.version))

            self.files = parse_files(f)

    def write(self, file_handle):
        # The date header is the moment the index is written to disk
        self.headers['date'] = datetime.datetime.utcnow().isoformat()

        for line in self.itercontent():
            file_handle.write(line)



##############################
# Index file parsing functions
##############################

def split_lines(lines, delim, num_fields):
    """Split each element in the sequence 'lines' into its component fields."""<commit_msg>Sort Index output while writing<commit_after>import os
import collections
import datetime
import itertools


class Index(object):
    version = 1

    def __init__(self, root_path):
        self.headers = collections.OrderedDict()
        self.files = collections.OrderedDict()

        # Default to version in class; if reading file, this will get overwritten
        self.headers['version'] = Index.version
        self.headers['root_path'] = root_path

    def itercontent(self):
        """Return an iterator over the content,
        converted to strings for writing to disk"""

        header_strings = ("{}: {}\n".format(k, v) for k, v in self.headers.iteritems())
        separator = ("\n",)
        file_strings = ("{} {}\n".format(hash_, path) for path, hash_ in self.files.iteritems())

        return itertools.chain(header_strings, separator, file_strings)

    def add(self, path, hash_):
        self.files[path] = hash_

    def remove(self, path):
        del self.files[path]

    def read(self, path):
        with open(path, 'r') as f:
            self.headers = parse_headers(f)

            if 'version' not in self.headers:
                raise Exception("Invalid file format: no version header")

            if self.headers['version'] != str(Index.version):
                raise Exception("Unsupported file format version: file is {}, parser is {}".format(self.headers['version'], Index.version))

            self.files = parse_files(f)

    def write(self, file_handle):
        # The date header is the moment the index is written to disk
        self.headers['date'] = datetime.datetime.utcnow().isoformat()

        # Write headers -- but sort the keys before writing
        for key in sorted(self.headers):
            file_handle.write("{}: {}\n".format(key, self.headers[key]))

        # Separate the headers from the files list with a blank line
        file_handle.write("\n")

        # Write files and hashes -- but sort the paths before writing
        for path in sorted(self.files):
            file_handle.write("{} {}\n".format(self.files[path], path))


##############################
# Index file parsing functions
##############################

def split_lines(lines, delim, num_fields):
    """Split each element in the sequence 'lines' into its component fields."""
{'fraction_non_alphanumeric': 0.14005668398677373, 'fraction_numerical': 0.0004723665564478035, 'mean_word_length': 2.829113924050633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '188021', 'n_tokens_mistral': 1158, 'n_tokens_neox': 1113, 'n_words': 366}