<commit_before>"""
The core of Metalsmyth (this really needs a better name) is a Stack,
which reads files from a source directory, registers middleware
and processes files.
"""
import glob
import os

import frontmatter


class Stack(object):
    """
    A Stack takes a source directory, output directory, optional middleware and metadata
    """
    def __init__(self, source='src', dest='build', *middleware, **metadata):
        self.source = source
        self.dest = dest
        self.middleware = list(middleware)
        self.metadata = dict(metadata)

    def get_files(self):
        """
        Read and parse files from a directory,
        return a dictionary of path => post
        """
        files = {}
        for filename in os.listdir(self.source):
            path = os.path.join(self.source, filename)
            files[filename] = frontmatter.load(path, 
                filename=filename, 
                slug=os.path.basename(filename))

        return files

    def run(self):
        "Run each middleware function on files"
        # load files from source directory
        files = self.get_files()

        # loop through each middleware
        for func in self.middleware:

            # call each one, ignoring return value
            func(files, self)

        # store and return the result
        self.files = files
        return files

    def build(self):
        "Build out results to dest directory (creating if needed)"

        # ensure a build dir
        if not os.path.isdir(self.dest):
            os.makedirs(self.dest)

        # make sure we have files
        files = getattr(self, 'files', self.run())

        # write the content of each post to dest, using keys as filenames
        for filename, post in self.files.items():

            # join filename to dest dir
            path = os.path.join(self.dest, filename)
            with open(path, 'w') as f:
                f.write(post.content.encode('utf-8'))

    def serialize(self, as_dict=False, sort=None):
        """
        Dump built files as a list or dictionary, for JSON or other serialization.

            sort: a key function to sort a list, or simply True
        """
        files = getattr(self, 'files', self.run())

        if as_dict:
            return dict((fn, p.to_dict()) for fn, p in files.items())

        # generate a list
        data = (p.to_dict() for p in files.values())

        if callable(sort):
            return sorted(data, key=sort)

        elif sort is True:
            return sorted(data)

        return list(data)

<commit_msg>Write binary files<commit_after>"""
The core of Metalsmyth (this really needs a better name) is a Stack,
which reads files from a source directory, registers middleware
and processes files.
"""
import glob
import os

import frontmatter


class Stack(object):
    """
    A Stack takes a source directory, output directory, optional middleware and metadata
    """
    def __init__(self, source='src', dest='build', *middleware, **metadata):
        self.source = source
        self.dest = dest
        self.middleware = list(middleware)
        self.metadata = dict(metadata)

    def get_files(self):
        """
        Read and parse files from a directory,
        return a dictionary of path => post
        """
        files = {}
        for filename in os.listdir(self.source):
            path = os.path.join(self.source, filename)
            files[filename] = frontmatter.load(path, 
                filename=filename, 
                slug=os.path.basename(filename))

        return files

    def run(self):
        "Run each middleware function on files"
        # load files from source directory
        files = self.get_files()

        # loop through each middleware
        for func in self.middleware:

            # call each one, ignoring return value
            func(files, self)

        # store and return the result
        self.files = files
        return files

    def build(self):
        "Build out results to dest directory (creating if needed)"

        # ensure a build dir
        if not os.path.isdir(self.dest):
            os.makedirs(self.dest)

        # make sure we have files
        files = getattr(self, 'files', self.run())

        # write the content of each post to dest, using keys as filenames
        for filename, post in self.files.items():

            # join filename to dest dir
            path = os.path.join(self.dest, filename)
            with open(path, 'wb') as f:
                f.write(post.content.encode('utf-8'))

    def serialize(self, as_dict=False, sort=None):
        """
        Dump built files as a list or dictionary, for JSON or other serialization.

            sort: a key function to sort a list, or simply True
        """
        files = getattr(self, 'files', self.run())

        if as_dict:
            return dict((fn, p.to_dict()) for fn, p in files.items())

        # generate a list
        data = (p.to_dict() for p in files.values())

        if callable(sort):
            return sorted(data, key=sort)

        elif sort is True:
            return sorted(data)

        return list(data)


{'fraction_non_alphanumeric': 0.09199845380749903, 'fraction_numerical': 0.00038654812524159255, 'mean_word_length': 2.0174927113702625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7520500', 'n_tokens_mistral': 1413, 'n_tokens_neox': 1359, 'n_words': 506}