<commit_before>from collections import OrderedDict
import os
from inspect import getsource
import StringIO
import glob
import logging
from hashlib import md5

from .processors import (
    Bundle,
    ClosureBuilder,
    HashFileNames,
    SCSS,
    YUI,
)

from .base import NullOutputter #, NullCompiler, NullMinifier,
from .outputters.blobstore import Blobstore
from .outputters.filesystem import Filesystem

PROCESSORS = {}
OUTPUTTERS = {}

def register_processor(name, processor_class):
    PROCESSORS[name] = processor_class

def register_outputter(name, outputter_class):
    OUTPUTTERS[name] = outputter_class


register_processor("bundle", Bundle)
register_processor("closure", ClosureBuilder,)
register_processor("hashfilenames", HashFileNames)
register_processor("scss", SCSS)
register_processor("yui", YUI)

register_outputter("null", NullOutputter)
register_outputter("blobstore", Blobstore)
register_outputter("filesystem", Filesystem)


class Node(object):
    def __init__(self, parent):
        #the list of files (including wildcards) which we started with
        self.inputs = []

        #the expanded list of input files, with folder/* wildcards expanded
        self.input_files = []

        # filename:StringIO(contents) mappings of the files we will output
        # this is what gets modified in each step of the pipeline
        self.outputs = OrderedDict()

        self.output_filename_changes_done = False

        self.parent = parent<commit_msg>Make it obvious that it's the closure_builder script<commit_after>from collections import OrderedDict
import os
from inspect import getsource
import StringIO
import glob
import logging
from hashlib import md5

from .processors import (
    Bundle,
    ClosureBuilder,
    HashFileNames,
    SCSS,
    YUI,
)

from .base import NullOutputter #, NullCompiler, NullMinifier,
from .outputters.blobstore import Blobstore
from .outputters.filesystem import Filesystem

PROCESSORS = {}
OUTPUTTERS = {}

def register_processor(name, processor_class):
    PROCESSORS[name] = processor_class

def register_outputter(name, outputter_class):
    OUTPUTTERS[name] = outputter_class


register_processor("bundle", Bundle)
register_processor("closure_builder", ClosureBuilder)
register_processor("hashfilenames", HashFileNames)
register_processor("scss", SCSS)
register_processor("yui", YUI)

register_outputter("null", NullOutputter)
register_outputter("blobstore", Blobstore)
register_outputter("filesystem", Filesystem)


class Node(object):
    def __init__(self, parent):
        #the list of files (including wildcards) which we started with
        self.inputs = []

        #the expanded list of input files, with folder/* wildcards expanded
        self.input_files = []

        # filename:StringIO(contents) mappings of the files we will output
        # this is what gets modified in each step of the pipeline
        self.outputs = OrderedDict()

        self.output_filename_changes_done = False

        self.parent = parent
{'fraction_non_alphanumeric': 0.07728181212524983, 'fraction_numerical': 0.0006662225183211193, 'mean_word_length': 4.5, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '840540', 'n_tokens_mistral': 847, 'n_tokens_neox': 820, 'n_words': 250}