<commit_before>#!/usr/bin/env python

# TODO: right now we assume that input files are pre-split.
# TODO: start up and close down mappers and reducers.

import threading


def simple_run(job, input, registry, map_name, part_name, reduce_name,
        map_tasks=1, reduce_tasks=1):
    map_out = job.map_data(input, registry, map_name, part_name, map_tasks,
            reduce_tasks)
    reduce_out = job.reduce_data(map_out, registry, map_name, part_name,
            map_tasks, reduce_tasks)


class Program(object):
    """Mrs Program
    
    A Program consists of a registry of functions and a run function.
    """
    def __init__(self, run, registry):
        self.run = run
        self.registry = registry


class Job(object):
    """Keep track of all operations that need to be performed."""
    def __init__(self):
        self.datasets = []

    def map_data(self, *args):
        ds = MapData(*args)
        self.datasets.append(ds)
        return ds

    def reduce_data(self, *args):
        ds = ReduceData(*args)
        self.datasets.append(ds)
        return ds

<commit_msg>removed unused Program class<commit_after>#!/usr/bin/env python

# TODO: right now we assume that input files are pre-split.
# TODO: start up and close down mappers and reducers.

import threading


def simple_run(job, input, registry, map_name, part_name, reduce_name,
        map_tasks=1, reduce_tasks=1):
    map_out = job.map_data(input, registry, map_name, part_name, map_tasks,
            reduce_tasks)
    reduce_out = job.reduce_data(map_out, registry, map_name, part_name,
            map_tasks, reduce_tasks)


class Job(object):
    """Keep track of all operations that need to be performed."""
    def __init__(self):
        self.datasets = []

    def map_data(self, *args):
        ds = MapData(*args)
        self.datasets.append(ds)
        return ds

    def reduce_data(self, *args):
        ds = ReduceData(*args)
        self.datasets.append(ds)
        return ds


{'fraction_non_alphanumeric': 0.09843513377082282, 'fraction_numerical': 0.0020191822311963654, 'mean_word_length': 2.8635477582846005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '706127', 'n_tokens_mistral': 622, 'n_tokens_neox': 611, 'n_words': 181}