<commit_before>#!/usr/bin/env python

from docker.client import Client
from docker.utils import kwargs_from_env
from collections import defaultdict
import argparse
import os
import yaml


class Pipeline():
    def __init__(self, name, host=None, steps=[], debug=False, pull_images=True):
        if name is None:
            raise TypeError('Must provide a name for the pipeline')
        self.name = name
        self.host = host
        self.steps = steps
        self.debug = debug
        self.client = Pipeline.get_client()
        self.pull_images = pull_images
        self.remove_containers = False
        self.result = None

    def __unicode__(self):
        return u'<Pipeline: {} - steps: {} host: {}, debug: {} >'.format(self.name, len(self.steps), self.host, self.debug)

    def __str__(self):
        return unicode(self).encode('utf-8')

    @classmethod
    def from_dict(cls, pipeline_dict):
        pipeline_dict = defaultdict(lambda: None, pipeline_dict)
        step_dicts = pipeline_dict['steps']
        steps = list()
        for step_dict in step_dicts:
            step = Step.from_dict(step_dict)
            steps.append(step)
        name = pipeline_dict['name']
        host = pipeline_dict['host']
        return cls(name, host=host, steps=steps, debug=True)

    @classmethod
    def from_yaml(cls, file):
        with open(file, 'r') as yamlfile:
            pipeline_dict = yaml.load(yamlfile)
        return cls.from_dict(pipeline_dict)

    @classmethod
    def get_client(cls):
        # Using boot2docker instructions for now
        # http://docker-py.readthedocs.org/en/latest/boot2docker/
        # Workaround for requests.exceptions.SSLError: hostname '192.168.59.103' doesn't match 'boot2docker'
        client = Client(**kwargs_from_env(assert_hostname=False))
        return client

    def run(self):
        if self.debug:
            print "Running pipeline: {}".format(self)
        for each_step in self.steps:
            if self.pull_images:
                self.pull_image(each_step)
            container = self.create_container(each_step)
            self.start_container(container, each_step)
            self.result = self.get_result(container, each_step)
            self.finish_container(container, each_step)
            if self.result['code'] != 0:<commit_msg>Use debug and pull_images settings from dict/yaml<commit_after>#!/usr/bin/env python

from docker.client import Client
from docker.utils import kwargs_from_env
from collections import defaultdict
import argparse
import os
import yaml


class Pipeline():
    def __init__(self, name, host=None, steps=[], debug=False, pull_images=True):
        if name is None:
            raise TypeError('Must provide a name for the pipeline')
        self.name = name
        self.host = host
        self.steps = steps
        self.debug = debug
        self.client = Pipeline.get_client()
        self.pull_images = pull_images
        self.remove_containers = False
        self.result = None

    def __unicode__(self):
        return u'<Pipeline: {} - steps: {} host: {}, debug: {} >'.format(self.name, len(self.steps), self.host, self.debug)

    def __str__(self):
        return unicode(self).encode('utf-8')

    @classmethod
    def from_dict(cls, pipeline_dict):
        pipeline_dict = defaultdict(lambda: None, pipeline_dict)
        step_dicts = pipeline_dict['steps']
        steps = list()
        for step_dict in step_dicts:
            step = Step.from_dict(step_dict)
            steps.append(step)
        name = pipeline_dict['name']
        host = pipeline_dict['host']
        debug = pipeline_dict['debug'] or False
        pull_images = pipeline_dict['pull_images'] or False
        return cls(name, host=host, steps=steps, debug=debug, pull_images=pull_images)

    @classmethod
    def from_yaml(cls, file):
        with open(file, 'r') as yamlfile:
            pipeline_dict = yaml.load(yamlfile)
        return cls.from_dict(pipeline_dict)

    @classmethod
    def get_client(cls):
        # Using boot2docker instructions for now
        # http://docker-py.readthedocs.org/en/latest/boot2docker/
        # Workaround for requests.exceptions.SSLError: hostname '192.168.59.103' doesn't match 'boot2docker'
        client = Client(**kwargs_from_env(assert_hostname=False))
        return client

    def run(self):
        if self.debug:
            print "Running pipeline: {}".format(self)
        for each_step in self.steps:
            if self.pull_images:
                self.pull_image(each_step)
            container = self.create_container(each_step)
            self.start_container(container, each_step)
            self.result = self.get_result(container, each_step)
            self.finish_container(container, each_step)
            if self.result['code'] != 0:
{'fraction_non_alphanumeric': 0.10195182724252491, 'fraction_numerical': 0.006644518272425249, 'mean_word_length': 2.6028421839940163, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4061993', 'n_tokens_mistral': 1435, 'n_tokens_neox': 1384, 'n_words': 336}