<commit_before>import docker
import os, sys, time, subprocess, yaml, shutil, copy
import logging
import dockermix
from requests.exceptions import HTTPError

class ContainerMix:
  def __init__(self, conf_file=None, environment=None):
    self._setupLogging()
    self.containers = {}
    
    if environment:
      self.load(environment)
    else:
      if not conf_file.startswith('/'):
        conf_file = os.path.join(os.path.dirname(sys.argv[0]), conf_file)

      data = open(conf_file, 'r')
      self.config = yaml.load(data)      

  def get(self, container):
    return self.containers[container]

  def build(self):
    for container in self.config['containers']:
      #TODO: confirm base_image exists
      base = self.config['containers'][container]['base_image']
        
      self.log.info('Building container: %s using base template %s', container, base)
      
      build = Container(container, self.config['containers'][container])
      dockerfile = None
      if 'dockerfile' in self.config['containers'][container]:
        dockerfile = self.config['containers'][container]['dockerfile']
      build.build(dockerfile)

      self.containers[container] = build
      
  def destroy(self):
    for container in self.containers:
      self.log.info('Destroying container: %s', container)      
      self.containers[container].destroy()     

  def start(self):
    for container in self.containers:      
      self.containers[container].start()

  def stop(self):
    for container in self.containers:      
      self.containers[container].stop()

  def load(self, filename='envrionment.yml'):
    self.log.info('Loading environment from: %s', filename)      
    
    with open(filename, 'r') as input_file:
      environment = yaml.load(input_file)

      for container in environment['containers']:
        self.containers[container] = Container(container, environment['containers'][container])
    
  def save(self, filename='environment.yml'):
    self.log.info('Saving environment state to: %s', filename)      
      
    with open(filename, 'w') as output_file:
      output_file.write(self.dump())

  def status(self):
    columns = "{0:<14}{1:<19}{2:<34}{3:<11}\n" 
    result = columns.format("ID", "NODE", "COMMAND", "STATUS")
    for container in self.containers:
      container_id = self.containers[container].config['container_id']
      
      node_name = (container[:15] + '..') if len(container) > 17 else container

      command = ''
      status = 'OFF'
      try:
        state = docker.Client().inspect_container(container_id)
        command = "".join([state['Path']] + state['Args'] + [" "])
        command = (command[:30] + '..') if len(command) > 32 else command
        
        if state['State']['Running']:
          status = 'Running'
      except HTTPError:
        status = 'Destroyed'

      result += columns.format(container_id, node_name, command, status)

    return result

  def dump(self):
    result = {}
    result['containers'] = {}
    for container in self.containers:      
      output = result['containers'][container] = self.containers[container].config

    return yaml.dump(result, Dumper=yaml.SafeDumper)

  def _setupLogging(self):
    self.log = logging.getLogger('dockermix')
    self.log.setLevel(logging.DEBUG)

    formatter = logging.Formatter("%(asctime)s %(levelname)-10s %(message)s")
    filehandler = logging.FileHandler('dockermix.log', 'w')
    filehandler.setLevel(logging.DEBUG)
    filehandler.setFormatter(formatter)
    self.log.addHandler(filehandler)  


class Container:
  def __init__(self, name, config={}, build_tag=None):
    self.log = logging.getLogger('dockermix')

    self.config = config
    self.name = name
    
    self.build_tag = build_tag
    if not build_tag:
      self.build_tag = name + '-' + str(os.getpid())

    if 'command' not in self.config:
      # TODO: this should probably raise an error rather than default.
      print "Error: No command specified for container " + name
      self.log.error('No command specified in configuration')   
      exit(1)  
      #self.config['command'] = '/bin/true'
    
    self.docker_client = docker.Client()
    
    if 'base_image' not in self.config:
      self.config['base_image'] = 'ubuntu'
      
  def build(self, dockerfile=None):
    if dockerfile:        
      self._build_container(dockerfile)
    else:
      # If there's no dockerfile then we're just launching an empty base    
      self.config['image_id'] = self.config['base_image']
    
    self._start_container()
    
  def destroy(self):
    self.stop()
    self.docker_client.remove_container(self.config['container_id'])    
    self.docker_client.remove_image(self.build_tag)
<commit_msg>Adding some error conditions when data it left out of the specification<commit_after>import docker
import os, sys, time, subprocess, yaml, shutil, copy
import logging
import dockermix
from requests.exceptions import HTTPError

class ContainerMix:
  def __init__(self, conf_file=None, environment=None):
    self._setupLogging()
    self.containers = {}
    
    if environment:
      self.load(environment)
    else:
      if not conf_file.startswith('/'):
        conf_file = os.path.join(os.path.dirname(sys.argv[0]), conf_file)

      data = open(conf_file, 'r')
      self.config = yaml.load(data)      

  def get(self, container):
    return self.containers[container]

  def build(self):
    for container in self.config['containers']:
      #TODO: confirm base_image exists
      if not self.config['containers'][container]:
        sys.stderr.write("Error: no configuration found for container: " + container + "\n")
        exit(1)

      if 'base_image' not in self.config['containers'][container]:
        sys.stderr.write("Error: no base image specified for container: " + container)
        exit(1)

      base = self.config['containers'][container]['base_image']
        
      self.log.info('Building container: %s using base template %s', container, base)
      
      build = Container(container, self.config['containers'][container])
      dockerfile = None
      if 'dockerfile' in self.config['containers'][container]:
        dockerfile = self.config['containers'][container]['dockerfile']
      build.build(dockerfile)

      self.containers[container] = build
      
  def destroy(self):
    for container in self.containers:
      self.log.info('Destroying container: %s', container)      
      self.containers[container].destroy()     

  def start(self):
    for container in self.containers:      
      self.containers[container].start()

  def stop(self):
    for container in self.containers:      
      self.containers[container].stop()

  def load(self, filename='envrionment.yml'):
    self.log.info('Loading environment from: %s', filename)      
    
    with open(filename, 'r') as input_file:
      environment = yaml.load(input_file)

      for container in environment['containers']:
        self.containers[container] = Container(container, environment['containers'][container])
    
  def save(self, filename='environment.yml'):
    self.log.info('Saving environment state to: %s', filename)      
      
    with open(filename, 'w') as output_file:
      output_file.write(self.dump())

  def status(self):
    columns = "{0:<14}{1:<19}{2:<34}{3:<11}\n" 
    result = columns.format("ID", "NODE", "COMMAND", "STATUS")
    for container in self.containers:
      container_id = self.containers[container].config['container_id']
      
      node_name = (container[:15] + '..') if len(container) > 17 else container

      command = ''
      status = 'OFF'
      try:
        state = docker.Client().inspect_container(container_id)
        command = "".join([state['Path']] + state['Args'] + [" "])
        command = (command[:30] + '..') if len(command) > 32 else command
        
        if state['State']['Running']:
          status = 'Running'
      except HTTPError:
        status = 'Destroyed'

      result += columns.format(container_id, node_name, command, status)

    return result

  def dump(self):
    result = {}
    result['containers'] = {}
    for container in self.containers:      
      output = result['containers'][container] = self.containers[container].config

    return yaml.dump(result, Dumper=yaml.SafeDumper)

  def _setupLogging(self):
    self.log = logging.getLogger('dockermix')
    self.log.setLevel(logging.DEBUG)

    formatter = logging.Formatter("%(asctime)s %(levelname)-10s %(message)s")
    filehandler = logging.FileHandler('dockermix.log', 'w')
    filehandler.setLevel(logging.DEBUG)
    filehandler.setFormatter(formatter)
    self.log.addHandler(filehandler)  


class Container:
  def __init__(self, name, config={}, build_tag=None):
    self.log = logging.getLogger('dockermix')

    self.config = config
    self.name = name
    
    self.build_tag = build_tag
    if not build_tag:
      self.build_tag = name + '-' + str(os.getpid())

    if 'command' not in self.config:
      sys.stderr.write("Error: No command specified for container " + name + "\n")
      self.log.error('No command specified in configuration')   
      exit(1)  
      #self.config['command'] = '/bin/true'
    
    self.docker_client = docker.Client()
    
    if 'base_image' not in self.config:
      self.config['base_image'] = 'ubuntu'
      
  def build(self, dockerfile=None):
    if dockerfile:        
      self._build_container(dockerfile)
    else:
      # If there's no dockerfile then we're just launching an empty base    
      self.config['image_id'] = self.config['base_image']
    
    self._start_container()
    
  def destroy(self):
    self.stop()
    self.docker_client.remove_container(self.config['container_id'])    
    self.docker_client.remove_image(self.build_tag)

{'fraction_non_alphanumeric': 0.12997753726771494, 'fraction_numerical': 0.005105166428425566, 'mean_word_length': 3.0948996655518393, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7282550', 'n_tokens_mistral': 3009, 'n_tokens_neox': 2902, 'n_words': 651}